arcopy 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/COPYING +18 -0
- data/HACKING +61 -0
- data/README.md +285 -0
- data/Rakefile +16 -0
- data/bin/replicate +100 -0
- data/lib/replicate.rb +24 -0
- data/lib/replicate/active_record.rb +347 -0
- data/lib/replicate/dumper.rb +142 -0
- data/lib/replicate/emitter.rb +54 -0
- data/lib/replicate/loader.rb +157 -0
- data/lib/replicate/object.rb +57 -0
- data/lib/replicate/status.rb +54 -0
- data/test/active_record_test.rb +589 -0
- data/test/dumper_test.rb +108 -0
- data/test/dumpscript.rb +1 -0
- data/test/linked_dumpscript.rb +1 -0
- data/test/loader_test.rb +93 -0
- data/test/replicate_test.rb +10 -0
- metadata +135 -0
@@ -0,0 +1,54 @@
|
|
1
|
+
module Replicate
|
2
|
+
# Base class for Dumper / Loader classes. Manages a list of callback listeners
|
3
|
+
# and dispatches to each when #emit is called.
|
4
|
+
class Emitter
|
5
|
+
# Yields self to the block and calls #complete when block is finished.
|
6
|
+
def initialize
|
7
|
+
@listeners = []
|
8
|
+
if block_given?
|
9
|
+
yield self
|
10
|
+
complete
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Register a listener to be called for each loaded object with the
|
15
|
+
# type, id, attributes, object structure. Listeners are executed in the
|
16
|
+
# reverse order of which they were registered. Listeners registered later
|
17
|
+
# modify the view of listeners registered earlier.
|
18
|
+
#
|
19
|
+
# p - An optional Proc object. Must respond to call.
|
20
|
+
# block - An optional block.
|
21
|
+
#
|
22
|
+
# Returns nothing.
|
23
|
+
def listen(p=nil, &block)
|
24
|
+
@listeners.unshift p if p
|
25
|
+
@listeners.unshift block if block
|
26
|
+
end
|
27
|
+
|
28
|
+
# Sugar for creating a listener with an object instance. Instances of the
|
29
|
+
# class must respond to call(type, id, attrs, object).
|
30
|
+
#
|
31
|
+
# klass - The class to create. Must respond to new.
|
32
|
+
# args - Arguments to pass to new in addition to self.
|
33
|
+
#
|
34
|
+
# Returns the object created.
|
35
|
+
def use(klass, *args, &block)
|
36
|
+
instance = klass.new(self, *args, &block)
|
37
|
+
listen instance
|
38
|
+
instance
|
39
|
+
end
|
40
|
+
|
41
|
+
# Emit an object event to each listener.
|
42
|
+
#
|
43
|
+
# Returns the object.
|
44
|
+
def emit(type, id, attributes, object)
|
45
|
+
@listeners.each { |p| p.call(type, id, attributes, object) }
|
46
|
+
object
|
47
|
+
end
|
48
|
+
|
49
|
+
# Notify all listeners that processing is complete.
|
50
|
+
def complete
|
51
|
+
@listeners.each { |p| p.complete if p.respond_to?(:complete) }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
module Replicate
|
2
|
+
# Load replicants in a streaming fashion.
|
3
|
+
#
|
4
|
+
# The Loader reads [type, id, attributes] replicant tuples and creates
|
5
|
+
# objects in the current environment.
|
6
|
+
#
|
7
|
+
# Objects are expected to arrive in order such that a record referenced via
|
8
|
+
# foreign key always precedes the referencing record. The Loader maintains a
|
9
|
+
# mapping of primary keys from the dump system to the current environment.
|
10
|
+
# This mapping is used to properly establish new foreign key values on all
|
11
|
+
# records inserted.
|
12
|
+
class Loader < Emitter
|
13
|
+
|
14
|
+
# Stats hash.
|
15
|
+
attr_reader :stats
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
@keymap = Hash.new { |hash,k| hash[k] = {} }
|
19
|
+
@stats = Hash.new { |hash,k| hash[k] = 0 }
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
# Register a filter to write status information to the given stream. By
|
24
|
+
# default, a single line is used to report object counts while the dump is
|
25
|
+
# in progress; dump counts for each class are written when complete. The
|
26
|
+
# verbose and quiet options can be used to increase or decrease
|
27
|
+
# verbository.
|
28
|
+
#
|
29
|
+
# out - An IO object to write to, like stderr.
|
30
|
+
# verbose - Whether verbose output should be enabled.
|
31
|
+
# quiet - Whether quiet output should be enabled.
|
32
|
+
#
|
33
|
+
# Returns the Replicate::Status object.
|
34
|
+
def log_to(out=$stderr, verbose=false, quiet=false)
|
35
|
+
use Replicate::Status, 'load', out, verbose, quiet
|
36
|
+
end
|
37
|
+
|
38
|
+
# Feed a single replicant tuple into the loader.
|
39
|
+
#
|
40
|
+
# type - The class to create. Must respond to load_replicant.
|
41
|
+
# id - The remote system's id for this object.
|
42
|
+
# attrs - Hash of primitively typed objects.
|
43
|
+
#
|
44
|
+
# Returns the need object resulting from the load operation.
|
45
|
+
def feed(type, id, attrs)
|
46
|
+
type = type.to_s
|
47
|
+
object = load(type, id, attrs)
|
48
|
+
@stats[type] += 1
|
49
|
+
emit type, id, attrs, object
|
50
|
+
end
|
51
|
+
|
52
|
+
# Read multiple [type, id, attrs] replicant tuples from io and call the
|
53
|
+
# feed method to load and filter the object.
|
54
|
+
def read(io)
|
55
|
+
ActiveRecord::Base.transaction do
|
56
|
+
begin
|
57
|
+
if ActiveRecord::Base.connection.instance_values["config"][:adapter] == "postgresql"
|
58
|
+
ActiveRecord::Base.connection.execute("set constraints all deferred")
|
59
|
+
end
|
60
|
+
while object = Marshal.load(io)
|
61
|
+
type, id, attrs = object
|
62
|
+
feed type, id, attrs
|
63
|
+
end
|
64
|
+
rescue EOFError
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Load an individual replicant into the underlying datastore.
|
70
|
+
#
|
71
|
+
# type - Model class name as a String.
|
72
|
+
# id - Primary key id of the record on the dump system. This must be
|
73
|
+
# translated to the local system and stored in the keymap.
|
74
|
+
# attrs - Hash of attributes to set on the new record.
|
75
|
+
#
|
76
|
+
# Returns the new object instance.
|
77
|
+
def load(type, id, attributes)
|
78
|
+
model_class = constantize(type)
|
79
|
+
translate_ids type, id, attributes
|
80
|
+
begin
|
81
|
+
new_id, instance = model_class.load_replicant(type, id, attributes)
|
82
|
+
rescue => boom
|
83
|
+
warn "error: loading #{type} #{id} #{boom.class} #{boom}"
|
84
|
+
raise
|
85
|
+
end
|
86
|
+
register_id instance, type, id, new_id
|
87
|
+
instance
|
88
|
+
end
|
89
|
+
|
90
|
+
# Translate remote system id references in the attributes hash to their
|
91
|
+
# local system id values. The attributes hash may include special id
|
92
|
+
# values like this:
|
93
|
+
# { 'title' => 'hello there',
|
94
|
+
# 'repository_id' => [:id, 'Repository', 1234],
|
95
|
+
# 'label_ids' => [:id, 'Label', [333, 444, 555, 666, ...]]
|
96
|
+
# ... }
|
97
|
+
# These values are translated to local system ids. All object
|
98
|
+
# references must be loaded prior to the referencing object.
|
99
|
+
def translate_ids(type, id, attributes)
|
100
|
+
attributes.each do |key, value|
|
101
|
+
next unless value.is_a?(Array) && value[0] == :id
|
102
|
+
referenced_type, value = value[1].to_s, value[2]
|
103
|
+
local_ids =
|
104
|
+
Array(value).map do |remote_id|
|
105
|
+
if local_id = @keymap[referenced_type][remote_id]
|
106
|
+
local_id
|
107
|
+
else
|
108
|
+
warn "warn: #{referenced_type}(#{remote_id}) not in keymap, " +
|
109
|
+
"referenced by #{type}(#{id})##{key}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
if value.is_a?(Array)
|
113
|
+
attributes[key] = local_ids
|
114
|
+
else
|
115
|
+
attributes[key] = local_ids[0]
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
# Register an id in the keymap. Every object loaded must be stored here so
|
121
|
+
# that key references can be resolved.
|
122
|
+
def register_id(object, type, remote_id, local_id)
|
123
|
+
@keymap[type.to_s][remote_id] = local_id
|
124
|
+
c = object.class
|
125
|
+
while !['Object', 'ActiveRecord::Base'].include?(c.name)
|
126
|
+
@keymap[c.name][remote_id] = local_id
|
127
|
+
c = c.superclass
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
# Turn a string into an object by traversing constants. Identical to
|
132
|
+
# ActiveSupport's String#constantize implementation.
|
133
|
+
if Module.method(:const_get).arity == 1
|
134
|
+
# 1.8 implementation doesn't have the inherit argument to const_defined?
|
135
|
+
def constantize(string)
|
136
|
+
string.split('::').inject ::Object do |namespace, name|
|
137
|
+
if namespace.const_defined?(name)
|
138
|
+
namespace.const_get(name)
|
139
|
+
else
|
140
|
+
namespace.const_missing(name)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
else
|
145
|
+
# 1.9 implement has the inherit argument to const_defined?. Use it!
|
146
|
+
def constantize(string)
|
147
|
+
string.split('::').inject ::Object do |namespace, name|
|
148
|
+
if namespace.const_defined?(name, false)
|
149
|
+
namespace.const_get(name)
|
150
|
+
else
|
151
|
+
namespace.const_missing(name)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Replicate
|
2
|
+
# Simple OpenStruct style object that supports the dump and load protocols.
|
3
|
+
# Useful in tests and also when you want to dump an object that doesn't
|
4
|
+
# implement the dump and load methods.
|
5
|
+
#
|
6
|
+
# >> object = Replicate::Object.new :name => 'Joe', :age => 24
|
7
|
+
# >> object.age
|
8
|
+
# >> 24
|
9
|
+
# >> object.attributes
|
10
|
+
# { 'name' => 'Joe', 'age' => 24 }
|
11
|
+
#
|
12
|
+
class Object
|
13
|
+
attr_accessor :id
|
14
|
+
attr_accessor :attributes
|
15
|
+
|
16
|
+
def initialize(id=nil, attributes={})
|
17
|
+
attributes, id = id, nil if id.is_a?(Hash)
|
18
|
+
@id = id || self.class.generate_id
|
19
|
+
self.attributes = attributes
|
20
|
+
end
|
21
|
+
|
22
|
+
def attributes=(hash)
|
23
|
+
@attributes = {}
|
24
|
+
hash.each { |key, value| write_attribute key, value }
|
25
|
+
end
|
26
|
+
|
27
|
+
def [](key)
|
28
|
+
@attributes[key.to_s]
|
29
|
+
end
|
30
|
+
|
31
|
+
def []=(key, value)
|
32
|
+
@attributes[key.to_s] = value
|
33
|
+
end
|
34
|
+
|
35
|
+
def write_attribute(key, value)
|
36
|
+
meta = (class<<self;self;end)
|
37
|
+
meta.send(:define_method, key) { value }
|
38
|
+
meta.send(:define_method, "#{key}=") { |val| write_attribute(key, val) }
|
39
|
+
@attributes[key.to_s] = value
|
40
|
+
value
|
41
|
+
end
|
42
|
+
|
43
|
+
def dump_replicant(dumper, opts={})
|
44
|
+
dumper.write self.class, @id, @attributes, self
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.load_replicant(type, id, attrs)
|
48
|
+
object = new(generate_id, attrs)
|
49
|
+
[object.id, object]
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.generate_id
|
53
|
+
@last_id ||= 0
|
54
|
+
@last_id += 1
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Replicate
|
2
|
+
# Replicate filter that writes info to the console. Used by the Dumper and
|
3
|
+
# Loader to get basic console output.
|
4
|
+
class Status
|
5
|
+
def initialize(dumper, prefix, out, verbose=false, quiet=false)
|
6
|
+
@dumper = dumper
|
7
|
+
@prefix = prefix
|
8
|
+
@out = out
|
9
|
+
@verbose = verbose
|
10
|
+
@quiet = quiet
|
11
|
+
@count = 0
|
12
|
+
end
|
13
|
+
|
14
|
+
def call(type, id, attrs, object)
|
15
|
+
@count += 1
|
16
|
+
if @verbose
|
17
|
+
verbose_log type, id, attrs, object
|
18
|
+
elsif !@quiet
|
19
|
+
normal_log type, id, attrs, object
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def verbose_log(type, id, attrs, object)
|
24
|
+
desc_attr = %w[name login email number title].find { |k| attrs.key?(k) }
|
25
|
+
desc = desc_attr ? attrs[desc_attr] : id
|
26
|
+
@out.puts "#{@prefix}: %-30s %s" % [type.sub('Replicate::', ''), desc]
|
27
|
+
end
|
28
|
+
|
29
|
+
def normal_log(type, id, attrs, object)
|
30
|
+
@out.write " %sing: %4d objects \r" % [@prefix, @count]
|
31
|
+
end
|
32
|
+
|
33
|
+
def complete
|
34
|
+
dump_stats if !@quiet
|
35
|
+
end
|
36
|
+
|
37
|
+
def dump_stats(stats=@dumper.stats.dup)
|
38
|
+
@out.puts "==> #{@prefix}ed #{@count} total objects: "
|
39
|
+
width = 0
|
40
|
+
stats.keys.each do |key|
|
41
|
+
class_name = format_class_name(key)
|
42
|
+
stats[class_name] = stats.delete(key)
|
43
|
+
width = class_name.size if class_name.size > width
|
44
|
+
end
|
45
|
+
stats.to_a.sort_by { |k,n| k }.each do |class_name, count|
|
46
|
+
@out.write "%-#{width + 1}s %5d\n" % [class_name, count]
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def format_class_name(class_name)
|
51
|
+
class_name.sub(/Replicate::/, '')
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,589 @@
|
|
1
|
+
$VERBOSE = nil
|
2
|
+
require_relative 'test_helper'
|
3
|
+
require 'minitest/around/unit'
|
4
|
+
require 'stringio'
|
5
|
+
require 'active_record'
|
6
|
+
require 'active_record/version'
|
7
|
+
version = ActiveRecord::VERSION::STRING
|
8
|
+
warn "Using activerecord #{version}"
|
9
|
+
require 'replicate'
|
10
|
+
|
11
|
+
# create the sqlite db on disk
|
12
|
+
dbfile = File.expand_path('../db', __FILE__)
|
13
|
+
File.unlink dbfile if File.exist?(dbfile)
|
14
|
+
ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => dbfile)
|
15
|
+
# load schema
|
16
|
+
ActiveRecord::Migration.verbose = false
|
17
|
+
ActiveRecord::Schema.define do
|
18
|
+
create_table "users", :force => true do |t|
|
19
|
+
t.string "login"
|
20
|
+
t.datetime "created_at"
|
21
|
+
t.datetime "updated_at"
|
22
|
+
end
|
23
|
+
|
24
|
+
create_table "profiles", :force => true do |t|
|
25
|
+
t.integer "user_id"
|
26
|
+
t.string "name"
|
27
|
+
t.string "homepage"
|
28
|
+
end
|
29
|
+
|
30
|
+
create_table "emails", :force => true do |t|
|
31
|
+
t.integer "user_id"
|
32
|
+
t.string "email"
|
33
|
+
t.datetime "created_at"
|
34
|
+
end
|
35
|
+
|
36
|
+
if version[0,3] > '2.2'
|
37
|
+
create_table "domains", :force => true do |t|
|
38
|
+
t.string "host"
|
39
|
+
end
|
40
|
+
|
41
|
+
create_table "web_pages", :force => true do |t|
|
42
|
+
t.string "url"
|
43
|
+
t.string "domain_host"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
create_table "notes", :force => true do |t|
|
48
|
+
t.integer "notable_id"
|
49
|
+
t.string "notable_type"
|
50
|
+
end
|
51
|
+
|
52
|
+
create_table "namespaced", :force => true
|
53
|
+
end
|
54
|
+
|
55
|
+
# models
|
56
|
+
class User < ActiveRecord::Base
|
57
|
+
has_one :profile, :dependent => :destroy
|
58
|
+
has_many :emails, -> { order('id') }, :dependent => :destroy
|
59
|
+
has_many :notes, :as => :notable
|
60
|
+
replicate_natural_key :login
|
61
|
+
end
|
62
|
+
|
63
|
+
class Profile < ActiveRecord::Base
|
64
|
+
belongs_to :user
|
65
|
+
replicate_natural_key :user_id
|
66
|
+
end
|
67
|
+
|
68
|
+
class Email < ActiveRecord::Base
|
69
|
+
belongs_to :user
|
70
|
+
replicate_natural_key :user_id, :email
|
71
|
+
end
|
72
|
+
|
73
|
+
class WebPage < ActiveRecord::Base
|
74
|
+
belongs_to :domain, :foreign_key => 'domain_host', :primary_key => 'host'
|
75
|
+
end
|
76
|
+
|
77
|
+
class Domain < ActiveRecord::Base
|
78
|
+
replicate_natural_key :host
|
79
|
+
end
|
80
|
+
|
81
|
+
class Note < ActiveRecord::Base
|
82
|
+
belongs_to :notable, :polymorphic => true
|
83
|
+
end
|
84
|
+
|
85
|
+
class User::Namespaced < ActiveRecord::Base
|
86
|
+
self.table_name = "namespaced"
|
87
|
+
end
|
88
|
+
|
89
|
+
# The test case loads some fixture data once and uses transaction rollback to
|
90
|
+
# reset fixture state for each test's setup.
|
91
|
+
class ActiveRecordTest < Minitest::Test
|
92
|
+
i_suck_and_my_tests_are_order_dependent!
|
93
|
+
|
94
|
+
def around(&block)
|
95
|
+
ActiveRecord::Base.connection.transaction do
|
96
|
+
self.class.fixtures
|
97
|
+
|
98
|
+
@rtomayko = User.find_by_login('rtomayko')
|
99
|
+
@kneath = User.find_by_login('kneath')
|
100
|
+
@tmm1 = User.find_by_login('tmm1')
|
101
|
+
|
102
|
+
User.replicate_associations = []
|
103
|
+
|
104
|
+
@dumper = Replicate::Dumper.new
|
105
|
+
@loader = Replicate::Loader.new
|
106
|
+
yield
|
107
|
+
raise ActiveRecord::Rollback
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.fixtures
|
112
|
+
user = User.create! :login => 'rtomayko'
|
113
|
+
user.create_profile :name => 'Ryan Tomayko', :homepage => 'http://tomayko.com'
|
114
|
+
user.emails.create! :email => 'ryan@github.com'
|
115
|
+
user.emails.create! :email => 'rtomayko@gmail.com'
|
116
|
+
|
117
|
+
user = User.create! :login => 'kneath'
|
118
|
+
user.create_profile :name => 'Kyle Neath', :homepage => 'http://warpspire.com'
|
119
|
+
user.emails.create! :email => 'kyle@github.com'
|
120
|
+
|
121
|
+
user = User.create! :login => 'tmm1'
|
122
|
+
user.create_profile :name => 'tmm1', :homepage => 'https://github.com/tmm1'
|
123
|
+
|
124
|
+
github = Domain.create! :host => 'github.com'
|
125
|
+
WebPage.create! :url => 'http://github.com/about', :domain => github
|
126
|
+
end
|
127
|
+
|
128
|
+
def test_extension_modules_loaded
|
129
|
+
assert User.respond_to?(:load_replicant)
|
130
|
+
assert User.new.respond_to?(:dump_replicant)
|
131
|
+
end
|
132
|
+
|
133
|
+
def test_auto_dumping_belongs_to_associations
|
134
|
+
objects = []
|
135
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
136
|
+
|
137
|
+
rtomayko = User.find_by_login('rtomayko')
|
138
|
+
@dumper.dump rtomayko&.profile
|
139
|
+
|
140
|
+
assert_equal 2, objects.size
|
141
|
+
|
142
|
+
type, id, attrs, obj = objects.shift
|
143
|
+
assert_equal 'User', type
|
144
|
+
assert_equal rtomayko.id, id
|
145
|
+
assert_equal 'rtomayko', attrs['login']
|
146
|
+
assert_equal rtomayko.created_at, attrs['created_at']
|
147
|
+
assert_equal rtomayko, obj
|
148
|
+
|
149
|
+
type, id, attrs, obj = objects.shift
|
150
|
+
assert_equal 'Profile', type
|
151
|
+
assert_equal rtomayko.profile.id, id
|
152
|
+
assert_equal 'Ryan Tomayko', attrs['name']
|
153
|
+
assert_equal rtomayko.profile, obj
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_omit_dumping_of_attribute
|
157
|
+
objects = []
|
158
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
159
|
+
|
160
|
+
User.replicate_omit_attributes :created_at
|
161
|
+
rtomayko = User.find_by_login('rtomayko')
|
162
|
+
@dumper.dump rtomayko
|
163
|
+
|
164
|
+
assert_equal 2, objects.size
|
165
|
+
|
166
|
+
_type, _id, attrs, _obj = objects.shift
|
167
|
+
assert_nil attrs['created_at']
|
168
|
+
end
|
169
|
+
|
170
|
+
def test_omit_dumping_of_association
|
171
|
+
objects = []
|
172
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
173
|
+
|
174
|
+
User.replicate_omit_attributes :profile
|
175
|
+
rtomayko = User.find_by_login('rtomayko')
|
176
|
+
@dumper.dump rtomayko
|
177
|
+
|
178
|
+
assert_equal 1, objects.size
|
179
|
+
|
180
|
+
type, _id, _attrs, _obj = objects.shift
|
181
|
+
assert_equal 'User', type
|
182
|
+
end
|
183
|
+
|
184
|
+
if ActiveRecord::VERSION::STRING[0, 3] > '2.2'
|
185
|
+
def test_dump_and_load_non_standard_foreign_key_association
|
186
|
+
objects = []
|
187
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
188
|
+
|
189
|
+
github_about_page = WebPage.find_by_url('http://github.com/about')
|
190
|
+
assert_equal "github.com", github_about_page&.domain&.host
|
191
|
+
@dumper.dump github_about_page
|
192
|
+
|
193
|
+
WebPage.delete_all
|
194
|
+
Domain.delete_all
|
195
|
+
|
196
|
+
# load everything back up
|
197
|
+
objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
|
198
|
+
|
199
|
+
github_about_page = WebPage.find_by_url('http://github.com/about')
|
200
|
+
assert_equal "github.com", github_about_page.domain_host
|
201
|
+
assert_equal "github.com", github_about_page.domain.host
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def test_auto_dumping_has_one_associations
|
206
|
+
objects = []
|
207
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
208
|
+
|
209
|
+
rtomayko = User.find_by_login('rtomayko')
|
210
|
+
@dumper.dump rtomayko
|
211
|
+
|
212
|
+
assert_equal 2, objects.size
|
213
|
+
|
214
|
+
type, id, attrs, obj = objects.shift
|
215
|
+
assert_equal 'User', type
|
216
|
+
assert_equal rtomayko.id, id
|
217
|
+
assert_equal 'rtomayko', attrs['login']
|
218
|
+
assert_equal rtomayko.created_at, attrs['created_at']
|
219
|
+
assert_equal rtomayko, obj
|
220
|
+
|
221
|
+
type, id, attrs, obj = objects.shift
|
222
|
+
assert_equal 'Profile', type
|
223
|
+
assert_equal rtomayko.profile.id, id
|
224
|
+
assert_equal 'Ryan Tomayko', attrs['name']
|
225
|
+
assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
|
226
|
+
assert_equal rtomayko.profile, obj
|
227
|
+
end
|
228
|
+
|
229
|
+
def test_auto_dumping_does_not_fail_on_polymorphic_associations
|
230
|
+
objects = []
|
231
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
232
|
+
|
233
|
+
rtomayko = User.find_by_login('rtomayko')
|
234
|
+
note = Note.create!(:notable => rtomayko)
|
235
|
+
@dumper.dump note
|
236
|
+
|
237
|
+
assert_equal 3, objects.size
|
238
|
+
|
239
|
+
type, id, attrs, obj = objects.shift
|
240
|
+
assert_equal 'User', type
|
241
|
+
assert_equal rtomayko.id, id
|
242
|
+
|
243
|
+
type, id, attrs, obj = objects.shift
|
244
|
+
assert_equal 'Profile', type
|
245
|
+
|
246
|
+
type, id, attrs, obj = objects.shift
|
247
|
+
assert_equal 'Note', type
|
248
|
+
assert_equal note.id, id
|
249
|
+
assert_equal note.notable_type, attrs['notable_type']
|
250
|
+
assert_equal attrs["notable_id"], [:id, 'User', rtomayko.id]
|
251
|
+
assert_equal note, obj
|
252
|
+
end
|
253
|
+
|
254
|
+
def test_dumping_has_many_associations
|
255
|
+
objects = []
|
256
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
257
|
+
|
258
|
+
User.replicate_associations :emails
|
259
|
+
rtomayko = User.find_by_login('rtomayko')
|
260
|
+
@dumper.dump rtomayko
|
261
|
+
|
262
|
+
assert_equal 4, objects.size
|
263
|
+
|
264
|
+
type, id, attrs, obj = objects.shift
|
265
|
+
assert_equal 'User', type
|
266
|
+
assert_equal rtomayko.id, id
|
267
|
+
assert_equal 'rtomayko', attrs['login']
|
268
|
+
assert_equal rtomayko.created_at, attrs['created_at']
|
269
|
+
assert_equal rtomayko, obj
|
270
|
+
|
271
|
+
type, id, attrs, obj = objects.shift
|
272
|
+
assert_equal 'Profile', type
|
273
|
+
assert_equal rtomayko.profile.id, id
|
274
|
+
assert_equal 'Ryan Tomayko', attrs['name']
|
275
|
+
assert_equal rtomayko.profile, obj
|
276
|
+
|
277
|
+
type, id, attrs, obj = objects.shift
|
278
|
+
assert_equal 'Email', type
|
279
|
+
assert_equal 'ryan@github.com', attrs['email']
|
280
|
+
assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
|
281
|
+
assert_equal rtomayko.emails.first, obj
|
282
|
+
|
283
|
+
type, id, attrs, obj = objects.shift
|
284
|
+
assert_equal 'Email', type
|
285
|
+
assert_equal 'rtomayko@gmail.com', attrs['email']
|
286
|
+
assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
|
287
|
+
assert_equal rtomayko.emails.last, obj
|
288
|
+
end
|
289
|
+
|
290
|
+
def test_dumping_associations_at_dump_time
|
291
|
+
objects = []
|
292
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
293
|
+
|
294
|
+
rtomayko = User.find_by_login('rtomayko')
|
295
|
+
@dumper.dump rtomayko, :associations => [:emails], :omit => [:profile]
|
296
|
+
|
297
|
+
assert_equal 3, objects.size
|
298
|
+
|
299
|
+
type, id, attrs, obj = objects.shift
|
300
|
+
assert_equal 'User', type
|
301
|
+
assert_equal rtomayko.id, id
|
302
|
+
assert_equal 'rtomayko', attrs['login']
|
303
|
+
assert_equal rtomayko.created_at, attrs['created_at']
|
304
|
+
assert_equal rtomayko, obj
|
305
|
+
|
306
|
+
type, id, attrs, obj = objects.shift
|
307
|
+
assert_equal 'Email', type
|
308
|
+
assert_equal 'ryan@github.com', attrs['email']
|
309
|
+
assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
|
310
|
+
assert_equal rtomayko.emails.first, obj
|
311
|
+
|
312
|
+
type, id, attrs, obj = objects.shift
|
313
|
+
assert_equal 'Email', type
|
314
|
+
assert_equal 'rtomayko@gmail.com', attrs['email']
|
315
|
+
assert_equal [:id, 'User', rtomayko.id], attrs['user_id']
|
316
|
+
assert_equal rtomayko.emails.last, obj
|
317
|
+
end
|
318
|
+
|
319
|
+
def test_dumping_many_associations_at_dump_time
|
320
|
+
objects = []
|
321
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
322
|
+
|
323
|
+
users = User.where(:login => %w[rtomayko kneath])
|
324
|
+
@dumper.dump users, :associations => [:emails], :omit => [:profile]
|
325
|
+
|
326
|
+
assert_equal 5, objects.size
|
327
|
+
assert_equal ['Email', 'Email', 'Email', 'User', 'User'], objects.map { |type,_,_| type }.sort
|
328
|
+
end
|
329
|
+
|
330
|
+
def test_omit_attributes_at_dump_time
|
331
|
+
objects = []
|
332
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
333
|
+
|
334
|
+
rtomayko = User.find_by_login('rtomayko')
|
335
|
+
@dumper.dump rtomayko, :omit => [:created_at]
|
336
|
+
|
337
|
+
type, _id, attrs, _obj = objects.shift
|
338
|
+
assert_equal 'User', type
|
339
|
+
assert attrs['updated_at']
|
340
|
+
assert_nil attrs['created_at']
|
341
|
+
end
|
342
|
+
|
343
|
+
def test_dumping_polymorphic_associations
|
344
|
+
objects = []
|
345
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
346
|
+
|
347
|
+
User.replicate_associations :notes
|
348
|
+
rtomayko = User.find_by_login('rtomayko')
|
349
|
+
note = Note.create!(:notable => rtomayko)
|
350
|
+
@dumper.dump rtomayko
|
351
|
+
|
352
|
+
assert_equal 3, objects.size
|
353
|
+
|
354
|
+
type, id, attrs, obj = objects.shift
|
355
|
+
assert_equal 'User', type
|
356
|
+
assert_equal rtomayko.id, id
|
357
|
+
assert_equal 'rtomayko', attrs['login']
|
358
|
+
assert_equal rtomayko.created_at, attrs['created_at']
|
359
|
+
assert_equal rtomayko, obj
|
360
|
+
|
361
|
+
type, id, attrs, obj = objects.shift
|
362
|
+
assert_equal 'Profile', type
|
363
|
+
assert_equal rtomayko.profile.id, id
|
364
|
+
assert_equal 'Ryan Tomayko', attrs['name']
|
365
|
+
assert_equal rtomayko.profile, obj
|
366
|
+
|
367
|
+
type, id, attrs, obj = objects.shift
|
368
|
+
assert_equal 'Note', type
|
369
|
+
assert_equal note.notable_type, attrs['notable_type']
|
370
|
+
assert_equal [:id, 'User', rtomayko.id], attrs['notable_id']
|
371
|
+
assert_equal rtomayko.notes.first, obj
|
372
|
+
|
373
|
+
end
|
374
|
+
|
375
|
+
def test_dumping_empty_polymorphic_associations
|
376
|
+
objects = []
|
377
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
378
|
+
|
379
|
+
note = Note.create!()
|
380
|
+
@dumper.dump note
|
381
|
+
|
382
|
+
assert_equal 1, objects.size
|
383
|
+
|
384
|
+
type, _id, attrs, _obj = objects.shift
|
385
|
+
assert_equal 'Note', type
|
386
|
+
assert_nil attrs['notable_type']
|
387
|
+
assert_nil attrs['notable_id']
|
388
|
+
end
|
389
|
+
|
390
|
+
def test_dumps_polymorphic_namespaced_associations
|
391
|
+
objects = []
|
392
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
393
|
+
|
394
|
+
note = Note.create! :notable => User::Namespaced.create!
|
395
|
+
@dumper.dump note
|
396
|
+
|
397
|
+
assert_equal 2, objects.size
|
398
|
+
|
399
|
+
type, id, attrs, obj = objects.shift
|
400
|
+
assert_equal 'User::Namespaced', type
|
401
|
+
|
402
|
+
type, id, attrs, obj = objects.shift
|
403
|
+
assert_equal 'Note', type
|
404
|
+
end
|
405
|
+
|
406
|
+
def test_skips_belongs_to_information_if_omitted
|
407
|
+
objects = []
|
408
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
409
|
+
|
410
|
+
Profile.replicate_omit_attributes :user
|
411
|
+
@dumper.dump @rtomayko&.profile
|
412
|
+
|
413
|
+
assert_equal 1, objects.size
|
414
|
+
_type, _id, attrs, _obj = objects.shift
|
415
|
+
assert_equal @rtomayko.profile.user_id, attrs["user_id"]
|
416
|
+
end
|
417
|
+
|
418
|
+
def test_loading_everything
|
419
|
+
objects = []
|
420
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
421
|
+
|
422
|
+
# dump all users and associated objects and destroy
|
423
|
+
User.replicate_associations :emails
|
424
|
+
dumped_users = {}
|
425
|
+
%w[rtomayko kneath tmm1].each do |login|
|
426
|
+
user = User.find_by_login(login)
|
427
|
+
@dumper.dump user
|
428
|
+
user&.destroy
|
429
|
+
dumped_users[login] = user
|
430
|
+
end
|
431
|
+
assert_equal 9, objects.size
|
432
|
+
|
433
|
+
# insert another record to ensure id changes for loaded records
|
434
|
+
sr = User.create!(:login => 'sr')
|
435
|
+
sr.create_profile :name => 'Simon Rozet'
|
436
|
+
sr.emails.create :email => 'sr@github.com'
|
437
|
+
|
438
|
+
# load everything back up
|
439
|
+
objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
|
440
|
+
|
441
|
+
# verify attributes are set perfectly again
|
442
|
+
user = User.find_by_login('rtomayko')
|
443
|
+
assert_equal 'rtomayko', user.login
|
444
|
+
assert_equal dumped_users['rtomayko'].created_at, user.created_at
|
445
|
+
assert_equal dumped_users['rtomayko'].updated_at, user.updated_at
|
446
|
+
assert_equal 'Ryan Tomayko', user.profile.name
|
447
|
+
assert_equal 2, user.emails.size
|
448
|
+
|
449
|
+
# make sure everything was recreated
|
450
|
+
%w[rtomayko kneath tmm1].each do |login|
|
451
|
+
user = User.find_by_login(login)
|
452
|
+
assert user
|
453
|
+
assert user&.profile
|
454
|
+
assert !user.emails.empty?, "#{login} has no emails" if login != 'tmm1'
|
455
|
+
end
|
456
|
+
end
|
457
|
+
|
458
|
+
def test_loading_with_existing_records
|
459
|
+
objects = []
|
460
|
+
@dumper.listen { |type, id, attrs, obj| objects << [type, id, attrs, obj] }
|
461
|
+
|
462
|
+
# dump all users and associated objects and destroy
|
463
|
+
User.replicate_associations :emails
|
464
|
+
dumped_users = {}
|
465
|
+
%w[rtomayko kneath tmm1].each do |login|
|
466
|
+
user = User.find_by_login(login)
|
467
|
+
user&.profile&.update_attribute :name, 'CHANGED'
|
468
|
+
@dumper.dump user
|
469
|
+
dumped_users[login] = user
|
470
|
+
end
|
471
|
+
assert_equal 9, objects.size
|
472
|
+
|
473
|
+
# load everything back up
|
474
|
+
objects.each { |type, id, attrs, obj| @loader.feed type, id, attrs }
|
475
|
+
|
476
|
+
# ensure additional objects were not created
|
477
|
+
assert_equal 3, User.count
|
478
|
+
|
479
|
+
# verify attributes are set perfectly again
|
480
|
+
user = User.find_by_login('rtomayko')
|
481
|
+
assert_equal 'rtomayko', user.login
|
482
|
+
assert_equal dumped_users['rtomayko'].created_at, user.created_at
|
483
|
+
assert_equal dumped_users['rtomayko'].updated_at, user.updated_at
|
484
|
+
assert_equal 'CHANGED', user.profile.name
|
485
|
+
assert_equal 2, user.emails.size
|
486
|
+
|
487
|
+
# make sure everything was recreated
|
488
|
+
%w[rtomayko kneath tmm1].each do |login|
|
489
|
+
user = User.find_by_login(login)
|
490
|
+
assert user
|
491
|
+
assert user&.profile
|
492
|
+
assert_equal 'CHANGED', user.profile.name
|
493
|
+
assert !user.emails.empty?, "#{login} has no emails" if login != 'tmm1'
|
494
|
+
end
|
495
|
+
end
|
496
|
+
|
497
|
+
def test_loading_with_replicating_id
|
498
|
+
objects = []
|
499
|
+
@dumper.listen do |type, id, attrs, obj|
|
500
|
+
objects << [type, id, attrs, obj] if type == 'User'
|
501
|
+
end
|
502
|
+
|
503
|
+
dumped_users = {}
|
504
|
+
%w[rtomayko kneath tmm1].each do |login|
|
505
|
+
user = User.find_by_login(login)
|
506
|
+
@dumper.dump user
|
507
|
+
dumped_users[login] = user
|
508
|
+
end
|
509
|
+
assert_equal 3, objects.size
|
510
|
+
|
511
|
+
User.destroy_all
|
512
|
+
User.replicate_id = false
|
513
|
+
|
514
|
+
# load everything back up
|
515
|
+
objects.each { |type, id, attrs, obj| User.load_replicant type, id, attrs }
|
516
|
+
|
517
|
+
user = User.find_by_login('rtomayko')
|
518
|
+
assert_operator dumped_users['rtomayko'].id, :!=, user.id
|
519
|
+
|
520
|
+
User.destroy_all
|
521
|
+
User.replicate_id = true
|
522
|
+
|
523
|
+
# load everything back up
|
524
|
+
objects.each { |type, id, attrs, obj| User.load_replicant type, id, attrs }
|
525
|
+
|
526
|
+
user = User.find_by_login('rtomayko')
|
527
|
+
assert_equal dumped_users['rtomayko'].id, user.id
|
528
|
+
end
|
529
|
+
|
530
|
+
def test_loader_saves_without_validations
|
531
|
+
# note when a record is saved with validations
|
532
|
+
ran_validations = false
|
533
|
+
User.class_eval { validate { ran_validations = true } }
|
534
|
+
|
535
|
+
# check our assumptions
|
536
|
+
user = User.create(:login => 'defunkt')
|
537
|
+
assert ran_validations, "should run validations here"
|
538
|
+
ran_validations = false
|
539
|
+
|
540
|
+
# load one and verify validations are not run
|
541
|
+
user = nil
|
542
|
+
@loader.listen { |type, id, attrs, obj| user = obj }
|
543
|
+
@loader.feed 'User', 1, 'login' => 'rtomayko'
|
544
|
+
assert user
|
545
|
+
assert !ran_validations, 'validations should not run on save'
|
546
|
+
end
|
547
|
+
|
548
|
+
def test_loader_saves_without_callbacks
|
549
|
+
# note when a record is saved with callbacks
|
550
|
+
callbacks = false
|
551
|
+
User.class_eval { after_save { callbacks = true } }
|
552
|
+
User.class_eval { after_create { callbacks = true } }
|
553
|
+
User.class_eval { after_update { callbacks = true } }
|
554
|
+
User.class_eval { after_commit { callbacks = true } }
|
555
|
+
|
556
|
+
# check our assumptions
|
557
|
+
user = User.create(:login => 'defunkt')
|
558
|
+
assert callbacks, "should run callbacks here"
|
559
|
+
callbacks = false
|
560
|
+
|
561
|
+
# load one and verify validations are not run
|
562
|
+
user = nil
|
563
|
+
@loader.listen { |type, id, attrs, obj| user = obj }
|
564
|
+
@loader.feed 'User', 1, 'login' => 'rtomayko'
|
565
|
+
assert user
|
566
|
+
assert !callbacks, 'callbacks should not run on save'
|
567
|
+
end
|
568
|
+
|
569
|
+
def test_loader_saves_without_updating_created_at_timestamp
|
570
|
+
timestamp = Time.at((Time.now - (24 * 60 * 60)).to_i)
|
571
|
+
user = nil
|
572
|
+
@loader.listen { |type, id, attrs, obj| user = obj }
|
573
|
+
@loader.feed 'User', 23, 'login' => 'brianmario', 'created_at' => timestamp
|
574
|
+
assert_equal timestamp, user.created_at
|
575
|
+
user = User.find(user.id)
|
576
|
+
assert_equal timestamp, user.created_at
|
577
|
+
end
|
578
|
+
|
579
|
+
def test_loader_saves_without_updating_updated_at_timestamp
|
580
|
+
timestamp = Time.at((Time.now - (24 * 60 * 60)).to_i)
|
581
|
+
user = nil
|
582
|
+
@loader.listen { |type, id, attrs, obj| user = obj }
|
583
|
+
@loader.feed 'User', 29, 'login' => 'rtomayko', 'updated_at' => timestamp
|
584
|
+
assert_equal timestamp, user.updated_at
|
585
|
+
user = User.find(user.id)
|
586
|
+
assert_equal timestamp, user.updated_at
|
587
|
+
end
|
588
|
+
|
589
|
+
end
|