andyjeffries-rubyrep 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +83 -0
- data/License.txt +20 -0
- data/Manifest.txt +151 -0
- data/README.txt +37 -0
- data/bin/rubyrep +8 -0
- data/lib/rubyrep.rb +72 -0
- data/lib/rubyrep/base_runner.rb +195 -0
- data/lib/rubyrep/command_runner.rb +144 -0
- data/lib/rubyrep/committers/buffered_committer.rb +151 -0
- data/lib/rubyrep/committers/committers.rb +152 -0
- data/lib/rubyrep/configuration.rb +275 -0
- data/lib/rubyrep/connection_extenders/connection_extenders.rb +165 -0
- data/lib/rubyrep/connection_extenders/jdbc_extender.rb +65 -0
- data/lib/rubyrep/connection_extenders/mysql_extender.rb +59 -0
- data/lib/rubyrep/connection_extenders/postgresql_extender.rb +277 -0
- data/lib/rubyrep/database_proxy.rb +52 -0
- data/lib/rubyrep/direct_table_scan.rb +75 -0
- data/lib/rubyrep/generate_runner.rb +105 -0
- data/lib/rubyrep/initializer.rb +39 -0
- data/lib/rubyrep/log_helper.rb +30 -0
- data/lib/rubyrep/logged_change.rb +160 -0
- data/lib/rubyrep/logged_change_loader.rb +197 -0
- data/lib/rubyrep/noisy_connection.rb +80 -0
- data/lib/rubyrep/proxied_table_scan.rb +171 -0
- data/lib/rubyrep/proxy_block_cursor.rb +145 -0
- data/lib/rubyrep/proxy_connection.rb +431 -0
- data/lib/rubyrep/proxy_cursor.rb +44 -0
- data/lib/rubyrep/proxy_row_cursor.rb +43 -0
- data/lib/rubyrep/proxy_runner.rb +89 -0
- data/lib/rubyrep/replication_difference.rb +100 -0
- data/lib/rubyrep/replication_extenders/mysql_replication.rb +271 -0
- data/lib/rubyrep/replication_extenders/postgresql_replication.rb +236 -0
- data/lib/rubyrep/replication_extenders/replication_extenders.rb +26 -0
- data/lib/rubyrep/replication_helper.rb +142 -0
- data/lib/rubyrep/replication_initializer.rb +327 -0
- data/lib/rubyrep/replication_run.rb +142 -0
- data/lib/rubyrep/replication_runner.rb +166 -0
- data/lib/rubyrep/replicators/replicators.rb +42 -0
- data/lib/rubyrep/replicators/two_way_replicator.rb +361 -0
- data/lib/rubyrep/scan_progress_printers/progress_bar.rb +65 -0
- data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb +65 -0
- data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb +111 -0
- data/lib/rubyrep/scan_report_printers/scan_report_printers.rb +67 -0
- data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb +75 -0
- data/lib/rubyrep/scan_runner.rb +25 -0
- data/lib/rubyrep/session.rb +230 -0
- data/lib/rubyrep/sync_helper.rb +121 -0
- data/lib/rubyrep/sync_runner.rb +31 -0
- data/lib/rubyrep/syncers/syncers.rb +112 -0
- data/lib/rubyrep/syncers/two_way_syncer.rb +174 -0
- data/lib/rubyrep/table_scan.rb +54 -0
- data/lib/rubyrep/table_scan_helper.rb +46 -0
- data/lib/rubyrep/table_sorter.rb +70 -0
- data/lib/rubyrep/table_spec_resolver.rb +142 -0
- data/lib/rubyrep/table_sync.rb +90 -0
- data/lib/rubyrep/task_sweeper.rb +77 -0
- data/lib/rubyrep/trigger_mode_switcher.rb +63 -0
- data/lib/rubyrep/type_casting_cursor.rb +31 -0
- data/lib/rubyrep/uninstall_runner.rb +93 -0
- data/lib/rubyrep/version.rb +9 -0
- data/rubyrep +8 -0
- data/rubyrep.bat +4 -0
- data/setup.rb +1585 -0
- data/spec/base_runner_spec.rb +218 -0
- data/spec/buffered_committer_spec.rb +274 -0
- data/spec/command_runner_spec.rb +145 -0
- data/spec/committers_spec.rb +178 -0
- data/spec/configuration_spec.rb +203 -0
- data/spec/connection_extender_interface_spec.rb +141 -0
- data/spec/connection_extenders_registration_spec.rb +164 -0
- data/spec/database_proxy_spec.rb +48 -0
- data/spec/database_rake_spec.rb +40 -0
- data/spec/db_specific_connection_extenders_spec.rb +34 -0
- data/spec/db_specific_replication_extenders_spec.rb +38 -0
- data/spec/direct_table_scan_spec.rb +61 -0
- data/spec/dolphins.jpg +0 -0
- data/spec/generate_runner_spec.rb +84 -0
- data/spec/initializer_spec.rb +46 -0
- data/spec/log_helper_spec.rb +39 -0
- data/spec/logged_change_loader_spec.rb +68 -0
- data/spec/logged_change_spec.rb +470 -0
- data/spec/noisy_connection_spec.rb +78 -0
- data/spec/postgresql_replication_spec.rb +48 -0
- data/spec/postgresql_schema_support_spec.rb +212 -0
- data/spec/postgresql_support_spec.rb +63 -0
- data/spec/progress_bar_spec.rb +77 -0
- data/spec/proxied_table_scan_spec.rb +151 -0
- data/spec/proxy_block_cursor_spec.rb +197 -0
- data/spec/proxy_connection_spec.rb +423 -0
- data/spec/proxy_cursor_spec.rb +56 -0
- data/spec/proxy_row_cursor_spec.rb +66 -0
- data/spec/proxy_runner_spec.rb +70 -0
- data/spec/replication_difference_spec.rb +161 -0
- data/spec/replication_extender_interface_spec.rb +367 -0
- data/spec/replication_extenders_spec.rb +32 -0
- data/spec/replication_helper_spec.rb +178 -0
- data/spec/replication_initializer_spec.rb +509 -0
- data/spec/replication_run_spec.rb +443 -0
- data/spec/replication_runner_spec.rb +254 -0
- data/spec/replicators_spec.rb +36 -0
- data/spec/rubyrep_spec.rb +8 -0
- data/spec/scan_detail_reporter_spec.rb +119 -0
- data/spec/scan_progress_printers_spec.rb +68 -0
- data/spec/scan_report_printers_spec.rb +67 -0
- data/spec/scan_runner_spec.rb +50 -0
- data/spec/scan_summary_reporter_spec.rb +61 -0
- data/spec/session_spec.rb +253 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +305 -0
- data/spec/strange_name_support_spec.rb +135 -0
- data/spec/sync_helper_spec.rb +169 -0
- data/spec/sync_runner_spec.rb +78 -0
- data/spec/syncers_spec.rb +171 -0
- data/spec/table_scan_helper_spec.rb +36 -0
- data/spec/table_scan_spec.rb +49 -0
- data/spec/table_sorter_spec.rb +30 -0
- data/spec/table_spec_resolver_spec.rb +111 -0
- data/spec/table_sync_spec.rb +140 -0
- data/spec/task_sweeper_spec.rb +47 -0
- data/spec/trigger_mode_switcher_spec.rb +83 -0
- data/spec/two_way_replicator_spec.rb +721 -0
- data/spec/two_way_syncer_spec.rb +256 -0
- data/spec/type_casting_cursor_spec.rb +50 -0
- data/spec/uninstall_runner_spec.rb +93 -0
- metadata +190 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
module RR
|
|
2
|
+
module ReplicationExtenders
|
|
3
|
+
|
|
4
|
+
# Provides PostgreSQL specific functionality for database replication
|
|
5
|
+
module PostgreSQLReplication
|
|
6
|
+
RR::ReplicationExtenders.register :postgresql => self
|
|
7
|
+
|
|
8
|
+
# Returns the key clause that is used in the trigger function.
|
|
9
|
+
# * +trigger_var+: should be either 'NEW' or 'OLD'
|
|
10
|
+
# * +params+: the parameter hash as described in #create_rep_trigger
|
|
11
|
+
def key_clause(trigger_var, params)
|
|
12
|
+
params[:keys].
|
|
13
|
+
map { |key| "'#{key}#{params[:key_sep]}' || #{trigger_var}.#{quote_column_name(key)}"}.
|
|
14
|
+
join(" || '#{params[:key_sep]}' || ")
|
|
15
|
+
end
|
|
16
|
+
private :key_clause
|
|
17
|
+
|
|
18
|
+
# Returns the schema prefix (including dot) that will be used by the
|
|
19
|
+
# triggers to write into the rubyrep infrastructure tables.
|
|
20
|
+
# To avoid setting the wrong prefix, it will only return a schema prefix
|
|
21
|
+
# if the current search path
|
|
22
|
+
# * consists of only a single schema
|
|
23
|
+
# * does not consists of a variable search path
|
|
24
|
+
# (i. e. the default "$user")
|
|
25
|
+
def schema_prefix
|
|
26
|
+
unless @schema_prefix
|
|
27
|
+
search_path = select_one("show search_path")['search_path']
|
|
28
|
+
if search_path =~ /[,$]/
|
|
29
|
+
@schema_prefix = ""
|
|
30
|
+
else
|
|
31
|
+
@schema_prefix = %("#{search_path}".)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
@schema_prefix
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Creates or replaces the replication trigger function.
|
|
38
|
+
# See #create_replication_trigger for a descriptions of the +params+ hash.
|
|
39
|
+
def create_or_replace_replication_trigger_function(params)
|
|
40
|
+
# first check, if PL/SQL is already activated and if not, do so.
|
|
41
|
+
if select_all("select lanname from pg_language where lanname = 'plpgsql'").empty?
|
|
42
|
+
execute "CREATE LANGUAGE plpgsql"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
activity_check = ""
|
|
46
|
+
if params[:exclude_rr_activity] then
|
|
47
|
+
activity_check = <<-end_sql
|
|
48
|
+
PERFORM ACTIVE FROM #{schema_prefix}#{params[:activity_table]};
|
|
49
|
+
IF FOUND THEN
|
|
50
|
+
RETURN NULL;
|
|
51
|
+
END IF;
|
|
52
|
+
end_sql
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
version_string = select_value("select version();")
|
|
56
|
+
version = version_string.gsub(/^\s*postgresql\s*([0-9.]+).*$/i, '\1')
|
|
57
|
+
if version >= '8.4'
|
|
58
|
+
modification_check = <<-end_sql
|
|
59
|
+
IF NEW IS NOT DISTINCT FROM OLD THEN
|
|
60
|
+
RETURN NULL;
|
|
61
|
+
END IF;
|
|
62
|
+
end_sql
|
|
63
|
+
else
|
|
64
|
+
modification_check = ""
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# now create the trigger
|
|
68
|
+
execute(<<-end_sql)
|
|
69
|
+
CREATE OR REPLACE FUNCTION "#{params[:trigger_name]}"() RETURNS TRIGGER AS $change_trigger$
|
|
70
|
+
BEGIN
|
|
71
|
+
#{activity_check}
|
|
72
|
+
IF (TG_OP = 'DELETE') THEN
|
|
73
|
+
INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
|
|
74
|
+
SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
|
|
75
|
+
ELSIF (TG_OP = 'UPDATE') THEN
|
|
76
|
+
#{modification_check}
|
|
77
|
+
INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
|
|
78
|
+
SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
|
|
79
|
+
ELSIF (TG_OP = 'INSERT') THEN
|
|
80
|
+
INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
|
|
81
|
+
SELECT '#{params[:table]}', #{key_clause('NEW', params)}, 'I', now();
|
|
82
|
+
END IF;
|
|
83
|
+
RETURN NULL; -- result is ignored since this is an AFTER trigger
|
|
84
|
+
END;
|
|
85
|
+
$change_trigger$ LANGUAGE plpgsql;
|
|
86
|
+
end_sql
|
|
87
|
+
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Creates a trigger to log all changes for the given table.
|
|
91
|
+
# +params+ is a hash with all necessary information:
|
|
92
|
+
# * :+trigger_name+: name of the trigger
|
|
93
|
+
# * :+table+: name of the table that should be monitored
|
|
94
|
+
# * :+keys+: array of names of the key columns of the monitored table
|
|
95
|
+
# * :+log_table+: name of the table receiving all change notifications
|
|
96
|
+
# * :+activity_table+: name of the table receiving the rubyrep activity information
|
|
97
|
+
# * :+key_sep+: column seperator to be used in the key column of the log table
|
|
98
|
+
# * :+exclude_rr_activity+:
|
|
99
|
+
# if true, the trigger will check and filter out changes initiated by RubyRep
|
|
100
|
+
def create_replication_trigger(params)
|
|
101
|
+
create_or_replace_replication_trigger_function params
|
|
102
|
+
|
|
103
|
+
execute(<<-end_sql)
|
|
104
|
+
CREATE TRIGGER "#{params[:trigger_name]}"
|
|
105
|
+
AFTER INSERT OR UPDATE OR DELETE ON "#{params[:table]}"
|
|
106
|
+
FOR EACH ROW EXECUTE PROCEDURE #{schema_prefix}"#{params[:trigger_name]}"();
|
|
107
|
+
end_sql
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Removes a trigger and related trigger procedure.
|
|
111
|
+
# * +trigger_name+: name of the trigger
|
|
112
|
+
# * +table_name+: name of the table for which the trigger exists
|
|
113
|
+
def drop_replication_trigger(trigger_name, table_name)
|
|
114
|
+
execute "DROP TRIGGER \"#{trigger_name}\" ON \"#{table_name}\";"
|
|
115
|
+
execute "DROP FUNCTION \"#{trigger_name}\"();"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Returns +true+ if the named trigger exists for the named table.
|
|
119
|
+
# * +trigger_name+: name of the trigger
|
|
120
|
+
# * +table_name+: name of the table
|
|
121
|
+
def replication_trigger_exists?(trigger_name, table_name)
|
|
122
|
+
!select_all(<<-end_sql).empty?
|
|
123
|
+
select 1 from information_schema.triggers
|
|
124
|
+
where event_object_schema in (#{schemas})
|
|
125
|
+
and trigger_name = '#{trigger_name}'
|
|
126
|
+
and event_object_table = '#{table_name}'
|
|
127
|
+
end_sql
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Returns all unadjusted sequences of the given table.
|
|
131
|
+
# Parameters:
|
|
132
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
|
133
|
+
# * +table_name+: name of the table
|
|
134
|
+
# Return value: a hash with
|
|
135
|
+
# * key: sequence name
|
|
136
|
+
# * value: a hash with
|
|
137
|
+
# * :+increment+: current sequence increment
|
|
138
|
+
# * :+value+: current value
|
|
139
|
+
def sequence_values(rep_prefix, table_name)
|
|
140
|
+
result = {}
|
|
141
|
+
sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
|
|
142
|
+
select s.relname
|
|
143
|
+
from pg_class as t
|
|
144
|
+
join pg_depend as r on t.oid = r.refobjid
|
|
145
|
+
join pg_class as s on r.objid = s.oid
|
|
146
|
+
and s.relkind = 'S'
|
|
147
|
+
and t.relname = '#{table_name}' AND t.relnamespace IN
|
|
148
|
+
(SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
|
|
149
|
+
end_sql
|
|
150
|
+
sequence_names.each do |sequence_name|
|
|
151
|
+
row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
|
|
152
|
+
result[sequence_name] = {
|
|
153
|
+
:increment => row['increment_by'].to_i,
|
|
154
|
+
:value => row['last_value'].to_i
|
|
155
|
+
}
|
|
156
|
+
end
|
|
157
|
+
result
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Ensures that the sequences of the named table (normally the primary key
|
|
161
|
+
# column) are generated with the correct increment and offset.
|
|
162
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
|
163
|
+
# * +table_name+: name of the table (not used for Postgres)
|
|
164
|
+
# * +increment+: increment of the sequence
|
|
165
|
+
# * +offset+: offset
|
|
166
|
+
# * +left_sequence_values+:
|
|
167
|
+
# hash as returned by #sequence_values for the left database
|
|
168
|
+
# * +right_sequence_values+:
|
|
169
|
+
# hash as returned by #sequence_values for the right database
|
|
170
|
+
# * +adjustment_buffer+:
|
|
171
|
+
# the "gap" that is created during sequence update to avoid concurrency problems
|
|
172
|
+
# E. g. an increment of 2 and offset of 1 will lead to generation of odd
|
|
173
|
+
# numbers.
|
|
174
|
+
def update_sequences(
|
|
175
|
+
rep_prefix, table_name, increment, offset,
|
|
176
|
+
left_sequence_values, right_sequence_values, adjustment_buffer)
|
|
177
|
+
left_sequence_values.each do |sequence_name, left_current_value|
|
|
178
|
+
row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
|
|
179
|
+
current_increment = row['increment_by'].to_i
|
|
180
|
+
current_value = row['last_value'].to_i
|
|
181
|
+
unless current_increment == increment and current_value % increment == offset
|
|
182
|
+
max_current_value =
|
|
183
|
+
[left_current_value[:value], right_sequence_values[sequence_name][:value]].max +
|
|
184
|
+
adjustment_buffer
|
|
185
|
+
new_start = max_current_value - (max_current_value % increment) + increment + offset
|
|
186
|
+
execute(<<-end_sql)
|
|
187
|
+
alter sequence "#{sequence_name}" increment by #{increment} restart with #{new_start}
|
|
188
|
+
end_sql
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Restores the original sequence settings.
|
|
194
|
+
# (Actually it sets the sequence increment to 1. If before, it had a
|
|
195
|
+
# different value, then the restoration will not be correct.)
|
|
196
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
|
197
|
+
# * +table_name+: name of the table
|
|
198
|
+
def clear_sequence_setup(rep_prefix, table_name)
|
|
199
|
+
sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
|
|
200
|
+
select s.relname
|
|
201
|
+
from pg_class as t
|
|
202
|
+
join pg_depend as r on t.oid = r.refobjid
|
|
203
|
+
join pg_class as s on r.objid = s.oid
|
|
204
|
+
and s.relkind = 'S'
|
|
205
|
+
and t.relname = '#{table_name}' and t.relnamespace IN
|
|
206
|
+
(SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
|
|
207
|
+
end_sql
|
|
208
|
+
sequence_names.each do |sequence_name|
|
|
209
|
+
execute(<<-end_sql)
|
|
210
|
+
alter sequence "#{sequence_name}" increment by 1
|
|
211
|
+
end_sql
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Adds a big (8 byte value), auto-incrementing primary key column to the
|
|
216
|
+
# specified table.
|
|
217
|
+
# * table_name: name of the target table
|
|
218
|
+
# * key_name: name of the primary key column
|
|
219
|
+
def add_big_primary_key(table_name, key_name)
|
|
220
|
+
old_message_level = select_one("show client_min_messages")['client_min_messages']
|
|
221
|
+
execute "set client_min_messages = warning"
|
|
222
|
+
execute(<<-end_sql)
|
|
223
|
+
alter table "#{table_name}" add column #{key_name} bigserial
|
|
224
|
+
end_sql
|
|
225
|
+
|
|
226
|
+
execute(<<-end_sql)
|
|
227
|
+
alter table "#{table_name}" add constraint #{table_name}_#{key_name}_pkey primary key (#{key_name})
|
|
228
|
+
end_sql
|
|
229
|
+
|
|
230
|
+
ensure
|
|
231
|
+
execute "set client_min_messages = #{old_message_level}"
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
module RR
|
|
2
|
+
|
|
3
|
+
# Replication extenders are modules that provide database specific functionality
|
|
4
|
+
# required for replication. They are mixed into ActiveRecord database connections.
|
|
5
|
+
# This module itself only provides functionality to register and retrieve
|
|
6
|
+
# such extenders.
|
|
7
|
+
module ReplicationExtenders
|
|
8
|
+
# Returns a Hash of currently registered replication extenders.
|
|
9
|
+
# (Empty Hash if no replication extenders were defined.)
|
|
10
|
+
def self.extenders
|
|
11
|
+
@extenders ||= {}
|
|
12
|
+
@extenders
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Registers one or multiple replication extender.
|
|
16
|
+
# extender is a Hash with
|
|
17
|
+
# key:: The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
|
|
18
|
+
# value:: Name of the module implementing the replication extender
|
|
19
|
+
def self.register(extender)
|
|
20
|
+
@extenders ||= {}
|
|
21
|
+
@extenders.merge! extender
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
module RR
|
|
2
|
+
|
|
3
|
+
# Provides helper functionality for replicators.
|
|
4
|
+
# The methods exposed by this class are intended to provide a stable interface
|
|
5
|
+
# for third party replicators.
|
|
6
|
+
class ReplicationHelper
|
|
7
|
+
|
|
8
|
+
include LogHelper
|
|
9
|
+
|
|
10
|
+
# The current +ReplicationRun+ instance
|
|
11
|
+
attr_accessor :replication_run
|
|
12
|
+
|
|
13
|
+
# The active +Session+
|
|
14
|
+
def session; replication_run.session; end
|
|
15
|
+
|
|
16
|
+
# Current options
|
|
17
|
+
def options; @options ||= session.configuration.options; end
|
|
18
|
+
|
|
19
|
+
# Returns the options for the specified table name.
|
|
20
|
+
# * +table+: name of the table (left database version)
|
|
21
|
+
def options_for_table(table)
|
|
22
|
+
@options_for_table ||= {}
|
|
23
|
+
unless @options_for_table.include? table
|
|
24
|
+
@options_for_table[table] = session.configuration.options_for_table(table)
|
|
25
|
+
end
|
|
26
|
+
@options_for_table[table]
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Delegates to Session#corresponding_table
|
|
30
|
+
def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
|
|
31
|
+
|
|
32
|
+
# Returns +true+ if a new transaction was started since the last
|
|
33
|
+
# insert / update / delete.
|
|
34
|
+
def new_transaction?
|
|
35
|
+
committer.new_transaction?
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Delegates to Committers::BufferedCommitter#insert_record
|
|
39
|
+
def insert_record(database, table, values)
|
|
40
|
+
committer.insert_record(database, table, values)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Delegates to Committers::BufferedCommitter#update_record
|
|
44
|
+
def update_record(database, table, values, old_key = nil)
|
|
45
|
+
committer.update_record(database, table, values, old_key)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Delegates to Committers::BufferedCommitter#delete_record
|
|
49
|
+
def delete_record(database, table, values)
|
|
50
|
+
committer.delete_record(database, table, values)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Loads the specified record. Returns an according column_name => value hash.
|
|
54
|
+
# Parameters:
|
|
55
|
+
# * +database+: either :+left+ or :+right+
|
|
56
|
+
# * +table+: name of the table
|
|
57
|
+
# * +key+: A column_name => value hash for all primary key columns.
|
|
58
|
+
def load_record(database, table, key)
|
|
59
|
+
cursor = session.send(database).select_cursor(
|
|
60
|
+
:table => table,
|
|
61
|
+
:row_keys => [key],
|
|
62
|
+
:type_cast => true
|
|
63
|
+
)
|
|
64
|
+
row = nil
|
|
65
|
+
row = cursor.next_row if cursor.next?
|
|
66
|
+
cursor.clear
|
|
67
|
+
row
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The current Committer
|
|
71
|
+
attr_reader :committer
|
|
72
|
+
private :committer
|
|
73
|
+
|
|
74
|
+
# Asks the committer (if it exists) to finalize any open transactions
|
|
75
|
+
# +success+ should be true if there were no problems, false otherwise.
|
|
76
|
+
def finalize(success = true)
|
|
77
|
+
committer.finalize(success)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Converts the row values into their proper types as per table definition.
|
|
81
|
+
# * +table+: name of the table after whose columns is type-casted.
|
|
82
|
+
# * +row+: A column_name => value hash of the row
|
|
83
|
+
# Returns a copy of the column_name => value hash (with type-casted values).
|
|
84
|
+
def type_cast(table, row)
|
|
85
|
+
@table_columns ||= {}
|
|
86
|
+
unless @table_columns.include?(table)
|
|
87
|
+
column_array = session.left.columns(table)
|
|
88
|
+
column_hash = {}
|
|
89
|
+
column_array.each {|column| column_hash[column.name] = column}
|
|
90
|
+
@table_columns[table] = column_hash
|
|
91
|
+
end
|
|
92
|
+
columns = @table_columns[table]
|
|
93
|
+
type_casted_row = {}
|
|
94
|
+
row.each_pair do |column_name, value|
|
|
95
|
+
type_casted_row[column_name] = columns[column_name].type_cast(value)
|
|
96
|
+
end
|
|
97
|
+
type_casted_row
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Logs the outcome of a replication into the replication log table.
|
|
101
|
+
# * +diff+: the replicated ReplicationDifference
|
|
102
|
+
# * +outcome+: string summarizing the outcome of the replication
|
|
103
|
+
# * +details+: string with further details regarding the replication
|
|
104
|
+
def log_replication_outcome(diff, outcome, details = nil)
|
|
105
|
+
table = diff.changes[:left].table
|
|
106
|
+
key = diff.changes[:left].key
|
|
107
|
+
if key.size == 1
|
|
108
|
+
key = key.values[0]
|
|
109
|
+
else
|
|
110
|
+
key_parts = session.left.primary_key_names(table).map do |column_name|
|
|
111
|
+
%Q("#{column_name}"=>#{key[column_name].to_s.inspect})
|
|
112
|
+
end
|
|
113
|
+
key = key_parts.join(', ')
|
|
114
|
+
end
|
|
115
|
+
rep_outcome, rep_details = fit_description_columns(outcome, details)
|
|
116
|
+
diff_dump = diff.to_yaml[0...ReplicationInitializer::DIFF_DUMP_SIZE]
|
|
117
|
+
|
|
118
|
+
session.left.insert_record "#{options[:rep_prefix]}_logged_events", {
|
|
119
|
+
:activity => 'replication',
|
|
120
|
+
:change_table => table,
|
|
121
|
+
:diff_type => diff.type.to_s,
|
|
122
|
+
:change_key => key,
|
|
123
|
+
:left_change_type => (diff.changes[:left] ? diff.changes[:left].type.to_s : nil),
|
|
124
|
+
:right_change_type => (diff.changes[:right] ? diff.changes[:right].type.to_s : nil),
|
|
125
|
+
:description => rep_outcome,
|
|
126
|
+
:long_description => rep_details,
|
|
127
|
+
:event_time => Time.now,
|
|
128
|
+
:diff_dump => diff_dump
|
|
129
|
+
}
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Creates a new SyncHelper for the given +TableSync+ instance.
|
|
133
|
+
def initialize(replication_run)
|
|
134
|
+
self.replication_run = replication_run
|
|
135
|
+
|
|
136
|
+
# Creates the committer. Important as it gives the committer the
|
|
137
|
+
# opportunity to start transactions
|
|
138
|
+
committer_class = Committers::committers[options[:committer]]
|
|
139
|
+
@committer = committer_class.new(session)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
|
|
2
|
+
|
|
3
|
+
require 'rubyrep'
|
|
4
|
+
|
|
5
|
+
module RR
|
|
6
|
+
|
|
7
|
+
# Ensures all preconditions are met to start with replication
|
|
8
|
+
class ReplicationInitializer
|
|
9
|
+
|
|
10
|
+
# The active Session
|
|
11
|
+
attr_accessor :session
|
|
12
|
+
|
|
13
|
+
# Creates a new RepInititializer for the given Session
|
|
14
|
+
def initialize(session)
|
|
15
|
+
self.session = session
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Returns the options for the given table.
|
|
19
|
+
# If table is +nil+, returns general options.
|
|
20
|
+
def options(table = nil)
|
|
21
|
+
if table
|
|
22
|
+
session.configuration.options_for_table table
|
|
23
|
+
else
|
|
24
|
+
session.configuration.options
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Creates a trigger logging all table changes
|
|
29
|
+
# * database: either :+left+ or :+right+
|
|
30
|
+
# * table: name of the table
|
|
31
|
+
def create_trigger(database, table)
|
|
32
|
+
options = self.options(table)
|
|
33
|
+
|
|
34
|
+
params = {
|
|
35
|
+
:trigger_name => "#{options[:rep_prefix]}_#{table}",
|
|
36
|
+
:table => table,
|
|
37
|
+
:keys => session.send(database).primary_key_names(table),
|
|
38
|
+
:log_table => "#{options[:rep_prefix]}_pending_changes",
|
|
39
|
+
:activity_table => "#{options[:rep_prefix]}_running_flags",
|
|
40
|
+
:key_sep => options[:key_sep],
|
|
41
|
+
:exclude_rr_activity => false,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
session.send(database).create_replication_trigger params
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Returns +true+ if the replication trigger for the given table exists.
|
|
48
|
+
# * database: either :+left+ or :+right+
|
|
49
|
+
# * table: name of the table
|
|
50
|
+
def trigger_exists?(database, table)
|
|
51
|
+
trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
|
|
52
|
+
session.send(database).replication_trigger_exists? trigger_name, table
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Drops the replication trigger of the named table.
|
|
56
|
+
# * database: either :+left+ or :+right+
|
|
57
|
+
# * table: name of the table
|
|
58
|
+
def drop_trigger(database, table)
|
|
59
|
+
trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
|
|
60
|
+
session.send(database).drop_replication_trigger trigger_name, table
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Ensures that the sequences of the named table (normally the primary key
|
|
64
|
+
# column) are generated with the correct increment and offset in both
|
|
65
|
+
# left and right database.
|
|
66
|
+
# The sequence is always updated in both databases.
|
|
67
|
+
# * +table_pair+: a hash of names of corresponding :left and :right tables
|
|
68
|
+
# * +increment+: increment of the sequence
|
|
69
|
+
# * +left_offset+: offset of table in left database
|
|
70
|
+
# * +right_offset+: offset of table in right database
|
|
71
|
+
# E. g. an increment of 2 and offset of 1 will lead to generation of odd
|
|
72
|
+
# numbers.
|
|
73
|
+
def ensure_sequence_setup(table_pair, increment, left_offset, right_offset)
|
|
74
|
+
table_options = options(table_pair[:left])
|
|
75
|
+
if table_options[:adjust_sequences]
|
|
76
|
+
rep_prefix = table_options[:rep_prefix]
|
|
77
|
+
left_sequence_values = session.left.sequence_values rep_prefix, table_pair[:left]
|
|
78
|
+
right_sequence_values = session.right.sequence_values rep_prefix, table_pair[:right]
|
|
79
|
+
[:left, :right].each do |database|
|
|
80
|
+
offset = database == :left ? left_offset : right_offset
|
|
81
|
+
session.send(database).update_sequences \
|
|
82
|
+
rep_prefix, table_pair[database], increment, offset,
|
|
83
|
+
left_sequence_values, right_sequence_values, table_options[:sequence_adjustment_buffer]
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Restores the original sequence settings for the named table.
|
|
89
|
+
# (Actually it sets the sequence increment to 1. If before, it had a
|
|
90
|
+
# different value, then the restoration will not be correct.)
|
|
91
|
+
# * database: either :+left+ or :+right+
|
|
92
|
+
# * +table_name+: name of the table
|
|
93
|
+
def clear_sequence_setup(database, table)
|
|
94
|
+
table_options = options(table)
|
|
95
|
+
if table_options[:adjust_sequences]
|
|
96
|
+
session.send(database).clear_sequence_setup(
|
|
97
|
+
table_options[:rep_prefix], table
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Returns +true+ if the change log exists in the specified database.
|
|
103
|
+
# * database: either :+left+ or :+right+
|
|
104
|
+
def change_log_exists?(database)
|
|
105
|
+
session.send(database).tables.include? "#{options[:rep_prefix]}_pending_changes"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Returns +true+ if the replication log exists.
|
|
109
|
+
def event_log_exists?
|
|
110
|
+
session.left.tables.include? "#{options[:rep_prefix]}_logged_events"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Drops the change log table in the specified database
|
|
114
|
+
# * database: either :+left+ or :+right+
|
|
115
|
+
def drop_change_log(database)
|
|
116
|
+
session.send(database).drop_table "#{options[:rep_prefix]}_pending_changes"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Drops the replication log table.
|
|
120
|
+
def drop_event_log
|
|
121
|
+
session.left.drop_table "#{options[:rep_prefix]}_logged_events"
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Size of the replication log column diff_dump
|
|
125
|
+
DIFF_DUMP_SIZE = 2000
|
|
126
|
+
|
|
127
|
+
# Size fo the event log column 'description'
|
|
128
|
+
DESCRIPTION_SIZE = 255
|
|
129
|
+
|
|
130
|
+
# Size of the event log column 'long_description'
|
|
131
|
+
LONG_DESCRIPTION_SIZE = 1000
|
|
132
|
+
|
|
133
|
+
# Ensures that create_table and related statements don't print notices to
|
|
134
|
+
# stdout. Then restored original message setting.
|
|
135
|
+
# * +database+: either :+left+ or :+right+
|
|
136
|
+
def silence_ddl_notices(database)
|
|
137
|
+
if session.configuration.send(database)[:adapter] =~ /postgres/
|
|
138
|
+
old_message_level = session.send(database).
|
|
139
|
+
select_one("show client_min_messages")['client_min_messages']
|
|
140
|
+
session.send(database).execute "set client_min_messages = warning"
|
|
141
|
+
end
|
|
142
|
+
yield
|
|
143
|
+
ensure
|
|
144
|
+
if session.configuration.send(database)[:adapter] =~ /postgres/
|
|
145
|
+
session.send(database).execute "set client_min_messages = #{old_message_level}"
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Creates the replication log table.
|
|
150
|
+
def create_event_log
|
|
151
|
+
silence_ddl_notices(:left) do
|
|
152
|
+
table_name = "#{options[:rep_prefix]}_logged_events"
|
|
153
|
+
session.left.create_table "#{options[:rep_prefix]}_logged_events"
|
|
154
|
+
session.left.add_column table_name, :activity, :string
|
|
155
|
+
session.left.add_column table_name, :change_table, :string
|
|
156
|
+
session.left.add_column table_name, :diff_type, :string
|
|
157
|
+
session.left.add_column table_name, :change_key, :string
|
|
158
|
+
session.left.add_column table_name, :left_change_type, :string
|
|
159
|
+
session.left.add_column table_name, :right_change_type, :string
|
|
160
|
+
session.left.add_column table_name, :description, :string, :limit => DESCRIPTION_SIZE
|
|
161
|
+
session.left.add_column table_name, :long_description, :string, :limit => LONG_DESCRIPTION_SIZE
|
|
162
|
+
session.left.add_column table_name, :event_time, :timestamp
|
|
163
|
+
session.left.add_column table_name, :diff_dump, :string, :limit => DIFF_DUMP_SIZE
|
|
164
|
+
session.left.remove_column table_name, 'id'
|
|
165
|
+
session.left.add_big_primary_key table_name, 'id'
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Creates the change log table in the specified database
|
|
170
|
+
# * database: either :+left+ or :+right+
|
|
171
|
+
def create_change_log(database)
|
|
172
|
+
silence_ddl_notices(database) do
|
|
173
|
+
connection = session.send(database)
|
|
174
|
+
table_name = "#{options[:rep_prefix]}_pending_changes"
|
|
175
|
+
connection.create_table table_name
|
|
176
|
+
connection.add_column table_name, :change_table, :string
|
|
177
|
+
connection.add_column table_name, :change_key, :string
|
|
178
|
+
connection.add_column table_name, :change_new_key, :string
|
|
179
|
+
connection.add_column table_name, :change_type, :string
|
|
180
|
+
connection.add_column table_name, :change_time, :timestamp
|
|
181
|
+
connection.remove_column table_name, 'id'
|
|
182
|
+
connection.add_big_primary_key table_name, 'id'
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Adds to the current session's configuration an exclusion of rubyrep tables.
|
|
187
|
+
def exclude_rubyrep_tables
|
|
188
|
+
session.configuration.exclude_rubyrep_tables
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Checks in both databases, if the activity marker tables exist and if not,
|
|
192
|
+
# creates them.
|
|
193
|
+
def ensure_activity_markers
|
|
194
|
+
table_name = "#{options[:rep_prefix]}_running_flags"
|
|
195
|
+
[:left, :right].each do |database|
|
|
196
|
+
connection = session.send(database)
|
|
197
|
+
unless connection.tables.include? table_name
|
|
198
|
+
silence_ddl_notices(database) do
|
|
199
|
+
connection.create_table table_name
|
|
200
|
+
connection.add_column table_name, :active, :integer
|
|
201
|
+
connection.remove_column table_name, 'id'
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Checks if the event log table already exists and creates it if necessary
|
|
208
|
+
def ensure_event_log
|
|
209
|
+
create_event_log unless event_log_exists?
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Checks in both databases, if the change log tables exists and creates them
|
|
213
|
+
# if necessary
|
|
214
|
+
def ensure_change_logs
|
|
215
|
+
[:left, :right].each do |database|
|
|
216
|
+
create_change_log(database) unless change_log_exists?(database)
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Checks in both databases, if the infrastructure tables (change log, event
|
|
221
|
+
# log) exist and creates them if necessary.
|
|
222
|
+
def ensure_infrastructure
|
|
223
|
+
ensure_activity_markers
|
|
224
|
+
ensure_change_logs
|
|
225
|
+
ensure_event_log
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Checks in both databases, if the change_log tables exist. If yes, drops them.
|
|
229
|
+
def drop_change_logs
|
|
230
|
+
[:left, :right].each do |database|
|
|
231
|
+
drop_change_log(database) if change_log_exists?(database)
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Checks in both databases, if the activity_marker tables exist. If yes, drops them.
|
|
236
|
+
def drop_activity_markers
|
|
237
|
+
table_name = "#{options[:rep_prefix]}_running_flags"
|
|
238
|
+
[:left, :right].each do |database|
|
|
239
|
+
if session.send(database).tables.include? table_name
|
|
240
|
+
session.send(database).drop_table table_name
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Removes all rubyrep infrastructure tables from both databases.
|
|
246
|
+
def drop_infrastructure
|
|
247
|
+
drop_event_log if event_log_exists?
|
|
248
|
+
drop_change_logs
|
|
249
|
+
drop_activity_markers
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Checks for tables that have triggers but are not in the list of configured
|
|
253
|
+
# tables. Removes triggers and restores sequences of those tables.
|
|
254
|
+
# * +configured_table_pairs+:
|
|
255
|
+
# An array of table pairs (e. g. [{:left => 'xy', :right => 'xy2'}]).
|
|
256
|
+
def restore_unconfigured_tables(configured_table_pairs = session.configured_table_pairs)
|
|
257
|
+
[:left, :right].each do |database|
|
|
258
|
+
configured_tables = configured_table_pairs.map {|table_pair| table_pair[database]}
|
|
259
|
+
unconfigured_tables = session.send(database).tables - configured_tables
|
|
260
|
+
unconfigured_tables.each do |table|
|
|
261
|
+
if trigger_exists?(database, table)
|
|
262
|
+
drop_trigger(database, table)
|
|
263
|
+
session.send(database).execute(
|
|
264
|
+
"delete from #{options[:rep_prefix]}_pending_changes where change_table = '#{table}'")
|
|
265
|
+
end
|
|
266
|
+
clear_sequence_setup(database, table)
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Calls the potentially provided :+after_init+ handler after infrastructure
|
|
272
|
+
# tables are created.
|
|
273
|
+
def call_after_infrastructure_setup_handler
|
|
274
|
+
handler = session.configuration.options[:after_infrastructure_setup]
|
|
275
|
+
handler.call(session) if handler
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Prepares the database / tables for replication.
|
|
279
|
+
def prepare_replication
|
|
280
|
+
exclude_rubyrep_tables
|
|
281
|
+
|
|
282
|
+
puts "Verifying RubyRep tables"
|
|
283
|
+
ensure_infrastructure
|
|
284
|
+
|
|
285
|
+
call_after_infrastructure_setup_handler
|
|
286
|
+
|
|
287
|
+
puts "Checking for and removing rubyrep triggers from unconfigured tables"
|
|
288
|
+
restore_unconfigured_tables
|
|
289
|
+
|
|
290
|
+
puts "Verifying rubyrep triggers of configured tables"
|
|
291
|
+
unsynced_table_pairs = []
|
|
292
|
+
table_pairs = session.sort_table_pairs(session.configured_table_pairs)
|
|
293
|
+
table_pairs.each do |table_pair|
|
|
294
|
+
table_options = options(table_pair[:left])
|
|
295
|
+
ensure_sequence_setup table_pair,
|
|
296
|
+
table_options[:sequence_increment],
|
|
297
|
+
table_options[:left_sequence_offset],
|
|
298
|
+
table_options[:right_sequence_offset]
|
|
299
|
+
|
|
300
|
+
unsynced = false
|
|
301
|
+
[:left, :right].each do |database|
|
|
302
|
+
unless trigger_exists? database, table_pair[database]
|
|
303
|
+
create_trigger database, table_pair[database]
|
|
304
|
+
unsynced = true
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
if unsynced and table_options[:initial_sync]
|
|
308
|
+
unsynced_table_pairs << table_pair
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
unsynced_table_specs = unsynced_table_pairs.map do |table_pair|
|
|
312
|
+
"#{table_pair[:left]}, #{table_pair[:right]}"
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
unless unsynced_table_specs.empty?
|
|
316
|
+
puts "Executing initial table syncs"
|
|
317
|
+
runner = SyncRunner.new
|
|
318
|
+
runner.session = session
|
|
319
|
+
runner.options = {:table_specs => unsynced_table_specs}
|
|
320
|
+
runner.execute
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
puts "Starting replication"
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
end
|