rubyrep 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +4 -0
- data/License.txt +20 -0
- data/Manifest.txt +137 -0
- data/README.txt +37 -0
- data/Rakefile +30 -0
- data/bin/rubyrep +8 -0
- data/config/hoe.rb +72 -0
- data/config/mysql_config.rb +25 -0
- data/config/postgres_config.rb +21 -0
- data/config/proxied_test_config.rb +14 -0
- data/config/redmine_config.rb +17 -0
- data/config/rep_config.rb +20 -0
- data/config/requirements.rb +32 -0
- data/config/test_config.rb +20 -0
- data/lib/rubyrep/base_runner.rb +195 -0
- data/lib/rubyrep/command_runner.rb +144 -0
- data/lib/rubyrep/committers/buffered_committer.rb +140 -0
- data/lib/rubyrep/committers/committers.rb +146 -0
- data/lib/rubyrep/configuration.rb +240 -0
- data/lib/rubyrep/connection_extenders/connection_extenders.rb +133 -0
- data/lib/rubyrep/connection_extenders/jdbc_extender.rb +284 -0
- data/lib/rubyrep/connection_extenders/mysql_extender.rb +168 -0
- data/lib/rubyrep/connection_extenders/postgresql_extender.rb +261 -0
- data/lib/rubyrep/database_proxy.rb +52 -0
- data/lib/rubyrep/direct_table_scan.rb +75 -0
- data/lib/rubyrep/generate_runner.rb +105 -0
- data/lib/rubyrep/initializer.rb +39 -0
- data/lib/rubyrep/logged_change.rb +326 -0
- data/lib/rubyrep/proxied_table_scan.rb +171 -0
- data/lib/rubyrep/proxy_block_cursor.rb +145 -0
- data/lib/rubyrep/proxy_connection.rb +318 -0
- data/lib/rubyrep/proxy_cursor.rb +44 -0
- data/lib/rubyrep/proxy_row_cursor.rb +43 -0
- data/lib/rubyrep/proxy_runner.rb +89 -0
- data/lib/rubyrep/replication_difference.rb +91 -0
- data/lib/rubyrep/replication_extenders/mysql_replication.rb +271 -0
- data/lib/rubyrep/replication_extenders/postgresql_replication.rb +204 -0
- data/lib/rubyrep/replication_extenders/replication_extenders.rb +26 -0
- data/lib/rubyrep/replication_helper.rb +104 -0
- data/lib/rubyrep/replication_initializer.rb +307 -0
- data/lib/rubyrep/replication_run.rb +48 -0
- data/lib/rubyrep/replication_runner.rb +138 -0
- data/lib/rubyrep/replicators/replicators.rb +37 -0
- data/lib/rubyrep/replicators/two_way_replicator.rb +334 -0
- data/lib/rubyrep/scan_progress_printers/progress_bar.rb +65 -0
- data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb +65 -0
- data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb +111 -0
- data/lib/rubyrep/scan_report_printers/scan_report_printers.rb +67 -0
- data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb +75 -0
- data/lib/rubyrep/scan_runner.rb +25 -0
- data/lib/rubyrep/session.rb +177 -0
- data/lib/rubyrep/sync_helper.rb +111 -0
- data/lib/rubyrep/sync_runner.rb +31 -0
- data/lib/rubyrep/syncers/syncers.rb +112 -0
- data/lib/rubyrep/syncers/two_way_syncer.rb +174 -0
- data/lib/rubyrep/table_scan.rb +54 -0
- data/lib/rubyrep/table_scan_helper.rb +38 -0
- data/lib/rubyrep/table_sorter.rb +70 -0
- data/lib/rubyrep/table_spec_resolver.rb +136 -0
- data/lib/rubyrep/table_sync.rb +68 -0
- data/lib/rubyrep/trigger_mode_switcher.rb +63 -0
- data/lib/rubyrep/type_casting_cursor.rb +31 -0
- data/lib/rubyrep/uninstall_runner.rb +92 -0
- data/lib/rubyrep/version.rb +9 -0
- data/lib/rubyrep.rb +68 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/sims/performance/big_rep_spec.rb +100 -0
- data/sims/performance/big_scan_spec.rb +57 -0
- data/sims/performance/big_sync_spec.rb +141 -0
- data/sims/performance/performance.rake +228 -0
- data/sims/sim_helper.rb +24 -0
- data/spec/base_runner_spec.rb +218 -0
- data/spec/buffered_committer_spec.rb +271 -0
- data/spec/command_runner_spec.rb +145 -0
- data/spec/committers_spec.rb +174 -0
- data/spec/configuration_spec.rb +198 -0
- data/spec/connection_extender_interface_spec.rb +138 -0
- data/spec/connection_extenders_registration_spec.rb +129 -0
- data/spec/database_proxy_spec.rb +48 -0
- data/spec/database_rake_spec.rb +40 -0
- data/spec/db_specific_connection_extenders_spec.rb +34 -0
- data/spec/db_specific_replication_extenders_spec.rb +38 -0
- data/spec/direct_table_scan_spec.rb +61 -0
- data/spec/generate_runner_spec.rb +84 -0
- data/spec/initializer_spec.rb +46 -0
- data/spec/logged_change_spec.rb +480 -0
- data/spec/postgresql_replication_spec.rb +48 -0
- data/spec/postgresql_support_spec.rb +57 -0
- data/spec/progress_bar_spec.rb +77 -0
- data/spec/proxied_table_scan_spec.rb +151 -0
- data/spec/proxy_block_cursor_spec.rb +197 -0
- data/spec/proxy_connection_spec.rb +399 -0
- data/spec/proxy_cursor_spec.rb +56 -0
- data/spec/proxy_row_cursor_spec.rb +66 -0
- data/spec/proxy_runner_spec.rb +70 -0
- data/spec/replication_difference_spec.rb +160 -0
- data/spec/replication_extender_interface_spec.rb +365 -0
- data/spec/replication_extenders_spec.rb +32 -0
- data/spec/replication_helper_spec.rb +121 -0
- data/spec/replication_initializer_spec.rb +477 -0
- data/spec/replication_run_spec.rb +166 -0
- data/spec/replication_runner_spec.rb +213 -0
- data/spec/replicators_spec.rb +31 -0
- data/spec/rubyrep_spec.rb +8 -0
- data/spec/scan_detail_reporter_spec.rb +119 -0
- data/spec/scan_progress_printers_spec.rb +68 -0
- data/spec/scan_report_printers_spec.rb +67 -0
- data/spec/scan_runner_spec.rb +50 -0
- data/spec/scan_summary_reporter_spec.rb +61 -0
- data/spec/session_spec.rb +212 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +295 -0
- data/spec/sync_helper_spec.rb +157 -0
- data/spec/sync_runner_spec.rb +78 -0
- data/spec/syncers_spec.rb +171 -0
- data/spec/table_scan_helper_spec.rb +29 -0
- data/spec/table_scan_spec.rb +49 -0
- data/spec/table_sorter_spec.rb +31 -0
- data/spec/table_spec_resolver_spec.rb +102 -0
- data/spec/table_sync_spec.rb +84 -0
- data/spec/trigger_mode_switcher_spec.rb +83 -0
- data/spec/two_way_replicator_spec.rb +551 -0
- data/spec/two_way_syncer_spec.rb +256 -0
- data/spec/type_casting_cursor_spec.rb +50 -0
- data/spec/uninstall_runner_spec.rb +86 -0
- data/tasks/database.rake +439 -0
- data/tasks/deployment.rake +29 -0
- data/tasks/environment.rake +9 -0
- data/tasks/java.rake +37 -0
- data/tasks/redmine_test.rake +47 -0
- data/tasks/rspec.rake +68 -0
- data/tasks/rubyrep.tailor +18 -0
- data/tasks/stats.rake +19 -0
- data/tasks/task_helper.rb +20 -0
- data.tar.gz.sig +0 -0
- metadata +243 -0
- metadata.gz.sig +0 -0
@@ -0,0 +1,271 @@
|
|
1
|
+
module RR
|
2
|
+
module ReplicationExtenders
|
3
|
+
|
4
|
+
# Provides Mysql specific functionality for database replication
|
5
|
+
module MysqlReplication
|
6
|
+
RR::ReplicationExtenders.register :mysql => self
|
7
|
+
|
8
|
+
# Creates or replaces the replication trigger function.
|
9
|
+
# See #create_replication_trigger for a descriptions of the +params+ hash.
|
10
|
+
def create_or_replace_replication_trigger_function(params)
|
11
|
+
execute(<<-end_sql)
|
12
|
+
DROP PROCEDURE IF EXISTS #{params[:trigger_name]};
|
13
|
+
end_sql
|
14
|
+
|
15
|
+
activity_check = ""
|
16
|
+
if params[:exclude_rr_activity] then
|
17
|
+
activity_check = <<-end_sql
|
18
|
+
DECLARE active INT;
|
19
|
+
SELECT count(*) INTO active FROM #{params[:activity_table]};
|
20
|
+
IF active <> 0 THEN
|
21
|
+
LEAVE p;
|
22
|
+
END IF;
|
23
|
+
end_sql
|
24
|
+
end
|
25
|
+
|
26
|
+
execute(<<-end_sql)
|
27
|
+
CREATE PROCEDURE #{params[:trigger_name]}(change_key varchar(2000), change_new_key varchar(2000), change_type varchar(1))
|
28
|
+
p: BEGIN
|
29
|
+
#{activity_check}
|
30
|
+
INSERT INTO #{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
|
31
|
+
VALUES('#{params[:table]}', change_key, change_new_key, change_type, now());
|
32
|
+
END;
|
33
|
+
end_sql
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
# Returns the key clause that is used in the trigger function.
|
38
|
+
# * +trigger_var+: should be either 'NEW' or 'OLD'
|
39
|
+
# * +params+: the parameter hash as described in #create_rep_trigger
|
40
|
+
def key_clause(trigger_var, params)
|
41
|
+
"concat_ws('#{params[:key_sep]}', " +
|
42
|
+
params[:keys].map { |key| "'#{key}', #{trigger_var}.#{key}"}.join(", ") +
|
43
|
+
")"
|
44
|
+
end
|
45
|
+
private :key_clause
|
46
|
+
|
47
|
+
# Creates a trigger to log all changes for the given table.
|
48
|
+
# +params+ is a hash with all necessary information:
|
49
|
+
# * :+trigger_name+: name of the trigger
|
50
|
+
# * :+table+: name of the table that should be monitored
|
51
|
+
# * :+keys+: array of names of the key columns of the monitored table
|
52
|
+
# * :+log_table+: name of the table receiving all change notifications
|
53
|
+
# * :+activity_table+: name of the table receiving the rubyrep activity information
|
54
|
+
# * :+key_sep+: column seperator to be used in the key column of the log table
|
55
|
+
# * :+exclude_rr_activity+:
|
56
|
+
# if true, the trigger will check and filter out changes initiated by RubyRep
|
57
|
+
def create_replication_trigger(params)
|
58
|
+
create_or_replace_replication_trigger_function params
|
59
|
+
|
60
|
+
%w(insert update delete).each do |action|
|
61
|
+
execute(<<-end_sql)
|
62
|
+
DROP TRIGGER IF EXISTS #{params[:trigger_name]}_#{action};
|
63
|
+
end_sql
|
64
|
+
|
65
|
+
# The created triggers can handle the case where the trigger procedure
|
66
|
+
# is updated (that is: temporarily deleted and recreated) while the
|
67
|
+
# trigger is running.
|
68
|
+
# For that an MySQL internal exception is raised if the trigger
|
69
|
+
# procedure cannot be found. The exception is caught by an trigger
|
70
|
+
# internal handler.
|
71
|
+
# The handler causes the trigger to retry calling the
|
72
|
+
# trigger procedure several times with short breaks in between.
|
73
|
+
|
74
|
+
trigger_var = action == 'delete' ? 'OLD' : 'NEW'
|
75
|
+
if action == 'update'
|
76
|
+
call_statement = "CALL #{params[:trigger_name]}(#{key_clause('OLD', params)}, #{key_clause('NEW', params)}, '#{action[0,1].upcase}');"
|
77
|
+
else
|
78
|
+
call_statement = "CALL #{params[:trigger_name]}(#{key_clause(trigger_var, params)}, null, '#{action[0,1].upcase}');"
|
79
|
+
end
|
80
|
+
execute(<<-end_sql)
|
81
|
+
CREATE TRIGGER #{params[:trigger_name]}_#{action}
|
82
|
+
AFTER #{action} ON #{params[:table]} FOR EACH ROW BEGIN
|
83
|
+
DECLARE number_attempts INT DEFAULT 0;
|
84
|
+
DECLARE failed INT;
|
85
|
+
DECLARE CONTINUE HANDLER FOR 1305 BEGIN
|
86
|
+
DO SLEEP(0.05);
|
87
|
+
SET failed = 1;
|
88
|
+
SET number_attempts = number_attempts + 1;
|
89
|
+
END;
|
90
|
+
REPEAT
|
91
|
+
SET failed = 0;
|
92
|
+
#{call_statement}
|
93
|
+
UNTIL failed = 0 OR number_attempts >= 40 END REPEAT;
|
94
|
+
END;
|
95
|
+
end_sql
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
|
100
|
+
# Removes a trigger and related trigger procedure.
|
101
|
+
# * +trigger_name+: name of the trigger
|
102
|
+
# * +table_name+: name of the table for which the trigger exists
|
103
|
+
def drop_replication_trigger(trigger_name, table_name)
|
104
|
+
%w(insert update delete).each do |action|
|
105
|
+
execute "DROP TRIGGER #{trigger_name}_#{action};"
|
106
|
+
end
|
107
|
+
execute "DROP PROCEDURE #{trigger_name};"
|
108
|
+
end
|
109
|
+
|
110
|
+
# Returns +true+ if the named trigger exists for the named table.
|
111
|
+
# * +trigger_name+: name of the trigger
|
112
|
+
# * +table_name+: name of the table
|
113
|
+
def replication_trigger_exists?(trigger_name, table_name)
|
114
|
+
!select_all("select 1 from information_schema.triggers where trigger_schema = database() and trigger_name = '#{trigger_name}_insert' and event_object_table = '#{table_name}'").empty?
|
115
|
+
end
|
116
|
+
|
117
|
+
# Returns all unadjusted sequences of the given table.
|
118
|
+
# Parameters:
|
119
|
+
# * +rep_prefix+:
|
120
|
+
# The prefix put in front of all replication related database objects as
|
121
|
+
# specified via Configuration#options.
|
122
|
+
# Is used to create the sequences table.
|
123
|
+
# * +table_name+: name of the table
|
124
|
+
# Return value: a hash with
|
125
|
+
# * key: sequence name
|
126
|
+
# * value: a hash with
|
127
|
+
# * :+increment+: current sequence increment
|
128
|
+
# * :+value+: current value
|
129
|
+
def sequence_values(rep_prefix, table_name)
|
130
|
+
# check if the table has an auto_increment column, return if not
|
131
|
+
sequence_row = select_one(<<-end_sql)
|
132
|
+
show columns from #{table_name} where extra = 'auto_increment'
|
133
|
+
end_sql
|
134
|
+
return {} unless sequence_row
|
135
|
+
column_name = sequence_row['Field']
|
136
|
+
|
137
|
+
# check if the sequences table exists, create if necessary
|
138
|
+
sequence_table_name = "#{rep_prefix}_sequences"
|
139
|
+
unless tables.include?(sequence_table_name)
|
140
|
+
create_table "#{sequence_table_name}".to_sym,
|
141
|
+
:id => false, :options => 'ENGINE=MyISAM' do |t|
|
142
|
+
t.column :name, :string
|
143
|
+
t.column :current_value, :integer
|
144
|
+
t.column :increment, :integer
|
145
|
+
t.column :offset, :integer
|
146
|
+
end
|
147
|
+
ActiveRecord::Base.connection.execute(<<-end_sql) rescue nil
|
148
|
+
ALTER TABLE "#{sequence_table_name}"
|
149
|
+
ADD CONSTRAINT #{sequence_table_name}_pkey
|
150
|
+
PRIMARY KEY (name)
|
151
|
+
end_sql
|
152
|
+
end
|
153
|
+
|
154
|
+
sequence_row = select_one("select current_value, increment, offset from #{sequence_table_name} where name = '#{table_name}'")
|
155
|
+
if sequence_row == nil
|
156
|
+
current_max = select_one(<<-end_sql)['current_max'].to_i
|
157
|
+
select max(#{column_name}) as current_max from #{table_name}
|
158
|
+
end_sql
|
159
|
+
return {column_name => {
|
160
|
+
:increment => 1,
|
161
|
+
:value => current_max
|
162
|
+
}
|
163
|
+
}
|
164
|
+
else
|
165
|
+
return {column_name => {
|
166
|
+
:increment => sequence_row['increment'].to_i,
|
167
|
+
:value => sequence_row['offset'].to_i
|
168
|
+
}
|
169
|
+
}
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
# Ensures that the sequences of the named table (normally the primary key
|
174
|
+
# column) are generated with the correct increment and offset.
|
175
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
176
|
+
# * +table_name+: name of the table (not used for Postgres)
|
177
|
+
# * +increment+: increment of the sequence
|
178
|
+
# * +offset+: offset
|
179
|
+
# * +left_sequence_values+:
|
180
|
+
# hash as returned by #outdated_sequence_values for the left database
|
181
|
+
# * +right_sequence_values+:
|
182
|
+
# hash as returned by #outdated_sequence_values for the right database
|
183
|
+
# * +adjustment_buffer+:
|
184
|
+
# the "gap" that is created during sequence update to avoid concurrency problems
|
185
|
+
# E. g. an increment of 2 and offset of 1 will lead to generation of odd
|
186
|
+
# numbers.
|
187
|
+
def update_sequences(
|
188
|
+
rep_prefix, table_name, increment, offset,
|
189
|
+
left_sequence_values, right_sequence_values, adjustment_buffer)
|
190
|
+
return if left_sequence_values.empty?
|
191
|
+
column_name = left_sequence_values.keys[0]
|
192
|
+
|
193
|
+
# check if the sequences table exists, create if necessary
|
194
|
+
sequence_table_name = "#{rep_prefix}_sequences"
|
195
|
+
current_max =
|
196
|
+
[left_sequence_values[column_name][:value], right_sequence_values[column_name][:value]].max +
|
197
|
+
adjustment_buffer
|
198
|
+
new_start = current_max - (current_max % increment) + increment + offset
|
199
|
+
|
200
|
+
sequence_row = select_one("select current_value, increment, offset from #{sequence_table_name} where name = '#{table_name}'")
|
201
|
+
if sequence_row == nil
|
202
|
+
# no sequence exists yet for the table, create it and the according
|
203
|
+
# sequence trigger
|
204
|
+
execute(<<-end_sql)
|
205
|
+
insert into #{sequence_table_name}(name, current_value, increment, offset)
|
206
|
+
values('#{table_name}', #{new_start}, #{increment}, #{offset})
|
207
|
+
end_sql
|
208
|
+
trigger_name = "#{rep_prefix}_#{table_name}_sequence"
|
209
|
+
execute(<<-end_sql)
|
210
|
+
DROP TRIGGER IF EXISTS #{trigger_name};
|
211
|
+
end_sql
|
212
|
+
|
213
|
+
execute(<<-end_sql)
|
214
|
+
CREATE TRIGGER #{trigger_name}
|
215
|
+
BEFORE INSERT ON #{table_name} FOR EACH ROW BEGIN
|
216
|
+
IF NEW.#{column_name} = 0 THEN
|
217
|
+
UPDATE #{sequence_table_name}
|
218
|
+
SET current_value = LAST_INSERT_ID(current_value + increment)
|
219
|
+
WHERE name = '#{table_name}';
|
220
|
+
SET NEW.#{column_name} = LAST_INSERT_ID();
|
221
|
+
END IF;
|
222
|
+
END;
|
223
|
+
end_sql
|
224
|
+
elsif sequence_row['increment'].to_i != increment or sequence_row['offset'].to_i != offset
|
225
|
+
# sequence exists but with incorrect values; update it
|
226
|
+
execute(<<-end_sql)
|
227
|
+
update #{sequence_table_name}
|
228
|
+
set current_value = #{new_start},
|
229
|
+
increment = #{increment}, offset = #{offset}
|
230
|
+
where name = '#{table_name}'
|
231
|
+
end_sql
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
# Adds a big (8 byte value), auto-incrementing primary key column to the
|
236
|
+
# specified table.
|
237
|
+
# * table_name: name of the target table
|
238
|
+
# * key_name: name of the primary key column
|
239
|
+
def add_big_primary_key(table_name, key_name)
|
240
|
+
execute(<<-end_sql)
|
241
|
+
alter table #{table_name} add column #{key_name} bigint not null auto_increment primary key
|
242
|
+
end_sql
|
243
|
+
end
|
244
|
+
|
245
|
+
# Removes the custom sequence setup for the specified table.
|
246
|
+
# If no more rubyrep sequences are left, removes the sequence table.
|
247
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
248
|
+
# * +table_name+: name of the table
|
249
|
+
def clear_sequence_setup(rep_prefix, table_name)
|
250
|
+
sequence_table_name = "#{rep_prefix}_sequences"
|
251
|
+
if tables.include?(sequence_table_name)
|
252
|
+
trigger_name = "#{rep_prefix}_#{table_name}_sequence"
|
253
|
+
trigger_row = select_one(<<-end_sql)
|
254
|
+
select * from information_schema.triggers
|
255
|
+
where trigger_schema = database()
|
256
|
+
and trigger_name = '#{trigger_name}'
|
257
|
+
end_sql
|
258
|
+
if trigger_row
|
259
|
+
execute "DROP TRIGGER #{trigger_name}"
|
260
|
+
execute "delete from #{sequence_table_name} where name = '#{table_name}'"
|
261
|
+
unless select_one("select * from #{sequence_table_name}")
|
262
|
+
# no more sequences left --> delete sequence table
|
263
|
+
drop_table sequence_table_name.to_sym
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
@@ -0,0 +1,204 @@
|
|
1
|
+
module RR
|
2
|
+
module ReplicationExtenders
|
3
|
+
|
4
|
+
# Provides PostgreSQL specific functionality for database replication
|
5
|
+
module PostgreSQLReplication
|
6
|
+
RR::ReplicationExtenders.register :postgresql => self
|
7
|
+
|
8
|
+
# Returns the key clause that is used in the trigger function.
|
9
|
+
# * +trigger_var+: should be either 'NEW' or 'OLD'
|
10
|
+
# * +params+: the parameter hash as described in #create_rep_trigger
|
11
|
+
def key_clause(trigger_var, params)
|
12
|
+
params[:keys].
|
13
|
+
map { |key| "'#{key}#{params[:key_sep]}' || #{trigger_var}.#{key}"}.
|
14
|
+
join(" || '#{params[:key_sep]}' || ")
|
15
|
+
end
|
16
|
+
private :key_clause
|
17
|
+
|
18
|
+
# Creates or replaces the replication trigger function.
|
19
|
+
# See #create_replication_trigger for a descriptions of the +params+ hash.
|
20
|
+
def create_or_replace_replication_trigger_function(params)
|
21
|
+
# first check, if PL/SQL is already activated and if not, do so.
|
22
|
+
if select_all("select lanname from pg_language where lanname = 'plpgsql'").empty?
|
23
|
+
execute "CREATE LANGUAGE plpgsql"
|
24
|
+
end
|
25
|
+
|
26
|
+
activity_check = ""
|
27
|
+
if params[:exclude_rr_activity] then
|
28
|
+
activity_check = <<-end_sql
|
29
|
+
PERFORM ACTIVE FROM #{params[:activity_table]};
|
30
|
+
IF FOUND THEN
|
31
|
+
RETURN NULL;
|
32
|
+
END IF;
|
33
|
+
end_sql
|
34
|
+
end
|
35
|
+
|
36
|
+
# now create the trigger
|
37
|
+
execute(<<-end_sql)
|
38
|
+
CREATE OR REPLACE FUNCTION #{params[:trigger_name]}() RETURNS TRIGGER AS $change_trigger$
|
39
|
+
BEGIN
|
40
|
+
#{activity_check}
|
41
|
+
IF (TG_OP = 'DELETE') THEN
|
42
|
+
INSERT INTO #{params[:log_table]}(change_table, change_key, change_type, change_time)
|
43
|
+
SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
|
44
|
+
ELSIF (TG_OP = 'UPDATE') THEN
|
45
|
+
INSERT INTO #{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
|
46
|
+
SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
|
47
|
+
ELSIF (TG_OP = 'INSERT') THEN
|
48
|
+
INSERT INTO #{params[:log_table]}(change_table, change_key, change_type, change_time)
|
49
|
+
SELECT '#{params[:table]}', #{key_clause('NEW', params)}, 'I', now();
|
50
|
+
END IF;
|
51
|
+
RETURN NULL; -- result is ignored since this is an AFTER trigger
|
52
|
+
END;
|
53
|
+
$change_trigger$ LANGUAGE plpgsql;
|
54
|
+
end_sql
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
# Creates a trigger to log all changes for the given table.
|
59
|
+
# +params+ is a hash with all necessary information:
|
60
|
+
# * :+trigger_name+: name of the trigger
|
61
|
+
# * :+table+: name of the table that should be monitored
|
62
|
+
# * :+keys+: array of names of the key columns of the monitored table
|
63
|
+
# * :+log_table+: name of the table receiving all change notifications
|
64
|
+
# * :+activity_table+: name of the table receiving the rubyrep activity information
|
65
|
+
# * :+key_sep+: column seperator to be used in the key column of the log table
|
66
|
+
# * :+exclude_rr_activity+:
|
67
|
+
# if true, the trigger will check and filter out changes initiated by RubyRep
|
68
|
+
def create_replication_trigger(params)
|
69
|
+
create_or_replace_replication_trigger_function params
|
70
|
+
|
71
|
+
execute(<<-end_sql)
|
72
|
+
CREATE TRIGGER #{params[:trigger_name]}
|
73
|
+
AFTER INSERT OR UPDATE OR DELETE ON #{params[:table]}
|
74
|
+
FOR EACH ROW EXECUTE PROCEDURE #{params[:trigger_name]}();
|
75
|
+
end_sql
|
76
|
+
end
|
77
|
+
|
78
|
+
# Removes a trigger and related trigger procedure.
|
79
|
+
# * +trigger_name+: name of the trigger
|
80
|
+
# * +table_name+: name of the table for which the trigger exists
|
81
|
+
def drop_replication_trigger(trigger_name, table_name)
|
82
|
+
execute "DROP TRIGGER #{trigger_name} ON #{table_name};"
|
83
|
+
execute "DROP FUNCTION #{trigger_name}();"
|
84
|
+
end
|
85
|
+
|
86
|
+
# Returns +true+ if the named trigger exists for the named table.
|
87
|
+
# * +trigger_name+: name of the trigger
|
88
|
+
# * +table_name+: name of the table
|
89
|
+
def replication_trigger_exists?(trigger_name, table_name)
|
90
|
+
search_path = select_one("show search_path")['search_path']
|
91
|
+
schemas = search_path.split(/,/).map { |p| quote(p) }.join(',')
|
92
|
+
!select_all(<<-end_sql).empty?
|
93
|
+
select 1 from information_schema.triggers
|
94
|
+
where event_object_schema in (#{schemas})
|
95
|
+
and trigger_name = '#{trigger_name}'
|
96
|
+
and event_object_table = '#{table_name}'
|
97
|
+
end_sql
|
98
|
+
end
|
99
|
+
|
100
|
+
# Returns all unadjusted sequences of the given table.
|
101
|
+
# Parameters:
|
102
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
103
|
+
# * +table_name+: name of the table
|
104
|
+
# Return value: a hash with
|
105
|
+
# * key: sequence name
|
106
|
+
# * value: a hash with
|
107
|
+
# * :+increment+: current sequence increment
|
108
|
+
# * :+value+: current value
|
109
|
+
def sequence_values(rep_prefix, table_name)
|
110
|
+
result = {}
|
111
|
+
sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
|
112
|
+
select s.relname
|
113
|
+
from pg_class as t
|
114
|
+
join pg_depend as r on t.oid = r.refobjid
|
115
|
+
join pg_class as s on r.objid = s.oid
|
116
|
+
and s.relkind = 'S'
|
117
|
+
and t.relname = '#{table_name}'
|
118
|
+
end_sql
|
119
|
+
sequence_names.each do |sequence_name|
|
120
|
+
row = select_one("select last_value, increment_by from #{sequence_name}")
|
121
|
+
result[sequence_name] = {
|
122
|
+
:increment => row['increment_by'].to_i,
|
123
|
+
:value => row['last_value'].to_i
|
124
|
+
}
|
125
|
+
end
|
126
|
+
result
|
127
|
+
end
|
128
|
+
|
129
|
+
# Ensures that the sequences of the named table (normally the primary key
|
130
|
+
# column) are generated with the correct increment and offset.
|
131
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
132
|
+
# * +table_name+: name of the table (not used for Postgres)
|
133
|
+
# * +increment+: increment of the sequence
|
134
|
+
# * +offset+: offset
|
135
|
+
# * +left_sequence_values+:
|
136
|
+
# hash as returned by #outdated_sequence_values for the left database
|
137
|
+
# * +right_sequence_values+:
|
138
|
+
# hash as returned by #outdated_sequence_values for the right database
|
139
|
+
# * +adjustment_buffer+:
|
140
|
+
# the "gap" that is created during sequence update to avoid concurrency problems
|
141
|
+
# E. g. an increment of 2 and offset of 1 will lead to generation of odd
|
142
|
+
# numbers.
|
143
|
+
def update_sequences(
|
144
|
+
rep_prefix, table_name, increment, offset,
|
145
|
+
left_sequence_values, right_sequence_values, adjustment_buffer)
|
146
|
+
left_sequence_values.each do |sequence_name, left_current_value|
|
147
|
+
row = select_one("select last_value, increment_by from #{sequence_name}")
|
148
|
+
current_increment = row['increment_by'].to_i
|
149
|
+
current_value = row['last_value'].to_i
|
150
|
+
unless current_increment == increment and current_value % increment == offset
|
151
|
+
max_current_value =
|
152
|
+
[left_current_value[:value], right_sequence_values[sequence_name][:value]].max +
|
153
|
+
adjustment_buffer
|
154
|
+
new_start = max_current_value - (max_current_value % increment) + increment + offset
|
155
|
+
execute(<<-end_sql)
|
156
|
+
alter sequence "#{sequence_name}" increment by #{increment} restart with #{new_start}
|
157
|
+
end_sql
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
# Restores the original sequence settings.
|
163
|
+
# (Actually it sets the sequence increment to 1. If before, it had a
|
164
|
+
# different value, then the restoration will not be correct.)
|
165
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
166
|
+
# * +table_name+: name of the table
|
167
|
+
def clear_sequence_setup(rep_prefix, table_name)
|
168
|
+
sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
|
169
|
+
select s.relname
|
170
|
+
from pg_class as t
|
171
|
+
join pg_depend as r on t.oid = r.refobjid
|
172
|
+
join pg_class as s on r.objid = s.oid
|
173
|
+
and s.relkind = 'S'
|
174
|
+
and t.relname = '#{table_name}'
|
175
|
+
end_sql
|
176
|
+
sequence_names.each do |sequence_name|
|
177
|
+
execute(<<-end_sql)
|
178
|
+
alter sequence "#{sequence_name}" increment by 1
|
179
|
+
end_sql
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
# Adds a big (8 byte value), auto-incrementing primary key column to the
|
184
|
+
# specified table.
|
185
|
+
# * table_name: name of the target table
|
186
|
+
# * key_name: name of the primary key column
|
187
|
+
def add_big_primary_key(table_name, key_name)
|
188
|
+
old_message_level = select_one("show client_min_messages")['client_min_messages']
|
189
|
+
execute "set client_min_messages = warning"
|
190
|
+
execute(<<-end_sql)
|
191
|
+
alter table "#{table_name}" add column #{key_name} bigserial
|
192
|
+
end_sql
|
193
|
+
|
194
|
+
execute(<<-end_sql)
|
195
|
+
alter table "#{table_name}" add constraint #{table_name}_#{key_name}_pkey primary key (#{key_name})
|
196
|
+
end_sql
|
197
|
+
|
198
|
+
ensure
|
199
|
+
execute "set client_min_messages = #{old_message_level}"
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module RR
|
2
|
+
|
3
|
+
# Replication extenders are modules that provide database specific functionality
|
4
|
+
# required for replication. They are mixed into ActiveRecord database connections.
|
5
|
+
# This module itself only provides functionality to register and retrieve
|
6
|
+
# such extenders.
|
7
|
+
module ReplicationExtenders
|
8
|
+
# Returns a Hash of currently registered replication extenders.
|
9
|
+
# (Empty Hash if no replication extenders were defined.)
|
10
|
+
def self.extenders
|
11
|
+
@extenders ||= {}
|
12
|
+
@extenders
|
13
|
+
end
|
14
|
+
|
15
|
+
# Registers one or multiple replication extender.
|
16
|
+
# extender is a Hash with
|
17
|
+
# key:: The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
|
18
|
+
# value:: Name of the module implementing the replication extender
|
19
|
+
def self.register(extender)
|
20
|
+
@extenders ||= {}
|
21
|
+
@extenders.merge! extender
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
|
@@ -0,0 +1,104 @@
|
|
1
|
+
module RR
|
2
|
+
|
3
|
+
# Provides helper functionality for replicators.
|
4
|
+
# The methods exposed by this class are intended to provide a stable interface
|
5
|
+
# for third party replicators.
|
6
|
+
class ReplicationHelper
|
7
|
+
|
8
|
+
# The current +ReplicationRun+ instance
|
9
|
+
attr_accessor :replication_run
|
10
|
+
|
11
|
+
# The active +Session+
|
12
|
+
def session; replication_run.session; end
|
13
|
+
|
14
|
+
# Current options
|
15
|
+
def options; @options ||= session.configuration.options; end
|
16
|
+
|
17
|
+
# Delegates to Session#corresponding_table
|
18
|
+
def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
|
19
|
+
|
20
|
+
# Delegates to Committer#insert_record
|
21
|
+
def insert_record(database, table, values)
|
22
|
+
committer.insert_record(database, table, values)
|
23
|
+
end
|
24
|
+
|
25
|
+
# Delegates to Committer#insert_record
|
26
|
+
def update_record(database, table, values, old_key = nil)
|
27
|
+
committer.update_record(database, table, values, old_key)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Delegates to Committer#insert_record
|
31
|
+
def delete_record(database, table, values)
|
32
|
+
committer.delete_record(database, table, values)
|
33
|
+
end
|
34
|
+
|
35
|
+
# Loads the specified record. Returns an according column_name => value hash.
|
36
|
+
# Parameters:
|
37
|
+
# * +database+: either :+left+ or :+right+
|
38
|
+
# * +table+: name of the table
|
39
|
+
# * +key+: A column_name => value hash for all primary key columns.
|
40
|
+
def load_record(database, table, key)
|
41
|
+
cursor = session.send(database).select_cursor(
|
42
|
+
:table => table,
|
43
|
+
:row_keys => [key],
|
44
|
+
:type_cast => true
|
45
|
+
)
|
46
|
+
row = nil
|
47
|
+
row = cursor.next_row if cursor.next?
|
48
|
+
cursor.clear
|
49
|
+
row
|
50
|
+
end
|
51
|
+
|
52
|
+
# The current Committer
|
53
|
+
attr_reader :committer
|
54
|
+
private :committer
|
55
|
+
|
56
|
+
# Asks the committer (if it exists) to finalize any open transactions
|
57
|
+
# +success+ should be true if there were no problems, false otherwise.
|
58
|
+
def finalize(success = true)
|
59
|
+
committer.finalize(success)
|
60
|
+
end
|
61
|
+
|
62
|
+
# Logs the outcome of a replication into the replication log table.
|
63
|
+
# * +diff+: the replicated ReplicationDifference
|
64
|
+
# * +outcome+: string summarizing the outcome of the replication
|
65
|
+
# * +details+: string with further details regarding the replication
|
66
|
+
def log_replication_outcome(diff, outcome, details = nil)
|
67
|
+
table = diff.changes[:left].table
|
68
|
+
key = diff.changes[:left].key
|
69
|
+
if key.size == 1
|
70
|
+
key = key.values[0]
|
71
|
+
else
|
72
|
+
key_parts = session.left.primary_key_names(table).map do |column_name|
|
73
|
+
%Q("#{column_name}"=>#{key[column_name].to_s.inspect})
|
74
|
+
end
|
75
|
+
key = key_parts.join(', ')
|
76
|
+
end
|
77
|
+
rep_details = details == nil ? nil : details[0...ReplicationInitializer::LONG_DESCRIPTION_SIZE]
|
78
|
+
diff_dump = diff.to_yaml[0...ReplicationInitializer::DIFF_DUMP_SIZE]
|
79
|
+
|
80
|
+
session.left.insert_record "#{options[:rep_prefix]}_logged_events", {
|
81
|
+
:activity => 'replication',
|
82
|
+
:change_table => table,
|
83
|
+
:diff_type => diff.type.to_s,
|
84
|
+
:change_key => key,
|
85
|
+
:left_change_type => (diff.changes[:left] ? diff.changes[:left].type.to_s : nil),
|
86
|
+
:right_change_type => (diff.changes[:right] ? diff.changes[:right].type.to_s : nil),
|
87
|
+
:description => outcome.to_s,
|
88
|
+
:long_description => rep_details,
|
89
|
+
:event_time => Time.now,
|
90
|
+
:diff_dump => diff_dump
|
91
|
+
}
|
92
|
+
end
|
93
|
+
|
94
|
+
# Creates a new SyncHelper for the given +TableSync+ instance.
|
95
|
+
def initialize(replication_run)
|
96
|
+
self.replication_run = replication_run
|
97
|
+
|
98
|
+
# Creates the committer. Important as it gives the committer the
|
99
|
+
# opportunity to start transactions
|
100
|
+
committer_class = Committers::committers[options[:committer]]
|
101
|
+
@committer = committer_class.new(session)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|