andyjeffries-rubyrep 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +83 -0
- data/License.txt +20 -0
- data/Manifest.txt +151 -0
- data/README.txt +37 -0
- data/bin/rubyrep +8 -0
- data/lib/rubyrep.rb +72 -0
- data/lib/rubyrep/base_runner.rb +195 -0
- data/lib/rubyrep/command_runner.rb +144 -0
- data/lib/rubyrep/committers/buffered_committer.rb +151 -0
- data/lib/rubyrep/committers/committers.rb +152 -0
- data/lib/rubyrep/configuration.rb +275 -0
- data/lib/rubyrep/connection_extenders/connection_extenders.rb +165 -0
- data/lib/rubyrep/connection_extenders/jdbc_extender.rb +65 -0
- data/lib/rubyrep/connection_extenders/mysql_extender.rb +59 -0
- data/lib/rubyrep/connection_extenders/postgresql_extender.rb +277 -0
- data/lib/rubyrep/database_proxy.rb +52 -0
- data/lib/rubyrep/direct_table_scan.rb +75 -0
- data/lib/rubyrep/generate_runner.rb +105 -0
- data/lib/rubyrep/initializer.rb +39 -0
- data/lib/rubyrep/log_helper.rb +30 -0
- data/lib/rubyrep/logged_change.rb +160 -0
- data/lib/rubyrep/logged_change_loader.rb +197 -0
- data/lib/rubyrep/noisy_connection.rb +80 -0
- data/lib/rubyrep/proxied_table_scan.rb +171 -0
- data/lib/rubyrep/proxy_block_cursor.rb +145 -0
- data/lib/rubyrep/proxy_connection.rb +431 -0
- data/lib/rubyrep/proxy_cursor.rb +44 -0
- data/lib/rubyrep/proxy_row_cursor.rb +43 -0
- data/lib/rubyrep/proxy_runner.rb +89 -0
- data/lib/rubyrep/replication_difference.rb +100 -0
- data/lib/rubyrep/replication_extenders/mysql_replication.rb +271 -0
- data/lib/rubyrep/replication_extenders/postgresql_replication.rb +236 -0
- data/lib/rubyrep/replication_extenders/replication_extenders.rb +26 -0
- data/lib/rubyrep/replication_helper.rb +142 -0
- data/lib/rubyrep/replication_initializer.rb +327 -0
- data/lib/rubyrep/replication_run.rb +142 -0
- data/lib/rubyrep/replication_runner.rb +166 -0
- data/lib/rubyrep/replicators/replicators.rb +42 -0
- data/lib/rubyrep/replicators/two_way_replicator.rb +361 -0
- data/lib/rubyrep/scan_progress_printers/progress_bar.rb +65 -0
- data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb +65 -0
- data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb +111 -0
- data/lib/rubyrep/scan_report_printers/scan_report_printers.rb +67 -0
- data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb +75 -0
- data/lib/rubyrep/scan_runner.rb +25 -0
- data/lib/rubyrep/session.rb +230 -0
- data/lib/rubyrep/sync_helper.rb +121 -0
- data/lib/rubyrep/sync_runner.rb +31 -0
- data/lib/rubyrep/syncers/syncers.rb +112 -0
- data/lib/rubyrep/syncers/two_way_syncer.rb +174 -0
- data/lib/rubyrep/table_scan.rb +54 -0
- data/lib/rubyrep/table_scan_helper.rb +46 -0
- data/lib/rubyrep/table_sorter.rb +70 -0
- data/lib/rubyrep/table_spec_resolver.rb +142 -0
- data/lib/rubyrep/table_sync.rb +90 -0
- data/lib/rubyrep/task_sweeper.rb +77 -0
- data/lib/rubyrep/trigger_mode_switcher.rb +63 -0
- data/lib/rubyrep/type_casting_cursor.rb +31 -0
- data/lib/rubyrep/uninstall_runner.rb +93 -0
- data/lib/rubyrep/version.rb +9 -0
- data/rubyrep +8 -0
- data/rubyrep.bat +4 -0
- data/setup.rb +1585 -0
- data/spec/base_runner_spec.rb +218 -0
- data/spec/buffered_committer_spec.rb +274 -0
- data/spec/command_runner_spec.rb +145 -0
- data/spec/committers_spec.rb +178 -0
- data/spec/configuration_spec.rb +203 -0
- data/spec/connection_extender_interface_spec.rb +141 -0
- data/spec/connection_extenders_registration_spec.rb +164 -0
- data/spec/database_proxy_spec.rb +48 -0
- data/spec/database_rake_spec.rb +40 -0
- data/spec/db_specific_connection_extenders_spec.rb +34 -0
- data/spec/db_specific_replication_extenders_spec.rb +38 -0
- data/spec/direct_table_scan_spec.rb +61 -0
- data/spec/dolphins.jpg +0 -0
- data/spec/generate_runner_spec.rb +84 -0
- data/spec/initializer_spec.rb +46 -0
- data/spec/log_helper_spec.rb +39 -0
- data/spec/logged_change_loader_spec.rb +68 -0
- data/spec/logged_change_spec.rb +470 -0
- data/spec/noisy_connection_spec.rb +78 -0
- data/spec/postgresql_replication_spec.rb +48 -0
- data/spec/postgresql_schema_support_spec.rb +212 -0
- data/spec/postgresql_support_spec.rb +63 -0
- data/spec/progress_bar_spec.rb +77 -0
- data/spec/proxied_table_scan_spec.rb +151 -0
- data/spec/proxy_block_cursor_spec.rb +197 -0
- data/spec/proxy_connection_spec.rb +423 -0
- data/spec/proxy_cursor_spec.rb +56 -0
- data/spec/proxy_row_cursor_spec.rb +66 -0
- data/spec/proxy_runner_spec.rb +70 -0
- data/spec/replication_difference_spec.rb +161 -0
- data/spec/replication_extender_interface_spec.rb +367 -0
- data/spec/replication_extenders_spec.rb +32 -0
- data/spec/replication_helper_spec.rb +178 -0
- data/spec/replication_initializer_spec.rb +509 -0
- data/spec/replication_run_spec.rb +443 -0
- data/spec/replication_runner_spec.rb +254 -0
- data/spec/replicators_spec.rb +36 -0
- data/spec/rubyrep_spec.rb +8 -0
- data/spec/scan_detail_reporter_spec.rb +119 -0
- data/spec/scan_progress_printers_spec.rb +68 -0
- data/spec/scan_report_printers_spec.rb +67 -0
- data/spec/scan_runner_spec.rb +50 -0
- data/spec/scan_summary_reporter_spec.rb +61 -0
- data/spec/session_spec.rb +253 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +305 -0
- data/spec/strange_name_support_spec.rb +135 -0
- data/spec/sync_helper_spec.rb +169 -0
- data/spec/sync_runner_spec.rb +78 -0
- data/spec/syncers_spec.rb +171 -0
- data/spec/table_scan_helper_spec.rb +36 -0
- data/spec/table_scan_spec.rb +49 -0
- data/spec/table_sorter_spec.rb +30 -0
- data/spec/table_spec_resolver_spec.rb +111 -0
- data/spec/table_sync_spec.rb +140 -0
- data/spec/task_sweeper_spec.rb +47 -0
- data/spec/trigger_mode_switcher_spec.rb +83 -0
- data/spec/two_way_replicator_spec.rb +721 -0
- data/spec/two_way_syncer_spec.rb +256 -0
- data/spec/type_casting_cursor_spec.rb +50 -0
- data/spec/uninstall_runner_spec.rb +93 -0
- metadata +190 -0
@@ -0,0 +1,236 @@
|
|
1
|
+
module RR
|
2
|
+
module ReplicationExtenders
|
3
|
+
|
4
|
+
# Provides PostgreSQL specific functionality for database replication
|
5
|
+
module PostgreSQLReplication
|
6
|
+
RR::ReplicationExtenders.register :postgresql => self
|
7
|
+
|
8
|
+
# Returns the key clause that is used in the trigger function.
|
9
|
+
# * +trigger_var+: should be either 'NEW' or 'OLD'
|
10
|
+
# * +params+: the parameter hash as described in #create_rep_trigger
|
11
|
+
def key_clause(trigger_var, params)
|
12
|
+
params[:keys].
|
13
|
+
map { |key| "'#{key}#{params[:key_sep]}' || #{trigger_var}.#{quote_column_name(key)}"}.
|
14
|
+
join(" || '#{params[:key_sep]}' || ")
|
15
|
+
end
|
16
|
+
private :key_clause
|
17
|
+
|
18
|
+
# Returns the schema prefix (including dot) that will be used by the
|
19
|
+
# triggers to write into the rubyrep infrastructure tables.
|
20
|
+
# To avoid setting the wrong prefix, it will only return a schema prefix
|
21
|
+
# if the current search path
|
22
|
+
# * consists of only a single schema
|
23
|
+
# * does not consists of a variable search path
|
24
|
+
# (i. e. the default "$user")
|
25
|
+
def schema_prefix
|
26
|
+
unless @schema_prefix
|
27
|
+
search_path = select_one("show search_path")['search_path']
|
28
|
+
if search_path =~ /[,$]/
|
29
|
+
@schema_prefix = ""
|
30
|
+
else
|
31
|
+
@schema_prefix = %("#{search_path}".)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
@schema_prefix
|
35
|
+
end
|
36
|
+
|
37
|
+
# Creates or replaces the replication trigger function.
|
38
|
+
# See #create_replication_trigger for a descriptions of the +params+ hash.
|
39
|
+
def create_or_replace_replication_trigger_function(params)
|
40
|
+
# first check, if PL/SQL is already activated and if not, do so.
|
41
|
+
if select_all("select lanname from pg_language where lanname = 'plpgsql'").empty?
|
42
|
+
execute "CREATE LANGUAGE plpgsql"
|
43
|
+
end
|
44
|
+
|
45
|
+
activity_check = ""
|
46
|
+
if params[:exclude_rr_activity] then
|
47
|
+
activity_check = <<-end_sql
|
48
|
+
PERFORM ACTIVE FROM #{schema_prefix}#{params[:activity_table]};
|
49
|
+
IF FOUND THEN
|
50
|
+
RETURN NULL;
|
51
|
+
END IF;
|
52
|
+
end_sql
|
53
|
+
end
|
54
|
+
|
55
|
+
version_string = select_value("select version();")
|
56
|
+
version = version_string.gsub(/^\s*postgresql\s*([0-9.]+).*$/i, '\1')
|
57
|
+
if version >= '8.4'
|
58
|
+
modification_check = <<-end_sql
|
59
|
+
IF NEW IS NOT DISTINCT FROM OLD THEN
|
60
|
+
RETURN NULL;
|
61
|
+
END IF;
|
62
|
+
end_sql
|
63
|
+
else
|
64
|
+
modification_check = ""
|
65
|
+
end
|
66
|
+
|
67
|
+
# now create the trigger
|
68
|
+
execute(<<-end_sql)
|
69
|
+
CREATE OR REPLACE FUNCTION "#{params[:trigger_name]}"() RETURNS TRIGGER AS $change_trigger$
|
70
|
+
BEGIN
|
71
|
+
#{activity_check}
|
72
|
+
IF (TG_OP = 'DELETE') THEN
|
73
|
+
INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
|
74
|
+
SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
|
75
|
+
ELSIF (TG_OP = 'UPDATE') THEN
|
76
|
+
#{modification_check}
|
77
|
+
INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
|
78
|
+
SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
|
79
|
+
ELSIF (TG_OP = 'INSERT') THEN
|
80
|
+
INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
|
81
|
+
SELECT '#{params[:table]}', #{key_clause('NEW', params)}, 'I', now();
|
82
|
+
END IF;
|
83
|
+
RETURN NULL; -- result is ignored since this is an AFTER trigger
|
84
|
+
END;
|
85
|
+
$change_trigger$ LANGUAGE plpgsql;
|
86
|
+
end_sql
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
# Creates a trigger to log all changes for the given table.
|
91
|
+
# +params+ is a hash with all necessary information:
|
92
|
+
# * :+trigger_name+: name of the trigger
|
93
|
+
# * :+table+: name of the table that should be monitored
|
94
|
+
# * :+keys+: array of names of the key columns of the monitored table
|
95
|
+
# * :+log_table+: name of the table receiving all change notifications
|
96
|
+
# * :+activity_table+: name of the table receiving the rubyrep activity information
|
97
|
+
# * :+key_sep+: column seperator to be used in the key column of the log table
|
98
|
+
# * :+exclude_rr_activity+:
|
99
|
+
# if true, the trigger will check and filter out changes initiated by RubyRep
|
100
|
+
def create_replication_trigger(params)
|
101
|
+
create_or_replace_replication_trigger_function params
|
102
|
+
|
103
|
+
execute(<<-end_sql)
|
104
|
+
CREATE TRIGGER "#{params[:trigger_name]}"
|
105
|
+
AFTER INSERT OR UPDATE OR DELETE ON "#{params[:table]}"
|
106
|
+
FOR EACH ROW EXECUTE PROCEDURE #{schema_prefix}"#{params[:trigger_name]}"();
|
107
|
+
end_sql
|
108
|
+
end
|
109
|
+
|
110
|
+
# Removes a trigger and related trigger procedure.
|
111
|
+
# * +trigger_name+: name of the trigger
|
112
|
+
# * +table_name+: name of the table for which the trigger exists
|
113
|
+
def drop_replication_trigger(trigger_name, table_name)
|
114
|
+
execute "DROP TRIGGER \"#{trigger_name}\" ON \"#{table_name}\";"
|
115
|
+
execute "DROP FUNCTION \"#{trigger_name}\"();"
|
116
|
+
end
|
117
|
+
|
118
|
+
# Returns +true+ if the named trigger exists for the named table.
|
119
|
+
# * +trigger_name+: name of the trigger
|
120
|
+
# * +table_name+: name of the table
|
121
|
+
def replication_trigger_exists?(trigger_name, table_name)
|
122
|
+
!select_all(<<-end_sql).empty?
|
123
|
+
select 1 from information_schema.triggers
|
124
|
+
where event_object_schema in (#{schemas})
|
125
|
+
and trigger_name = '#{trigger_name}'
|
126
|
+
and event_object_table = '#{table_name}'
|
127
|
+
end_sql
|
128
|
+
end
|
129
|
+
|
130
|
+
# Returns all unadjusted sequences of the given table.
|
131
|
+
# Parameters:
|
132
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
133
|
+
# * +table_name+: name of the table
|
134
|
+
# Return value: a hash with
|
135
|
+
# * key: sequence name
|
136
|
+
# * value: a hash with
|
137
|
+
# * :+increment+: current sequence increment
|
138
|
+
# * :+value+: current value
|
139
|
+
def sequence_values(rep_prefix, table_name)
|
140
|
+
result = {}
|
141
|
+
sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
|
142
|
+
select s.relname
|
143
|
+
from pg_class as t
|
144
|
+
join pg_depend as r on t.oid = r.refobjid
|
145
|
+
join pg_class as s on r.objid = s.oid
|
146
|
+
and s.relkind = 'S'
|
147
|
+
and t.relname = '#{table_name}' AND t.relnamespace IN
|
148
|
+
(SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
|
149
|
+
end_sql
|
150
|
+
sequence_names.each do |sequence_name|
|
151
|
+
row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
|
152
|
+
result[sequence_name] = {
|
153
|
+
:increment => row['increment_by'].to_i,
|
154
|
+
:value => row['last_value'].to_i
|
155
|
+
}
|
156
|
+
end
|
157
|
+
result
|
158
|
+
end
|
159
|
+
|
160
|
+
# Ensures that the sequences of the named table (normally the primary key
|
161
|
+
# column) are generated with the correct increment and offset.
|
162
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
163
|
+
# * +table_name+: name of the table (not used for Postgres)
|
164
|
+
# * +increment+: increment of the sequence
|
165
|
+
# * +offset+: offset
|
166
|
+
# * +left_sequence_values+:
|
167
|
+
# hash as returned by #sequence_values for the left database
|
168
|
+
# * +right_sequence_values+:
|
169
|
+
# hash as returned by #sequence_values for the right database
|
170
|
+
# * +adjustment_buffer+:
|
171
|
+
# the "gap" that is created during sequence update to avoid concurrency problems
|
172
|
+
# E. g. an increment of 2 and offset of 1 will lead to generation of odd
|
173
|
+
# numbers.
|
174
|
+
def update_sequences(
|
175
|
+
rep_prefix, table_name, increment, offset,
|
176
|
+
left_sequence_values, right_sequence_values, adjustment_buffer)
|
177
|
+
left_sequence_values.each do |sequence_name, left_current_value|
|
178
|
+
row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
|
179
|
+
current_increment = row['increment_by'].to_i
|
180
|
+
current_value = row['last_value'].to_i
|
181
|
+
unless current_increment == increment and current_value % increment == offset
|
182
|
+
max_current_value =
|
183
|
+
[left_current_value[:value], right_sequence_values[sequence_name][:value]].max +
|
184
|
+
adjustment_buffer
|
185
|
+
new_start = max_current_value - (max_current_value % increment) + increment + offset
|
186
|
+
execute(<<-end_sql)
|
187
|
+
alter sequence "#{sequence_name}" increment by #{increment} restart with #{new_start}
|
188
|
+
end_sql
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
# Restores the original sequence settings.
|
194
|
+
# (Actually it sets the sequence increment to 1. If before, it had a
|
195
|
+
# different value, then the restoration will not be correct.)
|
196
|
+
# * +rep_prefix+: not used (necessary) for the Postgres
|
197
|
+
# * +table_name+: name of the table
|
198
|
+
def clear_sequence_setup(rep_prefix, table_name)
|
199
|
+
sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
|
200
|
+
select s.relname
|
201
|
+
from pg_class as t
|
202
|
+
join pg_depend as r on t.oid = r.refobjid
|
203
|
+
join pg_class as s on r.objid = s.oid
|
204
|
+
and s.relkind = 'S'
|
205
|
+
and t.relname = '#{table_name}' and t.relnamespace IN
|
206
|
+
(SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
|
207
|
+
end_sql
|
208
|
+
sequence_names.each do |sequence_name|
|
209
|
+
execute(<<-end_sql)
|
210
|
+
alter sequence "#{sequence_name}" increment by 1
|
211
|
+
end_sql
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
# Adds a big (8 byte value), auto-incrementing primary key column to the
|
216
|
+
# specified table.
|
217
|
+
# * table_name: name of the target table
|
218
|
+
# * key_name: name of the primary key column
|
219
|
+
def add_big_primary_key(table_name, key_name)
|
220
|
+
old_message_level = select_one("show client_min_messages")['client_min_messages']
|
221
|
+
execute "set client_min_messages = warning"
|
222
|
+
execute(<<-end_sql)
|
223
|
+
alter table "#{table_name}" add column #{key_name} bigserial
|
224
|
+
end_sql
|
225
|
+
|
226
|
+
execute(<<-end_sql)
|
227
|
+
alter table "#{table_name}" add constraint #{table_name}_#{key_name}_pkey primary key (#{key_name})
|
228
|
+
end_sql
|
229
|
+
|
230
|
+
ensure
|
231
|
+
execute "set client_min_messages = #{old_message_level}"
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module RR
|
2
|
+
|
3
|
+
# Replication extenders are modules that provide database specific functionality
|
4
|
+
# required for replication. They are mixed into ActiveRecord database connections.
|
5
|
+
# This module itself only provides functionality to register and retrieve
|
6
|
+
# such extenders.
|
7
|
+
module ReplicationExtenders
|
8
|
+
# Returns a Hash of currently registered replication extenders.
|
9
|
+
# (Empty Hash if no replication extenders were defined.)
|
10
|
+
def self.extenders
|
11
|
+
@extenders ||= {}
|
12
|
+
@extenders
|
13
|
+
end
|
14
|
+
|
15
|
+
# Registers one or multiple replication extender.
|
16
|
+
# extender is a Hash with
|
17
|
+
# key:: The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
|
18
|
+
# value:: Name of the module implementing the replication extender
|
19
|
+
def self.register(extender)
|
20
|
+
@extenders ||= {}
|
21
|
+
@extenders.merge! extender
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module RR
|
2
|
+
|
3
|
+
# Provides helper functionality for replicators.
|
4
|
+
# The methods exposed by this class are intended to provide a stable interface
|
5
|
+
# for third party replicators.
|
6
|
+
class ReplicationHelper
|
7
|
+
|
8
|
+
include LogHelper
|
9
|
+
|
10
|
+
# The current +ReplicationRun+ instance
|
11
|
+
attr_accessor :replication_run
|
12
|
+
|
13
|
+
# The active +Session+
|
14
|
+
def session; replication_run.session; end
|
15
|
+
|
16
|
+
# Current options
|
17
|
+
def options; @options ||= session.configuration.options; end
|
18
|
+
|
19
|
+
# Returns the options for the specified table name.
|
20
|
+
# * +table+: name of the table (left database version)
|
21
|
+
def options_for_table(table)
|
22
|
+
@options_for_table ||= {}
|
23
|
+
unless @options_for_table.include? table
|
24
|
+
@options_for_table[table] = session.configuration.options_for_table(table)
|
25
|
+
end
|
26
|
+
@options_for_table[table]
|
27
|
+
end
|
28
|
+
|
29
|
+
# Delegates to Session#corresponding_table
|
30
|
+
def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
|
31
|
+
|
32
|
+
# Returns +true+ if a new transaction was started since the last
|
33
|
+
# insert / update / delete.
|
34
|
+
def new_transaction?
|
35
|
+
committer.new_transaction?
|
36
|
+
end
|
37
|
+
|
38
|
+
# Delegates to Committers::BufferedCommitter#insert_record
|
39
|
+
def insert_record(database, table, values)
|
40
|
+
committer.insert_record(database, table, values)
|
41
|
+
end
|
42
|
+
|
43
|
+
# Delegates to Committers::BufferedCommitter#update_record
|
44
|
+
def update_record(database, table, values, old_key = nil)
|
45
|
+
committer.update_record(database, table, values, old_key)
|
46
|
+
end
|
47
|
+
|
48
|
+
# Delegates to Committers::BufferedCommitter#delete_record
|
49
|
+
def delete_record(database, table, values)
|
50
|
+
committer.delete_record(database, table, values)
|
51
|
+
end
|
52
|
+
|
53
|
+
# Loads the specified record. Returns an according column_name => value hash.
|
54
|
+
# Parameters:
|
55
|
+
# * +database+: either :+left+ or :+right+
|
56
|
+
# * +table+: name of the table
|
57
|
+
# * +key+: A column_name => value hash for all primary key columns.
|
58
|
+
def load_record(database, table, key)
|
59
|
+
cursor = session.send(database).select_cursor(
|
60
|
+
:table => table,
|
61
|
+
:row_keys => [key],
|
62
|
+
:type_cast => true
|
63
|
+
)
|
64
|
+
row = nil
|
65
|
+
row = cursor.next_row if cursor.next?
|
66
|
+
cursor.clear
|
67
|
+
row
|
68
|
+
end
|
69
|
+
|
70
|
+
# The current Committer
|
71
|
+
attr_reader :committer
|
72
|
+
private :committer
|
73
|
+
|
74
|
+
# Asks the committer (if it exists) to finalize any open transactions
|
75
|
+
# +success+ should be true if there were no problems, false otherwise.
|
76
|
+
def finalize(success = true)
|
77
|
+
committer.finalize(success)
|
78
|
+
end
|
79
|
+
|
80
|
+
# Converts the row values into their proper types as per table definition.
|
81
|
+
# * +table+: name of the table after whose columns is type-casted.
|
82
|
+
# * +row+: A column_name => value hash of the row
|
83
|
+
# Returns a copy of the column_name => value hash (with type-casted values).
|
84
|
+
def type_cast(table, row)
|
85
|
+
@table_columns ||= {}
|
86
|
+
unless @table_columns.include?(table)
|
87
|
+
column_array = session.left.columns(table)
|
88
|
+
column_hash = {}
|
89
|
+
column_array.each {|column| column_hash[column.name] = column}
|
90
|
+
@table_columns[table] = column_hash
|
91
|
+
end
|
92
|
+
columns = @table_columns[table]
|
93
|
+
type_casted_row = {}
|
94
|
+
row.each_pair do |column_name, value|
|
95
|
+
type_casted_row[column_name] = columns[column_name].type_cast(value)
|
96
|
+
end
|
97
|
+
type_casted_row
|
98
|
+
end
|
99
|
+
|
100
|
+
# Logs the outcome of a replication into the replication log table.
|
101
|
+
# * +diff+: the replicated ReplicationDifference
|
102
|
+
# * +outcome+: string summarizing the outcome of the replication
|
103
|
+
# * +details+: string with further details regarding the replication
|
104
|
+
def log_replication_outcome(diff, outcome, details = nil)
|
105
|
+
table = diff.changes[:left].table
|
106
|
+
key = diff.changes[:left].key
|
107
|
+
if key.size == 1
|
108
|
+
key = key.values[0]
|
109
|
+
else
|
110
|
+
key_parts = session.left.primary_key_names(table).map do |column_name|
|
111
|
+
%Q("#{column_name}"=>#{key[column_name].to_s.inspect})
|
112
|
+
end
|
113
|
+
key = key_parts.join(', ')
|
114
|
+
end
|
115
|
+
rep_outcome, rep_details = fit_description_columns(outcome, details)
|
116
|
+
diff_dump = diff.to_yaml[0...ReplicationInitializer::DIFF_DUMP_SIZE]
|
117
|
+
|
118
|
+
session.left.insert_record "#{options[:rep_prefix]}_logged_events", {
|
119
|
+
:activity => 'replication',
|
120
|
+
:change_table => table,
|
121
|
+
:diff_type => diff.type.to_s,
|
122
|
+
:change_key => key,
|
123
|
+
:left_change_type => (diff.changes[:left] ? diff.changes[:left].type.to_s : nil),
|
124
|
+
:right_change_type => (diff.changes[:right] ? diff.changes[:right].type.to_s : nil),
|
125
|
+
:description => rep_outcome,
|
126
|
+
:long_description => rep_details,
|
127
|
+
:event_time => Time.now,
|
128
|
+
:diff_dump => diff_dump
|
129
|
+
}
|
130
|
+
end
|
131
|
+
|
132
|
+
# Creates a new SyncHelper for the given +TableSync+ instance.
|
133
|
+
def initialize(replication_run)
|
134
|
+
self.replication_run = replication_run
|
135
|
+
|
136
|
+
# Creates the committer. Important as it gives the committer the
|
137
|
+
# opportunity to start transactions
|
138
|
+
committer_class = Committers::committers[options[:committer]]
|
139
|
+
@committer = committer_class.new(session)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,327 @@
|
|
1
|
+
$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
|
2
|
+
|
3
|
+
require 'rubyrep'
|
4
|
+
|
5
|
+
module RR
|
6
|
+
|
7
|
+
# Ensures all preconditions are met to start with replication
|
8
|
+
class ReplicationInitializer
|
9
|
+
|
10
|
+
# The active Session
|
11
|
+
attr_accessor :session
|
12
|
+
|
13
|
+
# Creates a new RepInititializer for the given Session
|
14
|
+
def initialize(session)
|
15
|
+
self.session = session
|
16
|
+
end
|
17
|
+
|
18
|
+
# Returns the options for the given table.
|
19
|
+
# If table is +nil+, returns general options.
|
20
|
+
def options(table = nil)
|
21
|
+
if table
|
22
|
+
session.configuration.options_for_table table
|
23
|
+
else
|
24
|
+
session.configuration.options
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# Creates a trigger logging all table changes
|
29
|
+
# * database: either :+left+ or :+right+
|
30
|
+
# * table: name of the table
|
31
|
+
def create_trigger(database, table)
|
32
|
+
options = self.options(table)
|
33
|
+
|
34
|
+
params = {
|
35
|
+
:trigger_name => "#{options[:rep_prefix]}_#{table}",
|
36
|
+
:table => table,
|
37
|
+
:keys => session.send(database).primary_key_names(table),
|
38
|
+
:log_table => "#{options[:rep_prefix]}_pending_changes",
|
39
|
+
:activity_table => "#{options[:rep_prefix]}_running_flags",
|
40
|
+
:key_sep => options[:key_sep],
|
41
|
+
:exclude_rr_activity => false,
|
42
|
+
}
|
43
|
+
|
44
|
+
session.send(database).create_replication_trigger params
|
45
|
+
end
|
46
|
+
|
47
|
+
# Returns +true+ if the replication trigger for the given table exists.
|
48
|
+
# * database: either :+left+ or :+right+
|
49
|
+
# * table: name of the table
|
50
|
+
def trigger_exists?(database, table)
|
51
|
+
trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
|
52
|
+
session.send(database).replication_trigger_exists? trigger_name, table
|
53
|
+
end
|
54
|
+
|
55
|
+
# Drops the replication trigger of the named table.
|
56
|
+
# * database: either :+left+ or :+right+
|
57
|
+
# * table: name of the table
|
58
|
+
def drop_trigger(database, table)
|
59
|
+
trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
|
60
|
+
session.send(database).drop_replication_trigger trigger_name, table
|
61
|
+
end
|
62
|
+
|
63
|
+
# Ensures that the sequences of the named table (normally the primary key
|
64
|
+
# column) are generated with the correct increment and offset in both
|
65
|
+
# left and right database.
|
66
|
+
# The sequence is always updated in both databases.
|
67
|
+
# * +table_pair+: a hash of names of corresponding :left and :right tables
|
68
|
+
# * +increment+: increment of the sequence
|
69
|
+
# * +left_offset+: offset of table in left database
|
70
|
+
# * +right_offset+: offset of table in right database
|
71
|
+
# E. g. an increment of 2 and offset of 1 will lead to generation of odd
|
72
|
+
# numbers.
|
73
|
+
def ensure_sequence_setup(table_pair, increment, left_offset, right_offset)
|
74
|
+
table_options = options(table_pair[:left])
|
75
|
+
if table_options[:adjust_sequences]
|
76
|
+
rep_prefix = table_options[:rep_prefix]
|
77
|
+
left_sequence_values = session.left.sequence_values rep_prefix, table_pair[:left]
|
78
|
+
right_sequence_values = session.right.sequence_values rep_prefix, table_pair[:right]
|
79
|
+
[:left, :right].each do |database|
|
80
|
+
offset = database == :left ? left_offset : right_offset
|
81
|
+
session.send(database).update_sequences \
|
82
|
+
rep_prefix, table_pair[database], increment, offset,
|
83
|
+
left_sequence_values, right_sequence_values, table_options[:sequence_adjustment_buffer]
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# Restores the original sequence settings for the named table.
|
89
|
+
# (Actually it sets the sequence increment to 1. If before, it had a
|
90
|
+
# different value, then the restoration will not be correct.)
|
91
|
+
# * database: either :+left+ or :+right+
|
92
|
+
# * +table_name+: name of the table
|
93
|
+
def clear_sequence_setup(database, table)
|
94
|
+
table_options = options(table)
|
95
|
+
if table_options[:adjust_sequences]
|
96
|
+
session.send(database).clear_sequence_setup(
|
97
|
+
table_options[:rep_prefix], table
|
98
|
+
)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# Returns +true+ if the change log exists in the specified database.
|
103
|
+
# * database: either :+left+ or :+right+
|
104
|
+
def change_log_exists?(database)
|
105
|
+
session.send(database).tables.include? "#{options[:rep_prefix]}_pending_changes"
|
106
|
+
end
|
107
|
+
|
108
|
+
# Returns +true+ if the replication log exists.
|
109
|
+
def event_log_exists?
|
110
|
+
session.left.tables.include? "#{options[:rep_prefix]}_logged_events"
|
111
|
+
end
|
112
|
+
|
113
|
+
# Drops the change log table in the specified database
|
114
|
+
# * database: either :+left+ or :+right+
|
115
|
+
def drop_change_log(database)
|
116
|
+
session.send(database).drop_table "#{options[:rep_prefix]}_pending_changes"
|
117
|
+
end
|
118
|
+
|
119
|
+
# Drops the replication log table.
|
120
|
+
def drop_event_log
|
121
|
+
session.left.drop_table "#{options[:rep_prefix]}_logged_events"
|
122
|
+
end
|
123
|
+
|
124
|
+
# Size of the replication log column diff_dump
|
125
|
+
DIFF_DUMP_SIZE = 2000
|
126
|
+
|
127
|
+
# Size fo the event log column 'description'
|
128
|
+
DESCRIPTION_SIZE = 255
|
129
|
+
|
130
|
+
# Size of the event log column 'long_description'
|
131
|
+
LONG_DESCRIPTION_SIZE = 1000
|
132
|
+
|
133
|
+
# Ensures that create_table and related statements don't print notices to
|
134
|
+
# stdout. Then restored original message setting.
|
135
|
+
# * +database+: either :+left+ or :+right+
|
136
|
+
def silence_ddl_notices(database)
|
137
|
+
if session.configuration.send(database)[:adapter] =~ /postgres/
|
138
|
+
old_message_level = session.send(database).
|
139
|
+
select_one("show client_min_messages")['client_min_messages']
|
140
|
+
session.send(database).execute "set client_min_messages = warning"
|
141
|
+
end
|
142
|
+
yield
|
143
|
+
ensure
|
144
|
+
if session.configuration.send(database)[:adapter] =~ /postgres/
|
145
|
+
session.send(database).execute "set client_min_messages = #{old_message_level}"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
# Creates the replication log table.
|
150
|
+
def create_event_log
|
151
|
+
silence_ddl_notices(:left) do
|
152
|
+
table_name = "#{options[:rep_prefix]}_logged_events"
|
153
|
+
session.left.create_table "#{options[:rep_prefix]}_logged_events"
|
154
|
+
session.left.add_column table_name, :activity, :string
|
155
|
+
session.left.add_column table_name, :change_table, :string
|
156
|
+
session.left.add_column table_name, :diff_type, :string
|
157
|
+
session.left.add_column table_name, :change_key, :string
|
158
|
+
session.left.add_column table_name, :left_change_type, :string
|
159
|
+
session.left.add_column table_name, :right_change_type, :string
|
160
|
+
session.left.add_column table_name, :description, :string, :limit => DESCRIPTION_SIZE
|
161
|
+
session.left.add_column table_name, :long_description, :string, :limit => LONG_DESCRIPTION_SIZE
|
162
|
+
session.left.add_column table_name, :event_time, :timestamp
|
163
|
+
session.left.add_column table_name, :diff_dump, :string, :limit => DIFF_DUMP_SIZE
|
164
|
+
session.left.remove_column table_name, 'id'
|
165
|
+
session.left.add_big_primary_key table_name, 'id'
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
# Creates the change log table in the specified database
|
170
|
+
# * database: either :+left+ or :+right+
|
171
|
+
def create_change_log(database)
|
172
|
+
silence_ddl_notices(database) do
|
173
|
+
connection = session.send(database)
|
174
|
+
table_name = "#{options[:rep_prefix]}_pending_changes"
|
175
|
+
connection.create_table table_name
|
176
|
+
connection.add_column table_name, :change_table, :string
|
177
|
+
connection.add_column table_name, :change_key, :string
|
178
|
+
connection.add_column table_name, :change_new_key, :string
|
179
|
+
connection.add_column table_name, :change_type, :string
|
180
|
+
connection.add_column table_name, :change_time, :timestamp
|
181
|
+
connection.remove_column table_name, 'id'
|
182
|
+
connection.add_big_primary_key table_name, 'id'
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
# Adds to the current session's configuration an exclusion of rubyrep tables.
|
187
|
+
def exclude_rubyrep_tables
|
188
|
+
session.configuration.exclude_rubyrep_tables
|
189
|
+
end
|
190
|
+
|
191
|
+
# Checks in both databases, if the activity marker tables exist and if not,
|
192
|
+
# creates them.
|
193
|
+
def ensure_activity_markers
|
194
|
+
table_name = "#{options[:rep_prefix]}_running_flags"
|
195
|
+
[:left, :right].each do |database|
|
196
|
+
connection = session.send(database)
|
197
|
+
unless connection.tables.include? table_name
|
198
|
+
silence_ddl_notices(database) do
|
199
|
+
connection.create_table table_name
|
200
|
+
connection.add_column table_name, :active, :integer
|
201
|
+
connection.remove_column table_name, 'id'
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
# Checks if the event log table already exists and creates it if necessary
|
208
|
+
def ensure_event_log
|
209
|
+
create_event_log unless event_log_exists?
|
210
|
+
end
|
211
|
+
|
212
|
+
# Checks in both databases, if the change log tables exists and creates them
|
213
|
+
# if necessary
|
214
|
+
def ensure_change_logs
|
215
|
+
[:left, :right].each do |database|
|
216
|
+
create_change_log(database) unless change_log_exists?(database)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
# Checks in both databases, if the infrastructure tables (change log, event
|
221
|
+
# log) exist and creates them if necessary.
|
222
|
+
def ensure_infrastructure
|
223
|
+
ensure_activity_markers
|
224
|
+
ensure_change_logs
|
225
|
+
ensure_event_log
|
226
|
+
end
|
227
|
+
|
228
|
+
# Checks in both databases, if the change_log tables exist. If yes, drops them.
|
229
|
+
def drop_change_logs
|
230
|
+
[:left, :right].each do |database|
|
231
|
+
drop_change_log(database) if change_log_exists?(database)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
# Checks in both databases, if the activity_marker tables exist. If yes, drops them.
|
236
|
+
def drop_activity_markers
|
237
|
+
table_name = "#{options[:rep_prefix]}_running_flags"
|
238
|
+
[:left, :right].each do |database|
|
239
|
+
if session.send(database).tables.include? table_name
|
240
|
+
session.send(database).drop_table table_name
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
# Removes all rubyrep infrastructure tables from both databases.
|
246
|
+
def drop_infrastructure
|
247
|
+
drop_event_log if event_log_exists?
|
248
|
+
drop_change_logs
|
249
|
+
drop_activity_markers
|
250
|
+
end
|
251
|
+
|
252
|
+
# Checks for tables that have triggers but are not in the list of configured
|
253
|
+
# tables. Removes triggers and restores sequences of those tables.
|
254
|
+
# * +configured_table_pairs+:
|
255
|
+
# An array of table pairs (e. g. [{:left => 'xy', :right => 'xy2'}]).
|
256
|
+
def restore_unconfigured_tables(configured_table_pairs = session.configured_table_pairs)
|
257
|
+
[:left, :right].each do |database|
|
258
|
+
configured_tables = configured_table_pairs.map {|table_pair| table_pair[database]}
|
259
|
+
unconfigured_tables = session.send(database).tables - configured_tables
|
260
|
+
unconfigured_tables.each do |table|
|
261
|
+
if trigger_exists?(database, table)
|
262
|
+
drop_trigger(database, table)
|
263
|
+
session.send(database).execute(
|
264
|
+
"delete from #{options[:rep_prefix]}_pending_changes where change_table = '#{table}'")
|
265
|
+
end
|
266
|
+
clear_sequence_setup(database, table)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
# Calls the potentially provided :+after_init+ handler after infrastructure
|
272
|
+
# tables are created.
|
273
|
+
def call_after_infrastructure_setup_handler
|
274
|
+
handler = session.configuration.options[:after_infrastructure_setup]
|
275
|
+
handler.call(session) if handler
|
276
|
+
end
|
277
|
+
|
278
|
+
# Prepares the database / tables for replication.
|
279
|
+
def prepare_replication
|
280
|
+
exclude_rubyrep_tables
|
281
|
+
|
282
|
+
puts "Verifying RubyRep tables"
|
283
|
+
ensure_infrastructure
|
284
|
+
|
285
|
+
call_after_infrastructure_setup_handler
|
286
|
+
|
287
|
+
puts "Checking for and removing rubyrep triggers from unconfigured tables"
|
288
|
+
restore_unconfigured_tables
|
289
|
+
|
290
|
+
puts "Verifying rubyrep triggers of configured tables"
|
291
|
+
unsynced_table_pairs = []
|
292
|
+
table_pairs = session.sort_table_pairs(session.configured_table_pairs)
|
293
|
+
table_pairs.each do |table_pair|
|
294
|
+
table_options = options(table_pair[:left])
|
295
|
+
ensure_sequence_setup table_pair,
|
296
|
+
table_options[:sequence_increment],
|
297
|
+
table_options[:left_sequence_offset],
|
298
|
+
table_options[:right_sequence_offset]
|
299
|
+
|
300
|
+
unsynced = false
|
301
|
+
[:left, :right].each do |database|
|
302
|
+
unless trigger_exists? database, table_pair[database]
|
303
|
+
create_trigger database, table_pair[database]
|
304
|
+
unsynced = true
|
305
|
+
end
|
306
|
+
end
|
307
|
+
if unsynced and table_options[:initial_sync]
|
308
|
+
unsynced_table_pairs << table_pair
|
309
|
+
end
|
310
|
+
end
|
311
|
+
unsynced_table_specs = unsynced_table_pairs.map do |table_pair|
|
312
|
+
"#{table_pair[:left]}, #{table_pair[:right]}"
|
313
|
+
end
|
314
|
+
|
315
|
+
unless unsynced_table_specs.empty?
|
316
|
+
puts "Executing initial table syncs"
|
317
|
+
runner = SyncRunner.new
|
318
|
+
runner.session = session
|
319
|
+
runner.options = {:table_specs => unsynced_table_specs}
|
320
|
+
runner.execute
|
321
|
+
end
|
322
|
+
|
323
|
+
puts "Starting replication"
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
end
|