andyjeffries-rubyrep 1.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (125) hide show
  1. data/History.txt +83 -0
  2. data/License.txt +20 -0
  3. data/Manifest.txt +151 -0
  4. data/README.txt +37 -0
  5. data/bin/rubyrep +8 -0
  6. data/lib/rubyrep.rb +72 -0
  7. data/lib/rubyrep/base_runner.rb +195 -0
  8. data/lib/rubyrep/command_runner.rb +144 -0
  9. data/lib/rubyrep/committers/buffered_committer.rb +151 -0
  10. data/lib/rubyrep/committers/committers.rb +152 -0
  11. data/lib/rubyrep/configuration.rb +275 -0
  12. data/lib/rubyrep/connection_extenders/connection_extenders.rb +165 -0
  13. data/lib/rubyrep/connection_extenders/jdbc_extender.rb +65 -0
  14. data/lib/rubyrep/connection_extenders/mysql_extender.rb +59 -0
  15. data/lib/rubyrep/connection_extenders/postgresql_extender.rb +277 -0
  16. data/lib/rubyrep/database_proxy.rb +52 -0
  17. data/lib/rubyrep/direct_table_scan.rb +75 -0
  18. data/lib/rubyrep/generate_runner.rb +105 -0
  19. data/lib/rubyrep/initializer.rb +39 -0
  20. data/lib/rubyrep/log_helper.rb +30 -0
  21. data/lib/rubyrep/logged_change.rb +160 -0
  22. data/lib/rubyrep/logged_change_loader.rb +197 -0
  23. data/lib/rubyrep/noisy_connection.rb +80 -0
  24. data/lib/rubyrep/proxied_table_scan.rb +171 -0
  25. data/lib/rubyrep/proxy_block_cursor.rb +145 -0
  26. data/lib/rubyrep/proxy_connection.rb +431 -0
  27. data/lib/rubyrep/proxy_cursor.rb +44 -0
  28. data/lib/rubyrep/proxy_row_cursor.rb +43 -0
  29. data/lib/rubyrep/proxy_runner.rb +89 -0
  30. data/lib/rubyrep/replication_difference.rb +100 -0
  31. data/lib/rubyrep/replication_extenders/mysql_replication.rb +271 -0
  32. data/lib/rubyrep/replication_extenders/postgresql_replication.rb +236 -0
  33. data/lib/rubyrep/replication_extenders/replication_extenders.rb +26 -0
  34. data/lib/rubyrep/replication_helper.rb +142 -0
  35. data/lib/rubyrep/replication_initializer.rb +327 -0
  36. data/lib/rubyrep/replication_run.rb +142 -0
  37. data/lib/rubyrep/replication_runner.rb +166 -0
  38. data/lib/rubyrep/replicators/replicators.rb +42 -0
  39. data/lib/rubyrep/replicators/two_way_replicator.rb +361 -0
  40. data/lib/rubyrep/scan_progress_printers/progress_bar.rb +65 -0
  41. data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb +65 -0
  42. data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb +111 -0
  43. data/lib/rubyrep/scan_report_printers/scan_report_printers.rb +67 -0
  44. data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb +75 -0
  45. data/lib/rubyrep/scan_runner.rb +25 -0
  46. data/lib/rubyrep/session.rb +230 -0
  47. data/lib/rubyrep/sync_helper.rb +121 -0
  48. data/lib/rubyrep/sync_runner.rb +31 -0
  49. data/lib/rubyrep/syncers/syncers.rb +112 -0
  50. data/lib/rubyrep/syncers/two_way_syncer.rb +174 -0
  51. data/lib/rubyrep/table_scan.rb +54 -0
  52. data/lib/rubyrep/table_scan_helper.rb +46 -0
  53. data/lib/rubyrep/table_sorter.rb +70 -0
  54. data/lib/rubyrep/table_spec_resolver.rb +142 -0
  55. data/lib/rubyrep/table_sync.rb +90 -0
  56. data/lib/rubyrep/task_sweeper.rb +77 -0
  57. data/lib/rubyrep/trigger_mode_switcher.rb +63 -0
  58. data/lib/rubyrep/type_casting_cursor.rb +31 -0
  59. data/lib/rubyrep/uninstall_runner.rb +93 -0
  60. data/lib/rubyrep/version.rb +9 -0
  61. data/rubyrep +8 -0
  62. data/rubyrep.bat +4 -0
  63. data/setup.rb +1585 -0
  64. data/spec/base_runner_spec.rb +218 -0
  65. data/spec/buffered_committer_spec.rb +274 -0
  66. data/spec/command_runner_spec.rb +145 -0
  67. data/spec/committers_spec.rb +178 -0
  68. data/spec/configuration_spec.rb +203 -0
  69. data/spec/connection_extender_interface_spec.rb +141 -0
  70. data/spec/connection_extenders_registration_spec.rb +164 -0
  71. data/spec/database_proxy_spec.rb +48 -0
  72. data/spec/database_rake_spec.rb +40 -0
  73. data/spec/db_specific_connection_extenders_spec.rb +34 -0
  74. data/spec/db_specific_replication_extenders_spec.rb +38 -0
  75. data/spec/direct_table_scan_spec.rb +61 -0
  76. data/spec/dolphins.jpg +0 -0
  77. data/spec/generate_runner_spec.rb +84 -0
  78. data/spec/initializer_spec.rb +46 -0
  79. data/spec/log_helper_spec.rb +39 -0
  80. data/spec/logged_change_loader_spec.rb +68 -0
  81. data/spec/logged_change_spec.rb +470 -0
  82. data/spec/noisy_connection_spec.rb +78 -0
  83. data/spec/postgresql_replication_spec.rb +48 -0
  84. data/spec/postgresql_schema_support_spec.rb +212 -0
  85. data/spec/postgresql_support_spec.rb +63 -0
  86. data/spec/progress_bar_spec.rb +77 -0
  87. data/spec/proxied_table_scan_spec.rb +151 -0
  88. data/spec/proxy_block_cursor_spec.rb +197 -0
  89. data/spec/proxy_connection_spec.rb +423 -0
  90. data/spec/proxy_cursor_spec.rb +56 -0
  91. data/spec/proxy_row_cursor_spec.rb +66 -0
  92. data/spec/proxy_runner_spec.rb +70 -0
  93. data/spec/replication_difference_spec.rb +161 -0
  94. data/spec/replication_extender_interface_spec.rb +367 -0
  95. data/spec/replication_extenders_spec.rb +32 -0
  96. data/spec/replication_helper_spec.rb +178 -0
  97. data/spec/replication_initializer_spec.rb +509 -0
  98. data/spec/replication_run_spec.rb +443 -0
  99. data/spec/replication_runner_spec.rb +254 -0
  100. data/spec/replicators_spec.rb +36 -0
  101. data/spec/rubyrep_spec.rb +8 -0
  102. data/spec/scan_detail_reporter_spec.rb +119 -0
  103. data/spec/scan_progress_printers_spec.rb +68 -0
  104. data/spec/scan_report_printers_spec.rb +67 -0
  105. data/spec/scan_runner_spec.rb +50 -0
  106. data/spec/scan_summary_reporter_spec.rb +61 -0
  107. data/spec/session_spec.rb +253 -0
  108. data/spec/spec.opts +1 -0
  109. data/spec/spec_helper.rb +305 -0
  110. data/spec/strange_name_support_spec.rb +135 -0
  111. data/spec/sync_helper_spec.rb +169 -0
  112. data/spec/sync_runner_spec.rb +78 -0
  113. data/spec/syncers_spec.rb +171 -0
  114. data/spec/table_scan_helper_spec.rb +36 -0
  115. data/spec/table_scan_spec.rb +49 -0
  116. data/spec/table_sorter_spec.rb +30 -0
  117. data/spec/table_spec_resolver_spec.rb +111 -0
  118. data/spec/table_sync_spec.rb +140 -0
  119. data/spec/task_sweeper_spec.rb +47 -0
  120. data/spec/trigger_mode_switcher_spec.rb +83 -0
  121. data/spec/two_way_replicator_spec.rb +721 -0
  122. data/spec/two_way_syncer_spec.rb +256 -0
  123. data/spec/type_casting_cursor_spec.rb +50 -0
  124. data/spec/uninstall_runner_spec.rb +93 -0
  125. metadata +190 -0
@@ -0,0 +1,236 @@
1
+ module RR
2
+ module ReplicationExtenders
3
+
4
+ # Provides PostgreSQL specific functionality for database replication
5
+ module PostgreSQLReplication
6
+ RR::ReplicationExtenders.register :postgresql => self
7
+
8
+ # Returns the key clause that is used in the trigger function.
9
+ # * +trigger_var+: should be either 'NEW' or 'OLD'
10
+ # * +params+: the parameter hash as described in #create_rep_trigger
11
+ def key_clause(trigger_var, params)
12
+ params[:keys].
13
+ map { |key| "'#{key}#{params[:key_sep]}' || #{trigger_var}.#{quote_column_name(key)}"}.
14
+ join(" || '#{params[:key_sep]}' || ")
15
+ end
16
+ private :key_clause
17
+
18
+ # Returns the schema prefix (including dot) that will be used by the
19
+ # triggers to write into the rubyrep infrastructure tables.
20
+ # To avoid setting the wrong prefix, it will only return a schema prefix
21
+ # if the current search path
22
+ # * consists of only a single schema
23
+ # * does not consists of a variable search path
24
+ # (i. e. the default "$user")
25
+ def schema_prefix
26
+ unless @schema_prefix
27
+ search_path = select_one("show search_path")['search_path']
28
+ if search_path =~ /[,$]/
29
+ @schema_prefix = ""
30
+ else
31
+ @schema_prefix = %("#{search_path}".)
32
+ end
33
+ end
34
+ @schema_prefix
35
+ end
36
+
37
+ # Creates or replaces the replication trigger function.
38
+ # See #create_replication_trigger for a descriptions of the +params+ hash.
39
+ def create_or_replace_replication_trigger_function(params)
40
+ # first check, if PL/SQL is already activated and if not, do so.
41
+ if select_all("select lanname from pg_language where lanname = 'plpgsql'").empty?
42
+ execute "CREATE LANGUAGE plpgsql"
43
+ end
44
+
45
+ activity_check = ""
46
+ if params[:exclude_rr_activity] then
47
+ activity_check = <<-end_sql
48
+ PERFORM ACTIVE FROM #{schema_prefix}#{params[:activity_table]};
49
+ IF FOUND THEN
50
+ RETURN NULL;
51
+ END IF;
52
+ end_sql
53
+ end
54
+
55
+ version_string = select_value("select version();")
56
+ version = version_string.gsub(/^\s*postgresql\s*([0-9.]+).*$/i, '\1')
57
+ if version >= '8.4'
58
+ modification_check = <<-end_sql
59
+ IF NEW IS NOT DISTINCT FROM OLD THEN
60
+ RETURN NULL;
61
+ END IF;
62
+ end_sql
63
+ else
64
+ modification_check = ""
65
+ end
66
+
67
+ # now create the trigger
68
+ execute(<<-end_sql)
69
+ CREATE OR REPLACE FUNCTION "#{params[:trigger_name]}"() RETURNS TRIGGER AS $change_trigger$
70
+ BEGIN
71
+ #{activity_check}
72
+ IF (TG_OP = 'DELETE') THEN
73
+ INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
74
+ SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
75
+ ELSIF (TG_OP = 'UPDATE') THEN
76
+ #{modification_check}
77
+ INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
78
+ SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
79
+ ELSIF (TG_OP = 'INSERT') THEN
80
+ INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
81
+ SELECT '#{params[:table]}', #{key_clause('NEW', params)}, 'I', now();
82
+ END IF;
83
+ RETURN NULL; -- result is ignored since this is an AFTER trigger
84
+ END;
85
+ $change_trigger$ LANGUAGE plpgsql;
86
+ end_sql
87
+
88
+ end
89
+
90
+ # Creates a trigger to log all changes for the given table.
91
+ # +params+ is a hash with all necessary information:
92
+ # * :+trigger_name+: name of the trigger
93
+ # * :+table+: name of the table that should be monitored
94
+ # * :+keys+: array of names of the key columns of the monitored table
95
+ # * :+log_table+: name of the table receiving all change notifications
96
+ # * :+activity_table+: name of the table receiving the rubyrep activity information
97
+ # * :+key_sep+: column seperator to be used in the key column of the log table
98
+ # * :+exclude_rr_activity+:
99
+ # if true, the trigger will check and filter out changes initiated by RubyRep
100
+ def create_replication_trigger(params)
101
+ create_or_replace_replication_trigger_function params
102
+
103
+ execute(<<-end_sql)
104
+ CREATE TRIGGER "#{params[:trigger_name]}"
105
+ AFTER INSERT OR UPDATE OR DELETE ON "#{params[:table]}"
106
+ FOR EACH ROW EXECUTE PROCEDURE #{schema_prefix}"#{params[:trigger_name]}"();
107
+ end_sql
108
+ end
109
+
110
+ # Removes a trigger and related trigger procedure.
111
+ # * +trigger_name+: name of the trigger
112
+ # * +table_name+: name of the table for which the trigger exists
113
+ def drop_replication_trigger(trigger_name, table_name)
114
+ execute "DROP TRIGGER \"#{trigger_name}\" ON \"#{table_name}\";"
115
+ execute "DROP FUNCTION \"#{trigger_name}\"();"
116
+ end
117
+
118
+ # Returns +true+ if the named trigger exists for the named table.
119
+ # * +trigger_name+: name of the trigger
120
+ # * +table_name+: name of the table
121
+ def replication_trigger_exists?(trigger_name, table_name)
122
+ !select_all(<<-end_sql).empty?
123
+ select 1 from information_schema.triggers
124
+ where event_object_schema in (#{schemas})
125
+ and trigger_name = '#{trigger_name}'
126
+ and event_object_table = '#{table_name}'
127
+ end_sql
128
+ end
129
+
130
+ # Returns all unadjusted sequences of the given table.
131
+ # Parameters:
132
+ # * +rep_prefix+: not used (necessary) for the Postgres
133
+ # * +table_name+: name of the table
134
+ # Return value: a hash with
135
+ # * key: sequence name
136
+ # * value: a hash with
137
+ # * :+increment+: current sequence increment
138
+ # * :+value+: current value
139
+ def sequence_values(rep_prefix, table_name)
140
+ result = {}
141
+ sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
142
+ select s.relname
143
+ from pg_class as t
144
+ join pg_depend as r on t.oid = r.refobjid
145
+ join pg_class as s on r.objid = s.oid
146
+ and s.relkind = 'S'
147
+ and t.relname = '#{table_name}' AND t.relnamespace IN
148
+ (SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
149
+ end_sql
150
+ sequence_names.each do |sequence_name|
151
+ row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
152
+ result[sequence_name] = {
153
+ :increment => row['increment_by'].to_i,
154
+ :value => row['last_value'].to_i
155
+ }
156
+ end
157
+ result
158
+ end
159
+
160
+ # Ensures that the sequences of the named table (normally the primary key
161
+ # column) are generated with the correct increment and offset.
162
+ # * +rep_prefix+: not used (necessary) for the Postgres
163
+ # * +table_name+: name of the table (not used for Postgres)
164
+ # * +increment+: increment of the sequence
165
+ # * +offset+: offset
166
+ # * +left_sequence_values+:
167
+ # hash as returned by #sequence_values for the left database
168
+ # * +right_sequence_values+:
169
+ # hash as returned by #sequence_values for the right database
170
+ # * +adjustment_buffer+:
171
+ # the "gap" that is created during sequence update to avoid concurrency problems
172
+ # E. g. an increment of 2 and offset of 1 will lead to generation of odd
173
+ # numbers.
174
+ def update_sequences(
175
+ rep_prefix, table_name, increment, offset,
176
+ left_sequence_values, right_sequence_values, adjustment_buffer)
177
+ left_sequence_values.each do |sequence_name, left_current_value|
178
+ row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
179
+ current_increment = row['increment_by'].to_i
180
+ current_value = row['last_value'].to_i
181
+ unless current_increment == increment and current_value % increment == offset
182
+ max_current_value =
183
+ [left_current_value[:value], right_sequence_values[sequence_name][:value]].max +
184
+ adjustment_buffer
185
+ new_start = max_current_value - (max_current_value % increment) + increment + offset
186
+ execute(<<-end_sql)
187
+ alter sequence "#{sequence_name}" increment by #{increment} restart with #{new_start}
188
+ end_sql
189
+ end
190
+ end
191
+ end
192
+
193
+ # Restores the original sequence settings.
194
+ # (Actually it sets the sequence increment to 1. If before, it had a
195
+ # different value, then the restoration will not be correct.)
196
+ # * +rep_prefix+: not used (necessary) for the Postgres
197
+ # * +table_name+: name of the table
198
+ def clear_sequence_setup(rep_prefix, table_name)
199
+ sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
200
+ select s.relname
201
+ from pg_class as t
202
+ join pg_depend as r on t.oid = r.refobjid
203
+ join pg_class as s on r.objid = s.oid
204
+ and s.relkind = 'S'
205
+ and t.relname = '#{table_name}' and t.relnamespace IN
206
+ (SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
207
+ end_sql
208
+ sequence_names.each do |sequence_name|
209
+ execute(<<-end_sql)
210
+ alter sequence "#{sequence_name}" increment by 1
211
+ end_sql
212
+ end
213
+ end
214
+
215
+ # Adds a big (8 byte value), auto-incrementing primary key column to the
216
+ # specified table.
217
+ # * table_name: name of the target table
218
+ # * key_name: name of the primary key column
219
+ def add_big_primary_key(table_name, key_name)
220
+ old_message_level = select_one("show client_min_messages")['client_min_messages']
221
+ execute "set client_min_messages = warning"
222
+ execute(<<-end_sql)
223
+ alter table "#{table_name}" add column #{key_name} bigserial
224
+ end_sql
225
+
226
+ execute(<<-end_sql)
227
+ alter table "#{table_name}" add constraint #{table_name}_#{key_name}_pkey primary key (#{key_name})
228
+ end_sql
229
+
230
+ ensure
231
+ execute "set client_min_messages = #{old_message_level}"
232
+ end
233
+ end
234
+ end
235
+ end
236
+
@@ -0,0 +1,26 @@
1
+ module RR
2
+
3
+ # Replication extenders are modules that provide database specific functionality
4
+ # required for replication. They are mixed into ActiveRecord database connections.
5
+ # This module itself only provides functionality to register and retrieve
6
+ # such extenders.
7
+ module ReplicationExtenders
8
+ # Returns a Hash of currently registered replication extenders.
9
+ # (Empty Hash if no replication extenders were defined.)
10
+ def self.extenders
11
+ @extenders ||= {}
12
+ @extenders
13
+ end
14
+
15
+ # Registers one or multiple replication extender.
16
+ # extender is a Hash with
17
+ # key:: The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
18
+ # value:: Name of the module implementing the replication extender
19
+ def self.register(extender)
20
+ @extenders ||= {}
21
+ @extenders.merge! extender
22
+ end
23
+ end
24
+ end
25
+
26
+
@@ -0,0 +1,142 @@
1
+ module RR
2
+
3
+ # Provides helper functionality for replicators.
4
+ # The methods exposed by this class are intended to provide a stable interface
5
+ # for third party replicators.
6
+ class ReplicationHelper
7
+
8
+ include LogHelper
9
+
10
+ # The current +ReplicationRun+ instance
11
+ attr_accessor :replication_run
12
+
13
+ # The active +Session+
14
+ def session; replication_run.session; end
15
+
16
+ # Current options
17
+ def options; @options ||= session.configuration.options; end
18
+
19
+ # Returns the options for the specified table name.
20
+ # * +table+: name of the table (left database version)
21
+ def options_for_table(table)
22
+ @options_for_table ||= {}
23
+ unless @options_for_table.include? table
24
+ @options_for_table[table] = session.configuration.options_for_table(table)
25
+ end
26
+ @options_for_table[table]
27
+ end
28
+
29
+ # Delegates to Session#corresponding_table
30
+ def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
31
+
32
+ # Returns +true+ if a new transaction was started since the last
33
+ # insert / update / delete.
34
+ def new_transaction?
35
+ committer.new_transaction?
36
+ end
37
+
38
+ # Delegates to Committers::BufferedCommitter#insert_record
39
+ def insert_record(database, table, values)
40
+ committer.insert_record(database, table, values)
41
+ end
42
+
43
+ # Delegates to Committers::BufferedCommitter#update_record
44
+ def update_record(database, table, values, old_key = nil)
45
+ committer.update_record(database, table, values, old_key)
46
+ end
47
+
48
+ # Delegates to Committers::BufferedCommitter#delete_record
49
+ def delete_record(database, table, values)
50
+ committer.delete_record(database, table, values)
51
+ end
52
+
53
+ # Loads the specified record. Returns an according column_name => value hash.
54
+ # Parameters:
55
+ # * +database+: either :+left+ or :+right+
56
+ # * +table+: name of the table
57
+ # * +key+: A column_name => value hash for all primary key columns.
58
+ def load_record(database, table, key)
59
+ cursor = session.send(database).select_cursor(
60
+ :table => table,
61
+ :row_keys => [key],
62
+ :type_cast => true
63
+ )
64
+ row = nil
65
+ row = cursor.next_row if cursor.next?
66
+ cursor.clear
67
+ row
68
+ end
69
+
70
+ # The current Committer
71
+ attr_reader :committer
72
+ private :committer
73
+
74
+ # Asks the committer (if it exists) to finalize any open transactions
75
+ # +success+ should be true if there were no problems, false otherwise.
76
+ def finalize(success = true)
77
+ committer.finalize(success)
78
+ end
79
+
80
+ # Converts the row values into their proper types as per table definition.
81
+ # * +table+: name of the table after whose columns is type-casted.
82
+ # * +row+: A column_name => value hash of the row
83
+ # Returns a copy of the column_name => value hash (with type-casted values).
84
+ def type_cast(table, row)
85
+ @table_columns ||= {}
86
+ unless @table_columns.include?(table)
87
+ column_array = session.left.columns(table)
88
+ column_hash = {}
89
+ column_array.each {|column| column_hash[column.name] = column}
90
+ @table_columns[table] = column_hash
91
+ end
92
+ columns = @table_columns[table]
93
+ type_casted_row = {}
94
+ row.each_pair do |column_name, value|
95
+ type_casted_row[column_name] = columns[column_name].type_cast(value)
96
+ end
97
+ type_casted_row
98
+ end
99
+
100
+ # Logs the outcome of a replication into the replication log table.
101
+ # * +diff+: the replicated ReplicationDifference
102
+ # * +outcome+: string summarizing the outcome of the replication
103
+ # * +details+: string with further details regarding the replication
104
+ def log_replication_outcome(diff, outcome, details = nil)
105
+ table = diff.changes[:left].table
106
+ key = diff.changes[:left].key
107
+ if key.size == 1
108
+ key = key.values[0]
109
+ else
110
+ key_parts = session.left.primary_key_names(table).map do |column_name|
111
+ %Q("#{column_name}"=>#{key[column_name].to_s.inspect})
112
+ end
113
+ key = key_parts.join(', ')
114
+ end
115
+ rep_outcome, rep_details = fit_description_columns(outcome, details)
116
+ diff_dump = diff.to_yaml[0...ReplicationInitializer::DIFF_DUMP_SIZE]
117
+
118
+ session.left.insert_record "#{options[:rep_prefix]}_logged_events", {
119
+ :activity => 'replication',
120
+ :change_table => table,
121
+ :diff_type => diff.type.to_s,
122
+ :change_key => key,
123
+ :left_change_type => (diff.changes[:left] ? diff.changes[:left].type.to_s : nil),
124
+ :right_change_type => (diff.changes[:right] ? diff.changes[:right].type.to_s : nil),
125
+ :description => rep_outcome,
126
+ :long_description => rep_details,
127
+ :event_time => Time.now,
128
+ :diff_dump => diff_dump
129
+ }
130
+ end
131
+
132
+ # Creates a new SyncHelper for the given +TableSync+ instance.
133
+ def initialize(replication_run)
134
+ self.replication_run = replication_run
135
+
136
+ # Creates the committer. Important as it gives the committer the
137
+ # opportunity to start transactions
138
+ committer_class = Committers::committers[options[:committer]]
139
+ @committer = committer_class.new(session)
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,327 @@
1
+ $LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
2
+
3
+ require 'rubyrep'
4
+
5
+ module RR
6
+
7
+ # Ensures all preconditions are met to start with replication
8
+ class ReplicationInitializer
9
+
10
+ # The active Session
11
+ attr_accessor :session
12
+
13
+ # Creates a new RepInititializer for the given Session
14
+ def initialize(session)
15
+ self.session = session
16
+ end
17
+
18
+ # Returns the options for the given table.
19
+ # If table is +nil+, returns general options.
20
+ def options(table = nil)
21
+ if table
22
+ session.configuration.options_for_table table
23
+ else
24
+ session.configuration.options
25
+ end
26
+ end
27
+
28
+ # Creates a trigger logging all table changes
29
+ # * database: either :+left+ or :+right+
30
+ # * table: name of the table
31
+ def create_trigger(database, table)
32
+ options = self.options(table)
33
+
34
+ params = {
35
+ :trigger_name => "#{options[:rep_prefix]}_#{table}",
36
+ :table => table,
37
+ :keys => session.send(database).primary_key_names(table),
38
+ :log_table => "#{options[:rep_prefix]}_pending_changes",
39
+ :activity_table => "#{options[:rep_prefix]}_running_flags",
40
+ :key_sep => options[:key_sep],
41
+ :exclude_rr_activity => false,
42
+ }
43
+
44
+ session.send(database).create_replication_trigger params
45
+ end
46
+
47
+ # Returns +true+ if the replication trigger for the given table exists.
48
+ # * database: either :+left+ or :+right+
49
+ # * table: name of the table
50
+ def trigger_exists?(database, table)
51
+ trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
52
+ session.send(database).replication_trigger_exists? trigger_name, table
53
+ end
54
+
55
+ # Drops the replication trigger of the named table.
56
+ # * database: either :+left+ or :+right+
57
+ # * table: name of the table
58
+ def drop_trigger(database, table)
59
+ trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
60
+ session.send(database).drop_replication_trigger trigger_name, table
61
+ end
62
+
63
+ # Ensures that the sequences of the named table (normally the primary key
64
+ # column) are generated with the correct increment and offset in both
65
+ # left and right database.
66
+ # The sequence is always updated in both databases.
67
+ # * +table_pair+: a hash of names of corresponding :left and :right tables
68
+ # * +increment+: increment of the sequence
69
+ # * +left_offset+: offset of table in left database
70
+ # * +right_offset+: offset of table in right database
71
+ # E. g. an increment of 2 and offset of 1 will lead to generation of odd
72
+ # numbers.
73
+ def ensure_sequence_setup(table_pair, increment, left_offset, right_offset)
74
+ table_options = options(table_pair[:left])
75
+ if table_options[:adjust_sequences]
76
+ rep_prefix = table_options[:rep_prefix]
77
+ left_sequence_values = session.left.sequence_values rep_prefix, table_pair[:left]
78
+ right_sequence_values = session.right.sequence_values rep_prefix, table_pair[:right]
79
+ [:left, :right].each do |database|
80
+ offset = database == :left ? left_offset : right_offset
81
+ session.send(database).update_sequences \
82
+ rep_prefix, table_pair[database], increment, offset,
83
+ left_sequence_values, right_sequence_values, table_options[:sequence_adjustment_buffer]
84
+ end
85
+ end
86
+ end
87
+
88
+ # Restores the original sequence settings for the named table.
89
+ # (Actually it sets the sequence increment to 1. If before, it had a
90
+ # different value, then the restoration will not be correct.)
91
+ # * database: either :+left+ or :+right+
92
+ # * +table_name+: name of the table
93
+ def clear_sequence_setup(database, table)
94
+ table_options = options(table)
95
+ if table_options[:adjust_sequences]
96
+ session.send(database).clear_sequence_setup(
97
+ table_options[:rep_prefix], table
98
+ )
99
+ end
100
+ end
101
+
102
+ # Returns +true+ if the change log exists in the specified database.
103
+ # * database: either :+left+ or :+right+
104
+ def change_log_exists?(database)
105
+ session.send(database).tables.include? "#{options[:rep_prefix]}_pending_changes"
106
+ end
107
+
108
+ # Returns +true+ if the replication log exists.
109
+ def event_log_exists?
110
+ session.left.tables.include? "#{options[:rep_prefix]}_logged_events"
111
+ end
112
+
113
+ # Drops the change log table in the specified database
114
+ # * database: either :+left+ or :+right+
115
+ def drop_change_log(database)
116
+ session.send(database).drop_table "#{options[:rep_prefix]}_pending_changes"
117
+ end
118
+
119
+ # Drops the replication log table.
120
+ def drop_event_log
121
+ session.left.drop_table "#{options[:rep_prefix]}_logged_events"
122
+ end
123
+
124
+ # Size of the replication log column diff_dump
125
+ DIFF_DUMP_SIZE = 2000
126
+
127
+ # Size fo the event log column 'description'
128
+ DESCRIPTION_SIZE = 255
129
+
130
+ # Size of the event log column 'long_description'
131
+ LONG_DESCRIPTION_SIZE = 1000
132
+
133
+ # Ensures that create_table and related statements don't print notices to
134
+ # stdout. Then restored original message setting.
135
+ # * +database+: either :+left+ or :+right+
136
+ def silence_ddl_notices(database)
137
+ if session.configuration.send(database)[:adapter] =~ /postgres/
138
+ old_message_level = session.send(database).
139
+ select_one("show client_min_messages")['client_min_messages']
140
+ session.send(database).execute "set client_min_messages = warning"
141
+ end
142
+ yield
143
+ ensure
144
+ if session.configuration.send(database)[:adapter] =~ /postgres/
145
+ session.send(database).execute "set client_min_messages = #{old_message_level}"
146
+ end
147
+ end
148
+
149
+ # Creates the replication log table.
150
+ def create_event_log
151
+ silence_ddl_notices(:left) do
152
+ table_name = "#{options[:rep_prefix]}_logged_events"
153
+ session.left.create_table "#{options[:rep_prefix]}_logged_events"
154
+ session.left.add_column table_name, :activity, :string
155
+ session.left.add_column table_name, :change_table, :string
156
+ session.left.add_column table_name, :diff_type, :string
157
+ session.left.add_column table_name, :change_key, :string
158
+ session.left.add_column table_name, :left_change_type, :string
159
+ session.left.add_column table_name, :right_change_type, :string
160
+ session.left.add_column table_name, :description, :string, :limit => DESCRIPTION_SIZE
161
+ session.left.add_column table_name, :long_description, :string, :limit => LONG_DESCRIPTION_SIZE
162
+ session.left.add_column table_name, :event_time, :timestamp
163
+ session.left.add_column table_name, :diff_dump, :string, :limit => DIFF_DUMP_SIZE
164
+ session.left.remove_column table_name, 'id'
165
+ session.left.add_big_primary_key table_name, 'id'
166
+ end
167
+ end
168
+
169
+ # Creates the change log table in the specified database
170
+ # * database: either :+left+ or :+right+
171
+ def create_change_log(database)
172
+ silence_ddl_notices(database) do
173
+ connection = session.send(database)
174
+ table_name = "#{options[:rep_prefix]}_pending_changes"
175
+ connection.create_table table_name
176
+ connection.add_column table_name, :change_table, :string
177
+ connection.add_column table_name, :change_key, :string
178
+ connection.add_column table_name, :change_new_key, :string
179
+ connection.add_column table_name, :change_type, :string
180
+ connection.add_column table_name, :change_time, :timestamp
181
+ connection.remove_column table_name, 'id'
182
+ connection.add_big_primary_key table_name, 'id'
183
+ end
184
+ end
185
+
186
+ # Adds to the current session's configuration an exclusion of rubyrep tables.
187
+ def exclude_rubyrep_tables
188
+ session.configuration.exclude_rubyrep_tables
189
+ end
190
+
191
+ # Checks in both databases, if the activity marker tables exist and if not,
192
+ # creates them.
193
+ def ensure_activity_markers
194
+ table_name = "#{options[:rep_prefix]}_running_flags"
195
+ [:left, :right].each do |database|
196
+ connection = session.send(database)
197
+ unless connection.tables.include? table_name
198
+ silence_ddl_notices(database) do
199
+ connection.create_table table_name
200
+ connection.add_column table_name, :active, :integer
201
+ connection.remove_column table_name, 'id'
202
+ end
203
+ end
204
+ end
205
+ end
206
+
207
+ # Checks if the event log table already exists and creates it if necessary
208
+ def ensure_event_log
209
+ create_event_log unless event_log_exists?
210
+ end
211
+
212
+ # Checks in both databases, if the change log tables exists and creates them
213
+ # if necessary
214
+ def ensure_change_logs
215
+ [:left, :right].each do |database|
216
+ create_change_log(database) unless change_log_exists?(database)
217
+ end
218
+ end
219
+
220
+ # Checks in both databases, if the infrastructure tables (change log, event
221
+ # log) exist and creates them if necessary.
222
+ def ensure_infrastructure
223
+ ensure_activity_markers
224
+ ensure_change_logs
225
+ ensure_event_log
226
+ end
227
+
228
+ # Checks in both databases, if the change_log tables exist. If yes, drops them.
229
+ def drop_change_logs
230
+ [:left, :right].each do |database|
231
+ drop_change_log(database) if change_log_exists?(database)
232
+ end
233
+ end
234
+
235
+ # Checks in both databases, if the activity_marker tables exist. If yes, drops them.
236
+ def drop_activity_markers
237
+ table_name = "#{options[:rep_prefix]}_running_flags"
238
+ [:left, :right].each do |database|
239
+ if session.send(database).tables.include? table_name
240
+ session.send(database).drop_table table_name
241
+ end
242
+ end
243
+ end
244
+
245
+ # Removes all rubyrep infrastructure tables from both databases.
246
+ def drop_infrastructure
247
+ drop_event_log if event_log_exists?
248
+ drop_change_logs
249
+ drop_activity_markers
250
+ end
251
+
252
+ # Checks for tables that have triggers but are not in the list of configured
253
+ # tables. Removes triggers and restores sequences of those tables.
254
+ # * +configured_table_pairs+:
255
+ # An array of table pairs (e. g. [{:left => 'xy', :right => 'xy2'}]).
256
+ def restore_unconfigured_tables(configured_table_pairs = session.configured_table_pairs)
257
+ [:left, :right].each do |database|
258
+ configured_tables = configured_table_pairs.map {|table_pair| table_pair[database]}
259
+ unconfigured_tables = session.send(database).tables - configured_tables
260
+ unconfigured_tables.each do |table|
261
+ if trigger_exists?(database, table)
262
+ drop_trigger(database, table)
263
+ session.send(database).execute(
264
+ "delete from #{options[:rep_prefix]}_pending_changes where change_table = '#{table}'")
265
+ end
266
+ clear_sequence_setup(database, table)
267
+ end
268
+ end
269
+ end
270
+
271
+ # Calls the potentially provided :+after_init+ handler after infrastructure
272
+ # tables are created.
273
+ def call_after_infrastructure_setup_handler
274
+ handler = session.configuration.options[:after_infrastructure_setup]
275
+ handler.call(session) if handler
276
+ end
277
+
278
+ # Prepares the database / tables for replication.
279
+ def prepare_replication
280
+ exclude_rubyrep_tables
281
+
282
+ puts "Verifying RubyRep tables"
283
+ ensure_infrastructure
284
+
285
+ call_after_infrastructure_setup_handler
286
+
287
+ puts "Checking for and removing rubyrep triggers from unconfigured tables"
288
+ restore_unconfigured_tables
289
+
290
+ puts "Verifying rubyrep triggers of configured tables"
291
+ unsynced_table_pairs = []
292
+ table_pairs = session.sort_table_pairs(session.configured_table_pairs)
293
+ table_pairs.each do |table_pair|
294
+ table_options = options(table_pair[:left])
295
+ ensure_sequence_setup table_pair,
296
+ table_options[:sequence_increment],
297
+ table_options[:left_sequence_offset],
298
+ table_options[:right_sequence_offset]
299
+
300
+ unsynced = false
301
+ [:left, :right].each do |database|
302
+ unless trigger_exists? database, table_pair[database]
303
+ create_trigger database, table_pair[database]
304
+ unsynced = true
305
+ end
306
+ end
307
+ if unsynced and table_options[:initial_sync]
308
+ unsynced_table_pairs << table_pair
309
+ end
310
+ end
311
+ unsynced_table_specs = unsynced_table_pairs.map do |table_pair|
312
+ "#{table_pair[:left]}, #{table_pair[:right]}"
313
+ end
314
+
315
+ unless unsynced_table_specs.empty?
316
+ puts "Executing initial table syncs"
317
+ runner = SyncRunner.new
318
+ runner.session = session
319
+ runner.options = {:table_specs => unsynced_table_specs}
320
+ runner.execute
321
+ end
322
+
323
+ puts "Starting replication"
324
+ end
325
+ end
326
+
327
+ end