andyjeffries-rubyrep 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. data/History.txt +83 -0
  2. data/License.txt +20 -0
  3. data/Manifest.txt +151 -0
  4. data/README.txt +37 -0
  5. data/bin/rubyrep +8 -0
  6. data/lib/rubyrep.rb +72 -0
  7. data/lib/rubyrep/base_runner.rb +195 -0
  8. data/lib/rubyrep/command_runner.rb +144 -0
  9. data/lib/rubyrep/committers/buffered_committer.rb +151 -0
  10. data/lib/rubyrep/committers/committers.rb +152 -0
  11. data/lib/rubyrep/configuration.rb +275 -0
  12. data/lib/rubyrep/connection_extenders/connection_extenders.rb +165 -0
  13. data/lib/rubyrep/connection_extenders/jdbc_extender.rb +65 -0
  14. data/lib/rubyrep/connection_extenders/mysql_extender.rb +59 -0
  15. data/lib/rubyrep/connection_extenders/postgresql_extender.rb +277 -0
  16. data/lib/rubyrep/database_proxy.rb +52 -0
  17. data/lib/rubyrep/direct_table_scan.rb +75 -0
  18. data/lib/rubyrep/generate_runner.rb +105 -0
  19. data/lib/rubyrep/initializer.rb +39 -0
  20. data/lib/rubyrep/log_helper.rb +30 -0
  21. data/lib/rubyrep/logged_change.rb +160 -0
  22. data/lib/rubyrep/logged_change_loader.rb +197 -0
  23. data/lib/rubyrep/noisy_connection.rb +80 -0
  24. data/lib/rubyrep/proxied_table_scan.rb +171 -0
  25. data/lib/rubyrep/proxy_block_cursor.rb +145 -0
  26. data/lib/rubyrep/proxy_connection.rb +431 -0
  27. data/lib/rubyrep/proxy_cursor.rb +44 -0
  28. data/lib/rubyrep/proxy_row_cursor.rb +43 -0
  29. data/lib/rubyrep/proxy_runner.rb +89 -0
  30. data/lib/rubyrep/replication_difference.rb +100 -0
  31. data/lib/rubyrep/replication_extenders/mysql_replication.rb +271 -0
  32. data/lib/rubyrep/replication_extenders/postgresql_replication.rb +236 -0
  33. data/lib/rubyrep/replication_extenders/replication_extenders.rb +26 -0
  34. data/lib/rubyrep/replication_helper.rb +142 -0
  35. data/lib/rubyrep/replication_initializer.rb +327 -0
  36. data/lib/rubyrep/replication_run.rb +142 -0
  37. data/lib/rubyrep/replication_runner.rb +166 -0
  38. data/lib/rubyrep/replicators/replicators.rb +42 -0
  39. data/lib/rubyrep/replicators/two_way_replicator.rb +361 -0
  40. data/lib/rubyrep/scan_progress_printers/progress_bar.rb +65 -0
  41. data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb +65 -0
  42. data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb +111 -0
  43. data/lib/rubyrep/scan_report_printers/scan_report_printers.rb +67 -0
  44. data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb +75 -0
  45. data/lib/rubyrep/scan_runner.rb +25 -0
  46. data/lib/rubyrep/session.rb +230 -0
  47. data/lib/rubyrep/sync_helper.rb +121 -0
  48. data/lib/rubyrep/sync_runner.rb +31 -0
  49. data/lib/rubyrep/syncers/syncers.rb +112 -0
  50. data/lib/rubyrep/syncers/two_way_syncer.rb +174 -0
  51. data/lib/rubyrep/table_scan.rb +54 -0
  52. data/lib/rubyrep/table_scan_helper.rb +46 -0
  53. data/lib/rubyrep/table_sorter.rb +70 -0
  54. data/lib/rubyrep/table_spec_resolver.rb +142 -0
  55. data/lib/rubyrep/table_sync.rb +90 -0
  56. data/lib/rubyrep/task_sweeper.rb +77 -0
  57. data/lib/rubyrep/trigger_mode_switcher.rb +63 -0
  58. data/lib/rubyrep/type_casting_cursor.rb +31 -0
  59. data/lib/rubyrep/uninstall_runner.rb +93 -0
  60. data/lib/rubyrep/version.rb +9 -0
  61. data/rubyrep +8 -0
  62. data/rubyrep.bat +4 -0
  63. data/setup.rb +1585 -0
  64. data/spec/base_runner_spec.rb +218 -0
  65. data/spec/buffered_committer_spec.rb +274 -0
  66. data/spec/command_runner_spec.rb +145 -0
  67. data/spec/committers_spec.rb +178 -0
  68. data/spec/configuration_spec.rb +203 -0
  69. data/spec/connection_extender_interface_spec.rb +141 -0
  70. data/spec/connection_extenders_registration_spec.rb +164 -0
  71. data/spec/database_proxy_spec.rb +48 -0
  72. data/spec/database_rake_spec.rb +40 -0
  73. data/spec/db_specific_connection_extenders_spec.rb +34 -0
  74. data/spec/db_specific_replication_extenders_spec.rb +38 -0
  75. data/spec/direct_table_scan_spec.rb +61 -0
  76. data/spec/dolphins.jpg +0 -0
  77. data/spec/generate_runner_spec.rb +84 -0
  78. data/spec/initializer_spec.rb +46 -0
  79. data/spec/log_helper_spec.rb +39 -0
  80. data/spec/logged_change_loader_spec.rb +68 -0
  81. data/spec/logged_change_spec.rb +470 -0
  82. data/spec/noisy_connection_spec.rb +78 -0
  83. data/spec/postgresql_replication_spec.rb +48 -0
  84. data/spec/postgresql_schema_support_spec.rb +212 -0
  85. data/spec/postgresql_support_spec.rb +63 -0
  86. data/spec/progress_bar_spec.rb +77 -0
  87. data/spec/proxied_table_scan_spec.rb +151 -0
  88. data/spec/proxy_block_cursor_spec.rb +197 -0
  89. data/spec/proxy_connection_spec.rb +423 -0
  90. data/spec/proxy_cursor_spec.rb +56 -0
  91. data/spec/proxy_row_cursor_spec.rb +66 -0
  92. data/spec/proxy_runner_spec.rb +70 -0
  93. data/spec/replication_difference_spec.rb +161 -0
  94. data/spec/replication_extender_interface_spec.rb +367 -0
  95. data/spec/replication_extenders_spec.rb +32 -0
  96. data/spec/replication_helper_spec.rb +178 -0
  97. data/spec/replication_initializer_spec.rb +509 -0
  98. data/spec/replication_run_spec.rb +443 -0
  99. data/spec/replication_runner_spec.rb +254 -0
  100. data/spec/replicators_spec.rb +36 -0
  101. data/spec/rubyrep_spec.rb +8 -0
  102. data/spec/scan_detail_reporter_spec.rb +119 -0
  103. data/spec/scan_progress_printers_spec.rb +68 -0
  104. data/spec/scan_report_printers_spec.rb +67 -0
  105. data/spec/scan_runner_spec.rb +50 -0
  106. data/spec/scan_summary_reporter_spec.rb +61 -0
  107. data/spec/session_spec.rb +253 -0
  108. data/spec/spec.opts +1 -0
  109. data/spec/spec_helper.rb +305 -0
  110. data/spec/strange_name_support_spec.rb +135 -0
  111. data/spec/sync_helper_spec.rb +169 -0
  112. data/spec/sync_runner_spec.rb +78 -0
  113. data/spec/syncers_spec.rb +171 -0
  114. data/spec/table_scan_helper_spec.rb +36 -0
  115. data/spec/table_scan_spec.rb +49 -0
  116. data/spec/table_sorter_spec.rb +30 -0
  117. data/spec/table_spec_resolver_spec.rb +111 -0
  118. data/spec/table_sync_spec.rb +140 -0
  119. data/spec/task_sweeper_spec.rb +47 -0
  120. data/spec/trigger_mode_switcher_spec.rb +83 -0
  121. data/spec/two_way_replicator_spec.rb +721 -0
  122. data/spec/two_way_syncer_spec.rb +256 -0
  123. data/spec/type_casting_cursor_spec.rb +50 -0
  124. data/spec/uninstall_runner_spec.rb +93 -0
  125. metadata +190 -0
@@ -0,0 +1,236 @@
1
+ module RR
2
+ module ReplicationExtenders
3
+
4
+ # Provides PostgreSQL specific functionality for database replication
5
+ module PostgreSQLReplication
6
+ RR::ReplicationExtenders.register :postgresql => self
7
+
8
+ # Returns the key clause that is used in the trigger function.
9
+ # * +trigger_var+: should be either 'NEW' or 'OLD'
10
+ # * +params+: the parameter hash as described in #create_rep_trigger
11
+ def key_clause(trigger_var, params)
12
+ params[:keys].
13
+ map { |key| "'#{key}#{params[:key_sep]}' || #{trigger_var}.#{quote_column_name(key)}"}.
14
+ join(" || '#{params[:key_sep]}' || ")
15
+ end
16
+ private :key_clause
17
+
18
+ # Returns the schema prefix (including dot) that will be used by the
19
+ # triggers to write into the rubyrep infrastructure tables.
20
+ # To avoid setting the wrong prefix, it will only return a schema prefix
21
+ # if the current search path
22
+ # * consists of only a single schema
23
+ # * does not consists of a variable search path
24
+ # (i. e. the default "$user")
25
+ def schema_prefix
26
+ unless @schema_prefix
27
+ search_path = select_one("show search_path")['search_path']
28
+ if search_path =~ /[,$]/
29
+ @schema_prefix = ""
30
+ else
31
+ @schema_prefix = %("#{search_path}".)
32
+ end
33
+ end
34
+ @schema_prefix
35
+ end
36
+
37
+ # Creates or replaces the replication trigger function.
38
+ # See #create_replication_trigger for a descriptions of the +params+ hash.
39
+ def create_or_replace_replication_trigger_function(params)
40
+ # first check, if PL/SQL is already activated and if not, do so.
41
+ if select_all("select lanname from pg_language where lanname = 'plpgsql'").empty?
42
+ execute "CREATE LANGUAGE plpgsql"
43
+ end
44
+
45
+ activity_check = ""
46
+ if params[:exclude_rr_activity] then
47
+ activity_check = <<-end_sql
48
+ PERFORM ACTIVE FROM #{schema_prefix}#{params[:activity_table]};
49
+ IF FOUND THEN
50
+ RETURN NULL;
51
+ END IF;
52
+ end_sql
53
+ end
54
+
55
+ version_string = select_value("select version();")
56
+ version = version_string.gsub(/^\s*postgresql\s*([0-9.]+).*$/i, '\1')
57
+ if version >= '8.4'
58
+ modification_check = <<-end_sql
59
+ IF NEW IS NOT DISTINCT FROM OLD THEN
60
+ RETURN NULL;
61
+ END IF;
62
+ end_sql
63
+ else
64
+ modification_check = ""
65
+ end
66
+
67
+ # now create the trigger
68
+ execute(<<-end_sql)
69
+ CREATE OR REPLACE FUNCTION "#{params[:trigger_name]}"() RETURNS TRIGGER AS $change_trigger$
70
+ BEGIN
71
+ #{activity_check}
72
+ IF (TG_OP = 'DELETE') THEN
73
+ INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
74
+ SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
75
+ ELSIF (TG_OP = 'UPDATE') THEN
76
+ #{modification_check}
77
+ INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
78
+ SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
79
+ ELSIF (TG_OP = 'INSERT') THEN
80
+ INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
81
+ SELECT '#{params[:table]}', #{key_clause('NEW', params)}, 'I', now();
82
+ END IF;
83
+ RETURN NULL; -- result is ignored since this is an AFTER trigger
84
+ END;
85
+ $change_trigger$ LANGUAGE plpgsql;
86
+ end_sql
87
+
88
+ end
89
+
90
+ # Creates a trigger to log all changes for the given table.
91
+ # +params+ is a hash with all necessary information:
92
+ # * :+trigger_name+: name of the trigger
93
+ # * :+table+: name of the table that should be monitored
94
+ # * :+keys+: array of names of the key columns of the monitored table
95
+ # * :+log_table+: name of the table receiving all change notifications
96
+ # * :+activity_table+: name of the table receiving the rubyrep activity information
97
+ # * :+key_sep+: column seperator to be used in the key column of the log table
98
+ # * :+exclude_rr_activity+:
99
+ # if true, the trigger will check and filter out changes initiated by RubyRep
100
+ def create_replication_trigger(params)
101
+ create_or_replace_replication_trigger_function params
102
+
103
+ execute(<<-end_sql)
104
+ CREATE TRIGGER "#{params[:trigger_name]}"
105
+ AFTER INSERT OR UPDATE OR DELETE ON "#{params[:table]}"
106
+ FOR EACH ROW EXECUTE PROCEDURE #{schema_prefix}"#{params[:trigger_name]}"();
107
+ end_sql
108
+ end
109
+
110
+ # Removes a trigger and related trigger procedure.
111
+ # * +trigger_name+: name of the trigger
112
+ # * +table_name+: name of the table for which the trigger exists
113
+ def drop_replication_trigger(trigger_name, table_name)
114
+ execute "DROP TRIGGER \"#{trigger_name}\" ON \"#{table_name}\";"
115
+ execute "DROP FUNCTION \"#{trigger_name}\"();"
116
+ end
117
+
118
+ # Returns +true+ if the named trigger exists for the named table.
119
+ # * +trigger_name+: name of the trigger
120
+ # * +table_name+: name of the table
121
+ def replication_trigger_exists?(trigger_name, table_name)
122
+ !select_all(<<-end_sql).empty?
123
+ select 1 from information_schema.triggers
124
+ where event_object_schema in (#{schemas})
125
+ and trigger_name = '#{trigger_name}'
126
+ and event_object_table = '#{table_name}'
127
+ end_sql
128
+ end
129
+
130
+ # Returns all unadjusted sequences of the given table.
131
+ # Parameters:
132
+ # * +rep_prefix+: not used (necessary) for the Postgres
133
+ # * +table_name+: name of the table
134
+ # Return value: a hash with
135
+ # * key: sequence name
136
+ # * value: a hash with
137
+ # * :+increment+: current sequence increment
138
+ # * :+value+: current value
139
+ def sequence_values(rep_prefix, table_name)
140
+ result = {}
141
+ sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
142
+ select s.relname
143
+ from pg_class as t
144
+ join pg_depend as r on t.oid = r.refobjid
145
+ join pg_class as s on r.objid = s.oid
146
+ and s.relkind = 'S'
147
+ and t.relname = '#{table_name}' AND t.relnamespace IN
148
+ (SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
149
+ end_sql
150
+ sequence_names.each do |sequence_name|
151
+ row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
152
+ result[sequence_name] = {
153
+ :increment => row['increment_by'].to_i,
154
+ :value => row['last_value'].to_i
155
+ }
156
+ end
157
+ result
158
+ end
159
+
160
+ # Ensures that the sequences of the named table (normally the primary key
161
+ # column) are generated with the correct increment and offset.
162
+ # * +rep_prefix+: not used (necessary) for the Postgres
163
+ # * +table_name+: name of the table (not used for Postgres)
164
+ # * +increment+: increment of the sequence
165
+ # * +offset+: offset
166
+ # * +left_sequence_values+:
167
+ # hash as returned by #sequence_values for the left database
168
+ # * +right_sequence_values+:
169
+ # hash as returned by #sequence_values for the right database
170
+ # * +adjustment_buffer+:
171
+ # the "gap" that is created during sequence update to avoid concurrency problems
172
+ # E. g. an increment of 2 and offset of 1 will lead to generation of odd
173
+ # numbers.
174
+ def update_sequences(
175
+ rep_prefix, table_name, increment, offset,
176
+ left_sequence_values, right_sequence_values, adjustment_buffer)
177
+ left_sequence_values.each do |sequence_name, left_current_value|
178
+ row = select_one("select last_value, increment_by from \"#{sequence_name}\"")
179
+ current_increment = row['increment_by'].to_i
180
+ current_value = row['last_value'].to_i
181
+ unless current_increment == increment and current_value % increment == offset
182
+ max_current_value =
183
+ [left_current_value[:value], right_sequence_values[sequence_name][:value]].max +
184
+ adjustment_buffer
185
+ new_start = max_current_value - (max_current_value % increment) + increment + offset
186
+ execute(<<-end_sql)
187
+ alter sequence "#{sequence_name}" increment by #{increment} restart with #{new_start}
188
+ end_sql
189
+ end
190
+ end
191
+ end
192
+
193
+ # Restores the original sequence settings.
194
+ # (Actually it sets the sequence increment to 1. If before, it had a
195
+ # different value, then the restoration will not be correct.)
196
+ # * +rep_prefix+: not used (necessary) for the Postgres
197
+ # * +table_name+: name of the table
198
+ def clear_sequence_setup(rep_prefix, table_name)
199
+ sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
200
+ select s.relname
201
+ from pg_class as t
202
+ join pg_depend as r on t.oid = r.refobjid
203
+ join pg_class as s on r.objid = s.oid
204
+ and s.relkind = 'S'
205
+ and t.relname = '#{table_name}' and t.relnamespace IN
206
+ (SELECT oid FROM pg_namespace WHERE nspname in (#{schemas}))
207
+ end_sql
208
+ sequence_names.each do |sequence_name|
209
+ execute(<<-end_sql)
210
+ alter sequence "#{sequence_name}" increment by 1
211
+ end_sql
212
+ end
213
+ end
214
+
215
+ # Adds a big (8 byte value), auto-incrementing primary key column to the
216
+ # specified table.
217
+ # * table_name: name of the target table
218
+ # * key_name: name of the primary key column
219
+ def add_big_primary_key(table_name, key_name)
220
+ old_message_level = select_one("show client_min_messages")['client_min_messages']
221
+ execute "set client_min_messages = warning"
222
+ execute(<<-end_sql)
223
+ alter table "#{table_name}" add column #{key_name} bigserial
224
+ end_sql
225
+
226
+ execute(<<-end_sql)
227
+ alter table "#{table_name}" add constraint #{table_name}_#{key_name}_pkey primary key (#{key_name})
228
+ end_sql
229
+
230
+ ensure
231
+ execute "set client_min_messages = #{old_message_level}"
232
+ end
233
+ end
234
+ end
235
+ end
236
+
@@ -0,0 +1,26 @@
1
+ module RR
2
+
3
+ # Replication extenders are modules that provide database specific functionality
4
+ # required for replication. They are mixed into ActiveRecord database connections.
5
+ # This module itself only provides functionality to register and retrieve
6
+ # such extenders.
7
+ module ReplicationExtenders
8
+ # Returns a Hash of currently registered replication extenders.
9
+ # (Empty Hash if no replication extenders were defined.)
10
+ def self.extenders
11
+ @extenders ||= {}
12
+ @extenders
13
+ end
14
+
15
+ # Registers one or multiple replication extender.
16
+ # extender is a Hash with
17
+ # key:: The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
18
+ # value:: Name of the module implementing the replication extender
19
+ def self.register(extender)
20
+ @extenders ||= {}
21
+ @extenders.merge! extender
22
+ end
23
+ end
24
+ end
25
+
26
+
@@ -0,0 +1,142 @@
1
+ module RR
2
+
3
+ # Provides helper functionality for replicators.
4
+ # The methods exposed by this class are intended to provide a stable interface
5
+ # for third party replicators.
6
+ class ReplicationHelper
7
+
8
+ include LogHelper
9
+
10
+ # The current +ReplicationRun+ instance
11
+ attr_accessor :replication_run
12
+
13
+ # The active +Session+
14
+ def session; replication_run.session; end
15
+
16
+ # Current options
17
+ def options; @options ||= session.configuration.options; end
18
+
19
+ # Returns the options for the specified table name.
20
+ # * +table+: name of the table (left database version)
21
+ def options_for_table(table)
22
+ @options_for_table ||= {}
23
+ unless @options_for_table.include? table
24
+ @options_for_table[table] = session.configuration.options_for_table(table)
25
+ end
26
+ @options_for_table[table]
27
+ end
28
+
29
+ # Delegates to Session#corresponding_table
30
+ def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
31
+
32
+ # Returns +true+ if a new transaction was started since the last
33
+ # insert / update / delete.
34
+ def new_transaction?
35
+ committer.new_transaction?
36
+ end
37
+
38
+ # Delegates to Committers::BufferedCommitter#insert_record
39
+ def insert_record(database, table, values)
40
+ committer.insert_record(database, table, values)
41
+ end
42
+
43
+ # Delegates to Committers::BufferedCommitter#update_record
44
+ def update_record(database, table, values, old_key = nil)
45
+ committer.update_record(database, table, values, old_key)
46
+ end
47
+
48
+ # Delegates to Committers::BufferedCommitter#delete_record
49
+ def delete_record(database, table, values)
50
+ committer.delete_record(database, table, values)
51
+ end
52
+
53
+ # Loads the specified record. Returns an according column_name => value hash.
54
+ # Parameters:
55
+ # * +database+: either :+left+ or :+right+
56
+ # * +table+: name of the table
57
+ # * +key+: A column_name => value hash for all primary key columns.
58
+ def load_record(database, table, key)
59
+ cursor = session.send(database).select_cursor(
60
+ :table => table,
61
+ :row_keys => [key],
62
+ :type_cast => true
63
+ )
64
+ row = nil
65
+ row = cursor.next_row if cursor.next?
66
+ cursor.clear
67
+ row
68
+ end
69
+
70
+ # The current Committer
71
+ attr_reader :committer
72
+ private :committer
73
+
74
+ # Asks the committer (if it exists) to finalize any open transactions
75
+ # +success+ should be true if there were no problems, false otherwise.
76
+ def finalize(success = true)
77
+ committer.finalize(success)
78
+ end
79
+
80
+ # Converts the row values into their proper types as per table definition.
81
+ # * +table+: name of the table after whose columns is type-casted.
82
+ # * +row+: A column_name => value hash of the row
83
+ # Returns a copy of the column_name => value hash (with type-casted values).
84
+ def type_cast(table, row)
85
+ @table_columns ||= {}
86
+ unless @table_columns.include?(table)
87
+ column_array = session.left.columns(table)
88
+ column_hash = {}
89
+ column_array.each {|column| column_hash[column.name] = column}
90
+ @table_columns[table] = column_hash
91
+ end
92
+ columns = @table_columns[table]
93
+ type_casted_row = {}
94
+ row.each_pair do |column_name, value|
95
+ type_casted_row[column_name] = columns[column_name].type_cast(value)
96
+ end
97
+ type_casted_row
98
+ end
99
+
100
+ # Logs the outcome of a replication into the replication log table.
101
+ # * +diff+: the replicated ReplicationDifference
102
+ # * +outcome+: string summarizing the outcome of the replication
103
+ # * +details+: string with further details regarding the replication
104
+ def log_replication_outcome(diff, outcome, details = nil)
105
+ table = diff.changes[:left].table
106
+ key = diff.changes[:left].key
107
+ if key.size == 1
108
+ key = key.values[0]
109
+ else
110
+ key_parts = session.left.primary_key_names(table).map do |column_name|
111
+ %Q("#{column_name}"=>#{key[column_name].to_s.inspect})
112
+ end
113
+ key = key_parts.join(', ')
114
+ end
115
+ rep_outcome, rep_details = fit_description_columns(outcome, details)
116
+ diff_dump = diff.to_yaml[0...ReplicationInitializer::DIFF_DUMP_SIZE]
117
+
118
+ session.left.insert_record "#{options[:rep_prefix]}_logged_events", {
119
+ :activity => 'replication',
120
+ :change_table => table,
121
+ :diff_type => diff.type.to_s,
122
+ :change_key => key,
123
+ :left_change_type => (diff.changes[:left] ? diff.changes[:left].type.to_s : nil),
124
+ :right_change_type => (diff.changes[:right] ? diff.changes[:right].type.to_s : nil),
125
+ :description => rep_outcome,
126
+ :long_description => rep_details,
127
+ :event_time => Time.now,
128
+ :diff_dump => diff_dump
129
+ }
130
+ end
131
+
132
+ # Creates a new SyncHelper for the given +TableSync+ instance.
133
+ def initialize(replication_run)
134
+ self.replication_run = replication_run
135
+
136
+ # Creates the committer. Important as it gives the committer the
137
+ # opportunity to start transactions
138
+ committer_class = Committers::committers[options[:committer]]
139
+ @committer = committer_class.new(session)
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,327 @@
1
+ $LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
2
+
3
+ require 'rubyrep'
4
+
5
+ module RR
6
+
7
+ # Ensures all preconditions are met to start with replication
8
+ class ReplicationInitializer
9
+
10
+ # The active Session
11
+ attr_accessor :session
12
+
13
+ # Creates a new RepInititializer for the given Session
14
+ def initialize(session)
15
+ self.session = session
16
+ end
17
+
18
+ # Returns the options for the given table.
19
+ # If table is +nil+, returns general options.
20
+ def options(table = nil)
21
+ if table
22
+ session.configuration.options_for_table table
23
+ else
24
+ session.configuration.options
25
+ end
26
+ end
27
+
28
+ # Creates a trigger logging all table changes
29
+ # * database: either :+left+ or :+right+
30
+ # * table: name of the table
31
+ def create_trigger(database, table)
32
+ options = self.options(table)
33
+
34
+ params = {
35
+ :trigger_name => "#{options[:rep_prefix]}_#{table}",
36
+ :table => table,
37
+ :keys => session.send(database).primary_key_names(table),
38
+ :log_table => "#{options[:rep_prefix]}_pending_changes",
39
+ :activity_table => "#{options[:rep_prefix]}_running_flags",
40
+ :key_sep => options[:key_sep],
41
+ :exclude_rr_activity => false,
42
+ }
43
+
44
+ session.send(database).create_replication_trigger params
45
+ end
46
+
47
+ # Returns +true+ if the replication trigger for the given table exists.
48
+ # * database: either :+left+ or :+right+
49
+ # * table: name of the table
50
+ def trigger_exists?(database, table)
51
+ trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
52
+ session.send(database).replication_trigger_exists? trigger_name, table
53
+ end
54
+
55
+ # Drops the replication trigger of the named table.
56
+ # * database: either :+left+ or :+right+
57
+ # * table: name of the table
58
+ def drop_trigger(database, table)
59
+ trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
60
+ session.send(database).drop_replication_trigger trigger_name, table
61
+ end
62
+
63
+ # Ensures that the sequences of the named table (normally the primary key
64
+ # column) are generated with the correct increment and offset in both
65
+ # left and right database.
66
+ # The sequence is always updated in both databases.
67
+ # * +table_pair+: a hash of names of corresponding :left and :right tables
68
+ # * +increment+: increment of the sequence
69
+ # * +left_offset+: offset of table in left database
70
+ # * +right_offset+: offset of table in right database
71
+ # E. g. an increment of 2 and offset of 1 will lead to generation of odd
72
+ # numbers.
73
+ def ensure_sequence_setup(table_pair, increment, left_offset, right_offset)
74
+ table_options = options(table_pair[:left])
75
+ if table_options[:adjust_sequences]
76
+ rep_prefix = table_options[:rep_prefix]
77
+ left_sequence_values = session.left.sequence_values rep_prefix, table_pair[:left]
78
+ right_sequence_values = session.right.sequence_values rep_prefix, table_pair[:right]
79
+ [:left, :right].each do |database|
80
+ offset = database == :left ? left_offset : right_offset
81
+ session.send(database).update_sequences \
82
+ rep_prefix, table_pair[database], increment, offset,
83
+ left_sequence_values, right_sequence_values, table_options[:sequence_adjustment_buffer]
84
+ end
85
+ end
86
+ end
87
+
88
+ # Restores the original sequence settings for the named table.
89
+ # (Actually it sets the sequence increment to 1. If before, it had a
90
+ # different value, then the restoration will not be correct.)
91
+ # * database: either :+left+ or :+right+
92
+ # * +table_name+: name of the table
93
+ def clear_sequence_setup(database, table)
94
+ table_options = options(table)
95
+ if table_options[:adjust_sequences]
96
+ session.send(database).clear_sequence_setup(
97
+ table_options[:rep_prefix], table
98
+ )
99
+ end
100
+ end
101
+
102
+ # Returns +true+ if the change log exists in the specified database.
103
+ # * database: either :+left+ or :+right+
104
+ def change_log_exists?(database)
105
+ session.send(database).tables.include? "#{options[:rep_prefix]}_pending_changes"
106
+ end
107
+
108
+ # Returns +true+ if the replication log exists.
109
+ def event_log_exists?
110
+ session.left.tables.include? "#{options[:rep_prefix]}_logged_events"
111
+ end
112
+
113
+ # Drops the change log table in the specified database
114
+ # * database: either :+left+ or :+right+
115
+ def drop_change_log(database)
116
+ session.send(database).drop_table "#{options[:rep_prefix]}_pending_changes"
117
+ end
118
+
119
+ # Drops the replication log table.
120
+ def drop_event_log
121
+ session.left.drop_table "#{options[:rep_prefix]}_logged_events"
122
+ end
123
+
124
+ # Size of the replication log column diff_dump
125
+ DIFF_DUMP_SIZE = 2000
126
+
127
+ # Size fo the event log column 'description'
128
+ DESCRIPTION_SIZE = 255
129
+
130
+ # Size of the event log column 'long_description'
131
+ LONG_DESCRIPTION_SIZE = 1000
132
+
133
+ # Ensures that create_table and related statements don't print notices to
134
+ # stdout. Then restored original message setting.
135
+ # * +database+: either :+left+ or :+right+
136
+ def silence_ddl_notices(database)
137
+ if session.configuration.send(database)[:adapter] =~ /postgres/
138
+ old_message_level = session.send(database).
139
+ select_one("show client_min_messages")['client_min_messages']
140
+ session.send(database).execute "set client_min_messages = warning"
141
+ end
142
+ yield
143
+ ensure
144
+ if session.configuration.send(database)[:adapter] =~ /postgres/
145
+ session.send(database).execute "set client_min_messages = #{old_message_level}"
146
+ end
147
+ end
148
+
149
+ # Creates the replication log table.
150
+ def create_event_log
151
+ silence_ddl_notices(:left) do
152
+ table_name = "#{options[:rep_prefix]}_logged_events"
153
+ session.left.create_table "#{options[:rep_prefix]}_logged_events"
154
+ session.left.add_column table_name, :activity, :string
155
+ session.left.add_column table_name, :change_table, :string
156
+ session.left.add_column table_name, :diff_type, :string
157
+ session.left.add_column table_name, :change_key, :string
158
+ session.left.add_column table_name, :left_change_type, :string
159
+ session.left.add_column table_name, :right_change_type, :string
160
+ session.left.add_column table_name, :description, :string, :limit => DESCRIPTION_SIZE
161
+ session.left.add_column table_name, :long_description, :string, :limit => LONG_DESCRIPTION_SIZE
162
+ session.left.add_column table_name, :event_time, :timestamp
163
+ session.left.add_column table_name, :diff_dump, :string, :limit => DIFF_DUMP_SIZE
164
+ session.left.remove_column table_name, 'id'
165
+ session.left.add_big_primary_key table_name, 'id'
166
+ end
167
+ end
168
+
169
+ # Creates the change log table in the specified database
170
+ # * database: either :+left+ or :+right+
171
+ def create_change_log(database)
172
+ silence_ddl_notices(database) do
173
+ connection = session.send(database)
174
+ table_name = "#{options[:rep_prefix]}_pending_changes"
175
+ connection.create_table table_name
176
+ connection.add_column table_name, :change_table, :string
177
+ connection.add_column table_name, :change_key, :string
178
+ connection.add_column table_name, :change_new_key, :string
179
+ connection.add_column table_name, :change_type, :string
180
+ connection.add_column table_name, :change_time, :timestamp
181
+ connection.remove_column table_name, 'id'
182
+ connection.add_big_primary_key table_name, 'id'
183
+ end
184
+ end
185
+
186
+ # Adds to the current session's configuration an exclusion of rubyrep tables.
187
+ def exclude_rubyrep_tables
188
+ session.configuration.exclude_rubyrep_tables
189
+ end
190
+
191
+ # Checks in both databases, if the activity marker tables exist and if not,
192
+ # creates them.
193
+ def ensure_activity_markers
194
+ table_name = "#{options[:rep_prefix]}_running_flags"
195
+ [:left, :right].each do |database|
196
+ connection = session.send(database)
197
+ unless connection.tables.include? table_name
198
+ silence_ddl_notices(database) do
199
+ connection.create_table table_name
200
+ connection.add_column table_name, :active, :integer
201
+ connection.remove_column table_name, 'id'
202
+ end
203
+ end
204
+ end
205
+ end
206
+
207
+ # Checks if the event log table already exists and creates it if necessary
208
+ def ensure_event_log
209
+ create_event_log unless event_log_exists?
210
+ end
211
+
212
+ # Checks in both databases, if the change log tables exists and creates them
213
+ # if necessary
214
+ def ensure_change_logs
215
+ [:left, :right].each do |database|
216
+ create_change_log(database) unless change_log_exists?(database)
217
+ end
218
+ end
219
+
220
+ # Checks in both databases, if the infrastructure tables (change log, event
221
+ # log) exist and creates them if necessary.
222
+ def ensure_infrastructure
223
+ ensure_activity_markers
224
+ ensure_change_logs
225
+ ensure_event_log
226
+ end
227
+
228
+ # Checks in both databases, if the change_log tables exist. If yes, drops them.
229
+ def drop_change_logs
230
+ [:left, :right].each do |database|
231
+ drop_change_log(database) if change_log_exists?(database)
232
+ end
233
+ end
234
+
235
+ # Checks in both databases, if the activity_marker tables exist. If yes, drops them.
236
+ def drop_activity_markers
237
+ table_name = "#{options[:rep_prefix]}_running_flags"
238
+ [:left, :right].each do |database|
239
+ if session.send(database).tables.include? table_name
240
+ session.send(database).drop_table table_name
241
+ end
242
+ end
243
+ end
244
+
245
+ # Removes all rubyrep infrastructure tables from both databases.
246
+ def drop_infrastructure
247
+ drop_event_log if event_log_exists?
248
+ drop_change_logs
249
+ drop_activity_markers
250
+ end
251
+
252
+ # Checks for tables that have triggers but are not in the list of configured
253
+ # tables. Removes triggers and restores sequences of those tables.
254
+ # * +configured_table_pairs+:
255
+ # An array of table pairs (e. g. [{:left => 'xy', :right => 'xy2'}]).
256
+ def restore_unconfigured_tables(configured_table_pairs = session.configured_table_pairs)
257
+ [:left, :right].each do |database|
258
+ configured_tables = configured_table_pairs.map {|table_pair| table_pair[database]}
259
+ unconfigured_tables = session.send(database).tables - configured_tables
260
+ unconfigured_tables.each do |table|
261
+ if trigger_exists?(database, table)
262
+ drop_trigger(database, table)
263
+ session.send(database).execute(
264
+ "delete from #{options[:rep_prefix]}_pending_changes where change_table = '#{table}'")
265
+ end
266
+ clear_sequence_setup(database, table)
267
+ end
268
+ end
269
+ end
270
+
271
+ # Calls the potentially provided :+after_init+ handler after infrastructure
272
+ # tables are created.
273
+ def call_after_infrastructure_setup_handler
274
+ handler = session.configuration.options[:after_infrastructure_setup]
275
+ handler.call(session) if handler
276
+ end
277
+
278
+ # Prepares the database / tables for replication.
279
+ def prepare_replication
280
+ exclude_rubyrep_tables
281
+
282
+ puts "Verifying RubyRep tables"
283
+ ensure_infrastructure
284
+
285
+ call_after_infrastructure_setup_handler
286
+
287
+ puts "Checking for and removing rubyrep triggers from unconfigured tables"
288
+ restore_unconfigured_tables
289
+
290
+ puts "Verifying rubyrep triggers of configured tables"
291
+ unsynced_table_pairs = []
292
+ table_pairs = session.sort_table_pairs(session.configured_table_pairs)
293
+ table_pairs.each do |table_pair|
294
+ table_options = options(table_pair[:left])
295
+ ensure_sequence_setup table_pair,
296
+ table_options[:sequence_increment],
297
+ table_options[:left_sequence_offset],
298
+ table_options[:right_sequence_offset]
299
+
300
+ unsynced = false
301
+ [:left, :right].each do |database|
302
+ unless trigger_exists? database, table_pair[database]
303
+ create_trigger database, table_pair[database]
304
+ unsynced = true
305
+ end
306
+ end
307
+ if unsynced and table_options[:initial_sync]
308
+ unsynced_table_pairs << table_pair
309
+ end
310
+ end
311
+ unsynced_table_specs = unsynced_table_pairs.map do |table_pair|
312
+ "#{table_pair[:left]}, #{table_pair[:right]}"
313
+ end
314
+
315
+ unless unsynced_table_specs.empty?
316
+ puts "Executing initial table syncs"
317
+ runner = SyncRunner.new
318
+ runner.session = session
319
+ runner.options = {:table_specs => unsynced_table_specs}
320
+ runner.execute
321
+ end
322
+
323
+ puts "Starting replication"
324
+ end
325
+ end
326
+
327
+ end