rubyrep 1.0.1 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +10 -0
- data/README.txt +1 -1
- data/lib/rubyrep/connection_extenders/connection_extenders.rb +2 -3
- data/lib/rubyrep/connection_extenders/jdbc_extender.rb +8 -150
- data/lib/rubyrep/connection_extenders/mysql_extender.rb +0 -109
- data/lib/rubyrep/connection_extenders/postgresql_extender.rb +0 -102
- data/lib/rubyrep/logged_change.rb +14 -1
- data/lib/rubyrep/proxy_connection.rb +82 -3
- data/lib/rubyrep/replication_run.rb +22 -15
- data/lib/rubyrep/table_spec_resolver.rb +14 -8
- data/lib/rubyrep/version.rb +1 -1
- data/spec/connection_extender_interface_spec.rb +14 -6
- data/spec/connection_extenders_registration_spec.rb +2 -2
- data/spec/proxy_connection_spec.rb +10 -18
- data/spec/replication_run_spec.rb +23 -2
- data/spec/table_spec_resolver_spec.rb +10 -1
- data/tasks/database.rake +10 -8
- data/tasks/java.rake +2 -1
- data.tar.gz.sig +0 -0
- metadata +2 -2
- metadata.gz.sig +0 -0
data/History.txt
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
== 1.0.2 2009-06-13
|
2
|
+
|
3
|
+
* 3 minor enhancements:
|
4
|
+
* Simplified low-level functions to read records from the database
|
5
|
+
(easier integration of additional databases)
|
6
|
+
* Better replication efficiency with large replication backlogs
|
7
|
+
* Reduced costs of an empty replication run
|
8
|
+
* Ignore tables that only exist in one database
|
9
|
+
(tables created for testing / backup don't block scan / sync / replication)
|
10
|
+
|
1
11
|
== 1.0.1 2009-03-25
|
2
12
|
|
3
13
|
* 3 minor enhancements:
|
data/README.txt
CHANGED
@@ -71,9 +71,8 @@ module RR
|
|
71
71
|
else
|
72
72
|
raise "No ConnectionExtender available for :#{config[:adapter]}"
|
73
73
|
end
|
74
|
-
|
75
|
-
|
76
|
-
|
74
|
+
connection.extend ConnectionExtenders.extenders[extender]
|
75
|
+
|
77
76
|
# Hack to get Postgres schema support under JRuby to par with the standard
|
78
77
|
# ruby version
|
79
78
|
if RUBY_PLATFORM =~ /java/ and config[:adapter].to_sym == :postgresql
|
@@ -7,106 +7,6 @@ module RR
|
|
7
7
|
module JdbcSQLExtender
|
8
8
|
RR::ConnectionExtenders.register :jdbc => self
|
9
9
|
|
10
|
-
# A cursor to iterate over the records returned by select_cursor.
|
11
|
-
# Only one row is kept in memory at a time.
|
12
|
-
module JdbcResultSet
|
13
|
-
# Returns true if there are more rows to read.
|
14
|
-
def next?
|
15
|
-
if @next_status == nil
|
16
|
-
@next_status = self.next
|
17
|
-
end
|
18
|
-
@next_status
|
19
|
-
end
|
20
|
-
|
21
|
-
# Returns the row as a column => value hash and moves the cursor to the next row.
|
22
|
-
def next_row
|
23
|
-
raise("no more rows available") unless next?
|
24
|
-
@next_status = nil
|
25
|
-
|
26
|
-
unless @columns
|
27
|
-
meta_data = self.getMetaData
|
28
|
-
stores_upper = self.getStatement.getConnection.getMetaData.storesUpperCaseIdentifiers
|
29
|
-
column_count = meta_data.getColumnCount
|
30
|
-
@columns = Array.new(column_count)
|
31
|
-
@columns.each_index do |i|
|
32
|
-
column_name = meta_data.getColumnName(i+1)
|
33
|
-
if stores_upper and not column_name =~ /[a-z]/
|
34
|
-
column_name.downcase!
|
35
|
-
end
|
36
|
-
@columns[i] = {
|
37
|
-
:index => i+1,
|
38
|
-
:name => column_name,
|
39
|
-
:type => meta_data.getColumnType(i+1)
|
40
|
-
#:scale => meta_data.getScale(i+1)
|
41
|
-
}
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
row = {}
|
46
|
-
@columns.each_index do |i|
|
47
|
-
row[@columns[i][:name]] = jdbc_to_ruby(@columns[i])
|
48
|
-
end
|
49
|
-
|
50
|
-
row
|
51
|
-
end
|
52
|
-
|
53
|
-
# Releases the databases resources hold by this cursor
|
54
|
-
def clear
|
55
|
-
@columns = nil
|
56
|
-
self.close
|
57
|
-
end
|
58
|
-
|
59
|
-
Types = java.sql.Types unless const_defined?(:Types)
|
60
|
-
|
61
|
-
# Converts the specified column of the current row to the proper ruby string
|
62
|
-
# column is a hash with the following elements:
|
63
|
-
# * :index: field number (starting with 1) of the result set field
|
64
|
-
# * :type: the java.sql.Type constant specifying the type of the result set field
|
65
|
-
def jdbc_to_ruby(column)
|
66
|
-
case column[:type]
|
67
|
-
when Types::BINARY, Types::BLOB, Types::LONGVARBINARY, Types::VARBINARY
|
68
|
-
is = self.getBinaryStream(column[:index])
|
69
|
-
if is == nil or self.wasNull
|
70
|
-
return nil
|
71
|
-
end
|
72
|
-
byte_list = org.jruby.util.ByteList.new(2048)
|
73
|
-
buffer = Java::byte[2048].new
|
74
|
-
while (n = is.read(buffer)) != -1
|
75
|
-
byte_list.append(buffer, 0, n)
|
76
|
-
end
|
77
|
-
is.close
|
78
|
-
return byte_list.toString
|
79
|
-
when Types::LONGVARCHAR, Types::CLOB
|
80
|
-
rss = self.getCharacterStream(column[:index])
|
81
|
-
if rss == nil or self.wasNull
|
82
|
-
return nil
|
83
|
-
end
|
84
|
-
str = java.lang.StringBuffer.new(2048)
|
85
|
-
cuf = Java::char[2048].new
|
86
|
-
while (n = rss.read(cuf)) != -1
|
87
|
-
str.append(cuf, 0, n)
|
88
|
-
end
|
89
|
-
rss.close
|
90
|
-
return str.toString
|
91
|
-
when Types::TIMESTAMP
|
92
|
-
time = self.getTimestamp(column[:index]);
|
93
|
-
if time == nil or self.wasNull
|
94
|
-
return nil
|
95
|
-
end
|
96
|
-
time_string = time.toString()
|
97
|
-
time_string = time_string.gsub(/ 00:00:00.0$/, '')
|
98
|
-
return time_string
|
99
|
-
else
|
100
|
-
value = self.getString(column[:index])
|
101
|
-
if value == nil or self.wasNull
|
102
|
-
return nil
|
103
|
-
end
|
104
|
-
return value
|
105
|
-
end
|
106
|
-
end
|
107
|
-
private :jdbc_to_ruby
|
108
|
-
end
|
109
|
-
|
110
10
|
# Monkey patch for activerecord-jdbc-adapter-0.7.2 as it doesn't set the
|
111
11
|
# +@active+ flag to false, thus ActiveRecord#active? incorrectly confirms
|
112
12
|
# the connection to still be active.
|
@@ -115,20 +15,6 @@ module RR
|
|
115
15
|
@active = false
|
116
16
|
end
|
117
17
|
|
118
|
-
# Executes the given sql query with the otional name written in the
|
119
|
-
# ActiveRecord log file.
|
120
|
-
# * +row_buffer_size+: not used.
|
121
|
-
# Returns the results as a Cursor object supporting
|
122
|
-
# * next? - returns true if there are more rows to read
|
123
|
-
# * next_row - returns the row as a column => value hash and moves the cursor to the next row
|
124
|
-
# * clear - clearing the cursor (making allocated memory available for GC)
|
125
|
-
def select_cursor(sql, row_buffer_size = 1000)
|
126
|
-
statement = @connection.connection.createStatement
|
127
|
-
statement.setFetchSize row_buffer_size
|
128
|
-
result_set = statement.executeQuery(sql)
|
129
|
-
result_set.send :extend, JdbcResultSet
|
130
|
-
end
|
131
|
-
|
132
18
|
# Returns an ordered list of primary key column names of the given table
|
133
19
|
def primary_key_names(table)
|
134
20
|
if not tables.include? table
|
@@ -169,46 +55,12 @@ module RR
|
|
169
55
|
end
|
170
56
|
end
|
171
57
|
|
172
|
-
require 'connection_extenders/postgresql_extender'
|
173
|
-
|
174
|
-
# Adds the correct query executioner functionality to the base class
|
175
|
-
class JdbcPostgreSQLFetcher < PostgreSQLFetcher
|
176
|
-
# Executes the given statements and returns the result set.
|
177
|
-
def execute(sql)
|
178
|
-
statement = connection.instance_variable_get(:@connection).connection.createStatement
|
179
|
-
execute_method = sql =~ /close/i ? :execute : :executeQuery
|
180
|
-
result_set = statement.send(execute_method, sql)
|
181
|
-
result_set.send :extend, RR::ConnectionExtenders::JdbcSQLExtender::JdbcResultSet
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
58
|
# PostgreSQL specific functionality not provided by the standard JDBC
|
186
59
|
# connection extender:
|
187
|
-
# * Integration of memory efficient select_cursor.
|
188
60
|
# * Hack to get schema support for Postgres under JRuby on par with the
|
189
61
|
# standard ruby version.
|
190
62
|
module JdbcPostgreSQLExtender
|
191
63
|
|
192
|
-
# Executes the given sql query with the otional name written in the
|
193
|
-
# ActiveRecord log file.
|
194
|
-
#
|
195
|
-
# :+row_buffer_size+ controls how many records are ready into memory at a
|
196
|
-
# time. Implemented using the PostgeSQL "DECLARE CURSOR" and "FETCH" constructs.
|
197
|
-
# This is necessary as the postgresql driver always reads the
|
198
|
-
# complete resultset into memory.
|
199
|
-
#
|
200
|
-
# Returns the results as a Cursor object supporting
|
201
|
-
# * next? - returns true if there are more rows to read
|
202
|
-
# * next_row - returns the row as a column => value hash and moves the cursor to the next row
|
203
|
-
# * clear - clearing the cursor (making allocated memory available for GC)
|
204
|
-
def select_cursor(sql, row_buffer_size = 1000)
|
205
|
-
cursor_name = "RR_#{Time.now.to_i}#{rand(1_000_000)}"
|
206
|
-
|
207
|
-
statement = @connection.connection.createStatement
|
208
|
-
statement.execute("DECLARE #{cursor_name} NO SCROLL CURSOR WITH HOLD FOR " + sql)
|
209
|
-
JdbcPostgreSQLFetcher.new(self, cursor_name, row_buffer_size)
|
210
|
-
end
|
211
|
-
|
212
64
|
# Returns the list of a table's column names, data types, and default values.
|
213
65
|
#
|
214
66
|
# The underlying query is roughly:
|
@@ -242,11 +94,16 @@ module RR
|
|
242
94
|
end
|
243
95
|
end
|
244
96
|
|
97
|
+
require 'jdbc_adapter/jdbc_postgre'
|
98
|
+
class JdbcPostgreSQLColumn < ActiveRecord::ConnectionAdapters::Column
|
99
|
+
include ::JdbcSpec::PostgreSQL::Column
|
100
|
+
end
|
101
|
+
|
245
102
|
# Returns the list of all column definitions for a table.
|
246
103
|
def columns(table_name, name = nil)
|
247
104
|
# Limit, precision, and scale are all handled by the superclass.
|
248
105
|
column_definitions(table_name).collect do |name, type, default, notnull|
|
249
|
-
|
106
|
+
JdbcPostgreSQLColumn.new(name, default, type, notnull == 'f')
|
250
107
|
end
|
251
108
|
end
|
252
109
|
|
@@ -281,4 +138,5 @@ module RR
|
|
281
138
|
end
|
282
139
|
end
|
283
140
|
end
|
284
|
-
end
|
141
|
+
end
|
142
|
+
|
@@ -1,120 +1,11 @@
|
|
1
|
-
# A cursor to iterate over the records returned by select_cursor.
|
2
|
-
# Only one row is kept in memory at a time.
|
3
|
-
|
4
|
-
module MysqlResultExtender
|
5
|
-
# Returns true if there are more rows to read.
|
6
|
-
def next?
|
7
|
-
@current_row_num ||= 0
|
8
|
-
@num_rows ||= self.num_rows()
|
9
|
-
@current_row_num < @num_rows
|
10
|
-
end
|
11
|
-
|
12
|
-
# Returns the row as a column => value hash and moves the cursor to the next row.
|
13
|
-
def next_row
|
14
|
-
raise("no more rows available") unless next?
|
15
|
-
row = fetch_hash()
|
16
|
-
@current_row_num += 1
|
17
|
-
row
|
18
|
-
end
|
19
|
-
|
20
|
-
# Releases the database resources hold by this cursor
|
21
|
-
def clear
|
22
|
-
free
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
1
|
module RR
|
27
2
|
|
28
|
-
# Overwrites #select_cursor to allow fetching of MySQL results in chunks
|
29
|
-
class ProxyConnection
|
30
|
-
|
31
|
-
# Allow selecting of MySQL results in chunks.
|
32
|
-
# For full documentation of method interface refer to ProxyConnection#select_cursor.
|
33
|
-
def select_cursor_with_mysql_chunks(options)
|
34
|
-
if config[:adapter] != 'mysql' or !options.include?(:row_buffer_size) or options.include?(:query)
|
35
|
-
select_cursor_without_mysql_chunks options
|
36
|
-
else
|
37
|
-
ConnectionExtenders::MysqlFetcher.new(self, options)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
alias_method_chain :select_cursor, :mysql_chunks unless method_defined?(:select_cursor_without_mysql_chunks)
|
41
|
-
|
42
|
-
end
|
43
|
-
|
44
3
|
module ConnectionExtenders
|
45
4
|
|
46
|
-
# Fetches MySQL results in chunks
|
47
|
-
class MysqlFetcher
|
48
|
-
|
49
|
-
# The current database ProxyConnection
|
50
|
-
attr_accessor :connection
|
51
|
-
|
52
|
-
# hash of select options
|
53
|
-
attr_accessor :options
|
54
|
-
|
55
|
-
# column_name => value hash of the last returned row
|
56
|
-
attr_accessor :last_row
|
57
|
-
|
58
|
-
# Creates a new fetcher.
|
59
|
-
# * +connection+: the current database connection
|
60
|
-
# * +cursor_name+: name of the cursor from which to fetch
|
61
|
-
# * +row_buffer_size+: number of records to read at once
|
62
|
-
def initialize(connection, options)
|
63
|
-
self.connection = connection
|
64
|
-
self.options = options.clone
|
65
|
-
end
|
66
|
-
|
67
|
-
# Returns +true+ if there are more rows to read.
|
68
|
-
def next?
|
69
|
-
unless @current_result
|
70
|
-
if last_row
|
71
|
-
options.merge! :from => last_row, :exclude_starting_row => true
|
72
|
-
end
|
73
|
-
options[:query] =
|
74
|
-
connection.table_select_query(options[:table], options) +
|
75
|
-
" limit #{options[:row_buffer_size]}"
|
76
|
-
@current_result = connection.select_cursor_without_mysql_chunks(options)
|
77
|
-
end
|
78
|
-
@current_result.next?
|
79
|
-
end
|
80
|
-
|
81
|
-
# Returns the row as a column => value hash and moves the cursor to the next row.
|
82
|
-
def next_row
|
83
|
-
raise("no more rows available") unless next?
|
84
|
-
self.last_row = @current_result.next_row
|
85
|
-
unless @current_result.next?
|
86
|
-
@current_result.clear
|
87
|
-
@current_result = nil
|
88
|
-
end
|
89
|
-
self.last_row
|
90
|
-
end
|
91
|
-
|
92
|
-
# Closes the cursor and frees up all ressources
|
93
|
-
def clear
|
94
|
-
if @current_result
|
95
|
-
@current_result.clear
|
96
|
-
@current_result = nil
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
5
|
# Provides various MySQL specific functionality required by Rubyrep.
|
102
6
|
module MysqlExtender
|
103
7
|
RR::ConnectionExtenders.register :mysql => self
|
104
8
|
|
105
|
-
# Executes the given sql query with the optional name written in the
|
106
|
-
# ActiveRecord log file.
|
107
|
-
# :+row_buffer_size+ is not currently used.
|
108
|
-
# Returns the results as a Cursor object supporting
|
109
|
-
# * next? - returns true if there are more rows to read
|
110
|
-
# * next_row - returns the row as a column => value hash and moves the cursor to the next row
|
111
|
-
# * clear - clearing the cursor (making allocated memory available for GC)
|
112
|
-
def select_cursor(sql, row_buffer_size = 1000)
|
113
|
-
result = execute sql
|
114
|
-
result.send :extend, MysqlResultExtender
|
115
|
-
result
|
116
|
-
end
|
117
|
-
|
118
9
|
# Returns an ordered list of primary key column names of the given table
|
119
10
|
def primary_key_names(table)
|
120
11
|
row = self.select_one(<<-end_sql)
|
@@ -1,34 +1,5 @@
|
|
1
1
|
require 'time'
|
2
2
|
|
3
|
-
# A cursor to iterate over the records returned by select_cursor.
|
4
|
-
# Only one row is kept in memory at a time.
|
5
|
-
class PGresult
|
6
|
-
# Returns true if there are more rows to read.
|
7
|
-
def next?
|
8
|
-
@current_row_num ||= 0
|
9
|
-
@num_rows ||= self.ntuples()
|
10
|
-
@current_row_num < @num_rows
|
11
|
-
end
|
12
|
-
|
13
|
-
# Returns the row as a column => value hash and moves the cursor to the next row.
|
14
|
-
def next_row
|
15
|
-
raise("no more rows available") unless next?
|
16
|
-
row = {}
|
17
|
-
@fields ||= self.fields
|
18
|
-
@fields.each_with_index do |field, field_index|
|
19
|
-
if self.getisnull(@current_row_num, field_index)
|
20
|
-
value = nil
|
21
|
-
else
|
22
|
-
value = self.getvalue @current_row_num, field_index
|
23
|
-
end
|
24
|
-
|
25
|
-
row[@fields[field_index]] = value
|
26
|
-
end
|
27
|
-
@current_row_num += 1
|
28
|
-
row
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
3
|
# Hack:
|
33
4
|
# For some reasons these methods were removed in Rails 2.2.2, thus breaking
|
34
5
|
# the binary and multi-lingual data loading.
|
@@ -110,83 +81,10 @@ end
|
|
110
81
|
module RR
|
111
82
|
module ConnectionExtenders
|
112
83
|
|
113
|
-
# Fetches results from a PostgreSQL cursor object.
|
114
|
-
class PostgreSQLFetcher
|
115
|
-
|
116
|
-
# The current database connection
|
117
|
-
attr_accessor :connection
|
118
|
-
|
119
|
-
# Name of the cursor from which to fetch
|
120
|
-
attr_accessor :cursor_name
|
121
|
-
|
122
|
-
# Number of rows to be read at once
|
123
|
-
attr_accessor :row_buffer_size
|
124
|
-
|
125
|
-
# Creates a new fetcher.
|
126
|
-
# * +connection+: the current database connection
|
127
|
-
# * +cursor_name+: name of the cursor from which to fetch
|
128
|
-
# * +row_buffer_size+: number of records to read at once
|
129
|
-
def initialize(connection, cursor_name, row_buffer_size)
|
130
|
-
self.connection = connection
|
131
|
-
self.cursor_name = cursor_name
|
132
|
-
self.row_buffer_size = row_buffer_size
|
133
|
-
end
|
134
|
-
|
135
|
-
# Executes the specified SQL staements, returning the result
|
136
|
-
def execute(sql)
|
137
|
-
connection.execute sql
|
138
|
-
end
|
139
|
-
|
140
|
-
# Returns true if there are more rows to read.
|
141
|
-
def next?
|
142
|
-
@current_result ||= execute("FETCH FORWARD #{row_buffer_size} FROM #{cursor_name}")
|
143
|
-
@current_result.next?
|
144
|
-
end
|
145
|
-
|
146
|
-
# Returns the row as a column => value hash and moves the cursor to the next row.
|
147
|
-
def next_row
|
148
|
-
raise("no more rows available") unless next?
|
149
|
-
row = @current_result.next_row
|
150
|
-
unless @current_result.next?
|
151
|
-
@current_result.clear
|
152
|
-
@current_result = nil
|
153
|
-
end
|
154
|
-
row
|
155
|
-
end
|
156
|
-
|
157
|
-
# Closes the cursor and frees up all ressources
|
158
|
-
def clear
|
159
|
-
if @current_result
|
160
|
-
@current_result.clear
|
161
|
-
@current_result = nil
|
162
|
-
end
|
163
|
-
result = execute("CLOSE #{cursor_name}")
|
164
|
-
result.clear if result
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
84
|
# Provides various PostgreSQL specific functionality required by Rubyrep.
|
169
85
|
module PostgreSQLExtender
|
170
86
|
RR::ConnectionExtenders.register :postgresql => self
|
171
87
|
|
172
|
-
# Executes the given sql query with the otional name written in the
|
173
|
-
# ActiveRecord log file.
|
174
|
-
#
|
175
|
-
# :+row_buffer_size+ controls how many records are ready into memory at a
|
176
|
-
# time. Implemented using the PostgeSQL "DECLARE CURSOR" and "FETCH" constructs.
|
177
|
-
# This is necessary as the postgresql driver always reads the
|
178
|
-
# complete resultset into memory.
|
179
|
-
#
|
180
|
-
# Returns the results as a Cursor object supporting
|
181
|
-
# * next? - returns true if there are more rows to read
|
182
|
-
# * next_row - returns the row as a column => value hash and moves the cursor to the next row
|
183
|
-
# * clear - clearing the cursor (making allocated memory available for GC)
|
184
|
-
def select_cursor(sql, row_buffer_size = 1000)
|
185
|
-
cursor_name = "RR_#{Time.now.to_i}#{rand(1_000_000)}"
|
186
|
-
execute("DECLARE #{cursor_name} NO SCROLL CURSOR WITH HOLD FOR " + sql)
|
187
|
-
PostgreSQLFetcher.new(self, cursor_name, row_buffer_size)
|
188
|
-
end
|
189
|
-
|
190
88
|
# Returns an ordered list of primary key column names of the given table
|
191
89
|
def primary_key_names(table)
|
192
90
|
row = self.select_one(<<-end_sql)
|
@@ -46,7 +46,7 @@ module RR
|
|
46
46
|
# 2nd level tree:
|
47
47
|
# * key: the change_key value of the according change log records.
|
48
48
|
# * value:
|
49
|
-
#
|
49
|
+
# An array of according change log records (column_name => value hash).
|
50
50
|
# Additional entry of each change log hash:
|
51
51
|
# * key: 'array_index'
|
52
52
|
# * value: index to the change log record in +change_array+
|
@@ -84,6 +84,19 @@ module RR
|
|
84
84
|
|
85
85
|
self.last_updated = Time.now
|
86
86
|
|
87
|
+
# First, let's use a LIMIT clause (via :row_buffer_size option) to verify
|
88
|
+
# if there are any pending changes.
|
89
|
+
# (If there are many pending changes, this is (at least with PostgreSQL)
|
90
|
+
# much faster.)
|
91
|
+
cursor = connection.select_cursor(
|
92
|
+
:table => change_log_table,
|
93
|
+
:from => {'id' => current_id},
|
94
|
+
:exclude_starting_row => true,
|
95
|
+
:row_buffer_size => 1
|
96
|
+
)
|
97
|
+
return unless cursor.next?
|
98
|
+
|
99
|
+
# Something is here. Let's actually load it.
|
87
100
|
cursor = connection.select_cursor(
|
88
101
|
:table => change_log_table,
|
89
102
|
:from => {'id' => current_id},
|
@@ -9,6 +9,87 @@ require 'active_record/connection_adapters/abstract_adapter'
|
|
9
9
|
|
10
10
|
module RR
|
11
11
|
|
12
|
+
# Enables the fetching of (potential large) result sets in chunks.
|
13
|
+
class ResultFetcher
|
14
|
+
|
15
|
+
# The current database ProxyConnection
|
16
|
+
attr_accessor :connection
|
17
|
+
|
18
|
+
# hash of select options as described under ProxyConnection#select_cursor
|
19
|
+
attr_accessor :options
|
20
|
+
|
21
|
+
# column_name => value hash of the last returned row
|
22
|
+
attr_accessor :last_row
|
23
|
+
|
24
|
+
# The current row set: an array of column_name => value hashes
|
25
|
+
attr_accessor :rows
|
26
|
+
|
27
|
+
# Index to the current row in rows
|
28
|
+
attr_accessor :current_row_index
|
29
|
+
|
30
|
+
# Creates a new fetcher.
|
31
|
+
# * +connection+: the current ProxyConnection
|
32
|
+
# * +options+: hash of select options as described under ProxyConnection#select_cursor
|
33
|
+
def initialize(connection, options)
|
34
|
+
self.connection = connection
|
35
|
+
self.options = options.clone
|
36
|
+
end
|
37
|
+
|
38
|
+
# Returns +true+ if there are more rows to read.
|
39
|
+
def next?
|
40
|
+
unless self.rows
|
41
|
+
# Try to load some records
|
42
|
+
|
43
|
+
if options[:query] and last_row != nil
|
44
|
+
# A query was directly specified and all it's rows were returned
|
45
|
+
# ==> Finished.
|
46
|
+
return false
|
47
|
+
end
|
48
|
+
|
49
|
+
if options[:query]
|
50
|
+
# If a query has been directly specified, just directly execute it
|
51
|
+
query = options[:query]
|
52
|
+
else
|
53
|
+
# Otherwise build the query
|
54
|
+
if last_row
|
55
|
+
# There was a previous batch.
|
56
|
+
# Next batch will start after the last returned row
|
57
|
+
options.merge! :from => last_row, :exclude_starting_row => true
|
58
|
+
end
|
59
|
+
|
60
|
+
query = connection.table_select_query(options[:table], options)
|
61
|
+
|
62
|
+
if options[:row_buffer_size]
|
63
|
+
# Set the batch size
|
64
|
+
query += " limit #{options[:row_buffer_size]}"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
self.rows = connection.select_all query
|
69
|
+
self.current_row_index = 0
|
70
|
+
end
|
71
|
+
self.current_row_index < self.rows.size
|
72
|
+
end
|
73
|
+
|
74
|
+
# Returns the row as a column => value hash and moves the cursor to the next row.
|
75
|
+
def next_row
|
76
|
+
raise("no more rows available") unless next?
|
77
|
+
self.last_row = self.rows[self.current_row_index]
|
78
|
+
self.current_row_index += 1
|
79
|
+
|
80
|
+
if self.current_row_index == self.rows.size
|
81
|
+
self.rows = nil
|
82
|
+
end
|
83
|
+
|
84
|
+
self.last_row
|
85
|
+
end
|
86
|
+
|
87
|
+
# Frees up all ressources
|
88
|
+
def clear
|
89
|
+
self.rows = nil
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
12
93
|
# This class represents a remote activerecord database connection.
|
13
94
|
# Normally created by DatabaseProxy
|
14
95
|
class ProxyConnection
|
@@ -102,9 +183,7 @@ module RR
|
|
102
183
|
# * :+row_buffer_size+:
|
103
184
|
# Integer controlling how many rows a read into memory at one time.
|
104
185
|
def select_cursor(options)
|
105
|
-
|
106
|
-
query = options[:query] || table_select_query(options[:table], options)
|
107
|
-
cursor = connection.select_cursor query, row_buffer_size
|
186
|
+
cursor = ResultFetcher.new(self, options)
|
108
187
|
if options[:type_cast]
|
109
188
|
cursor = TypeCastingCursor.new(self, options[:table], cursor)
|
110
189
|
end
|
@@ -19,24 +19,31 @@ module RR
|
|
19
19
|
|
20
20
|
# Executes the replication run.
|
21
21
|
def run
|
22
|
-
|
23
|
-
|
22
|
+
return unless [:left, :right].any? do |database|
|
23
|
+
session.send(database).select_one(
|
24
|
+
"select id from #{session.configuration.options[:rep_prefix]}_pending_changes"
|
25
|
+
) != nil
|
26
|
+
end
|
27
|
+
begin
|
28
|
+
success = false
|
29
|
+
replicator # ensure that replicator is created and has chance to validate settings
|
24
30
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
31
|
+
loop do
|
32
|
+
begin
|
33
|
+
session.reload_changes # ensure the cache of change log records is up-to-date
|
34
|
+
diff = ReplicationDifference.new session
|
35
|
+
diff.load
|
36
|
+
break unless diff.loaded?
|
37
|
+
replicator.replicate_difference diff if diff.type != :no_diff
|
38
|
+
rescue Exception => e
|
39
|
+
helper.log_replication_outcome diff, e.message,
|
40
|
+
e.class.to_s + "\n" + e.backtrace.join("\n")
|
41
|
+
end
|
35
42
|
end
|
43
|
+
success = true # considered to be successful if we get till here
|
44
|
+
ensure
|
45
|
+
helper.finalize success
|
36
46
|
end
|
37
|
-
success = true # considered to be successful if we get till here
|
38
|
-
ensure
|
39
|
-
helper.finalize success
|
40
47
|
end
|
41
48
|
|
42
49
|
# Creates a new ReplicationRun instance.
|
@@ -55,7 +55,7 @@ module RR
|
|
55
55
|
#
|
56
56
|
# Takes care that a table is only returned once.
|
57
57
|
def resolve(included_table_specs, excluded_table_specs = [], verify = true)
|
58
|
-
table_pairs = expand_table_specs(included_table_specs)
|
58
|
+
table_pairs = expand_table_specs(included_table_specs, verify)
|
59
59
|
table_pairs = table_pairs_without_duplicates(table_pairs)
|
60
60
|
table_pairs = table_pairs_without_excluded(table_pairs, excluded_table_specs)
|
61
61
|
|
@@ -70,11 +70,15 @@ module RR
|
|
70
70
|
end
|
71
71
|
|
72
72
|
# Helper for #resolve
|
73
|
-
#
|
74
|
-
#
|
75
|
-
#
|
76
|
-
#
|
77
|
-
|
73
|
+
# Expands table specifications into table pairs.
|
74
|
+
# Parameters:
|
75
|
+
# * +table_specs+:
|
76
|
+
# An array of table specifications as described under #resolve.
|
77
|
+
# * +verify+:
|
78
|
+
# If +true+, table specs in regexp format only resolve if the table exists
|
79
|
+
# in left and right database.
|
80
|
+
# Return value: refer to #resolve for a detailed description
|
81
|
+
def expand_table_specs(table_specs, verify)
|
78
82
|
table_pairs = []
|
79
83
|
table_specs.each do |table_spec|
|
80
84
|
|
@@ -86,7 +90,9 @@ module RR
|
|
86
90
|
table_spec = table_spec.sub(/^\/(.*)\/$/,'\1') # remove leading and trailing slash
|
87
91
|
matching_tables = tables(:left).grep(Regexp.new(table_spec, Regexp::IGNORECASE, 'U'))
|
88
92
|
matching_tables.each do |table|
|
89
|
-
|
93
|
+
if !verify or tables(:right).include? table
|
94
|
+
table_pairs << {:left => table, :right => table}
|
95
|
+
end
|
90
96
|
end
|
91
97
|
when /.+,.+/ # matches e. g. 'users,users_backup'
|
92
98
|
pair = table_spec.match(/(.*),(.*)/)[1..2].map { |str| str.strip }
|
@@ -107,7 +113,7 @@ module RR
|
|
107
113
|
# * :+right+: name of the corresponding right table
|
108
114
|
# +excluded_table_specs+ is the array of table specifications to be excluded.
|
109
115
|
def table_pairs_without_excluded(table_pairs, excluded_table_specs)
|
110
|
-
excluded_tables = expand_table_specs(excluded_table_specs).map do |table_pair|
|
116
|
+
excluded_tables = expand_table_specs(excluded_table_specs, false).map do |table_pair|
|
111
117
|
table_pair[:left]
|
112
118
|
end
|
113
119
|
table_pairs.select {|table_pair| not excluded_tables.include? table_pair[:left]}
|
data/lib/rubyrep/version.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper.rb'
|
2
2
|
require 'yaml'
|
3
|
+
require 'digest/md5'
|
3
4
|
|
4
5
|
include RR
|
5
6
|
|
@@ -89,20 +90,27 @@ describe "ConnectionExtender", :shared => true do
|
|
89
90
|
)
|
90
91
|
result.next_row.should == {'first_id' => '3', 'second_id' => '1', 'name' => nil}
|
91
92
|
end
|
92
|
-
|
93
|
+
|
93
94
|
it "should read and write binary data correctly" do
|
94
95
|
session = Session.new
|
95
96
|
|
96
|
-
org_data =
|
97
|
+
org_data = File.new(File.dirname(__FILE__) + '/dolphins.jpg').read
|
97
98
|
result_data = nil
|
98
99
|
begin
|
99
100
|
session.left.begin_db_transaction
|
100
|
-
|
101
|
-
|
101
|
+
session.left.insert_record('extender_type_check', {'id' => 6, 'binary_test' => org_data})
|
102
|
+
|
103
|
+
row = session.left.select_one(
|
104
|
+
'select md5(binary_test) as md5 from extender_type_check where id = 6'
|
105
|
+
)
|
106
|
+
row['md5'].should == Digest::MD5.hexdigest(org_data)
|
102
107
|
|
103
|
-
|
104
|
-
|
108
|
+
cursor = session.left.select_cursor(
|
109
|
+
:query => "select id, binary_test from extender_type_check where id = 6"
|
110
|
+
)
|
111
|
+
cursor = TypeCastingCursor.new session.left, 'extender_type_check', cursor
|
105
112
|
result_data = cursor.next_row['binary_test']
|
113
|
+
Digest::MD5.hexdigest(result_data).should == Digest::MD5.hexdigest(org_data)
|
106
114
|
ensure
|
107
115
|
session.left.rollback_db_transaction
|
108
116
|
end
|
@@ -60,8 +60,8 @@ describe ConnectionExtenders, "Registration" do
|
|
60
60
|
|
61
61
|
it "db_connect should include the connection extender into connection" do
|
62
62
|
connection = ConnectionExtenders.db_connect Initializer.configuration.left
|
63
|
-
|
64
|
-
connection.should respond_to(:
|
63
|
+
|
64
|
+
connection.should respond_to(:primary_key_names)
|
65
65
|
end
|
66
66
|
|
67
67
|
it "db_connect should raise an Exception if no fitting connection extender is available" do
|
@@ -120,28 +120,20 @@ describe ProxyConnection do
|
|
120
120
|
@connection.primary_key_names('dummy_table').should == ['dummy_key']
|
121
121
|
end
|
122
122
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
it "select_cursor should use the default row buffer size if no explicit value is specified" do
|
129
|
-
@connection.connection.should_receive(:select_cursor) \
|
130
|
-
.with('bla', ProxyConnection::DEFAULT_ROW_BUFFER_SIZE)
|
131
|
-
@connection.select_cursor(:query => 'bla')
|
132
|
-
end
|
123
|
+
# Note:
|
124
|
+
# Additional select_cursor tests are executed via
|
125
|
+
# 'db_specific_connection_extenders_spec.rb'
|
126
|
+
# (To verify the behaviour for all supported databases)
|
133
127
|
|
134
|
-
it "select_cursor should
|
135
|
-
@connection.
|
136
|
-
|
137
|
-
|
128
|
+
it "select_cursor should return the result fetcher" do
|
129
|
+
fetcher = @connection.select_cursor(:table => 'scanner_records')
|
130
|
+
fetcher.connection.should == @connection
|
131
|
+
fetcher.options.should == {:table => 'scanner_records'}
|
138
132
|
end
|
139
133
|
|
140
134
|
it "select_cursor should return a type casting cursor if :type_cast option is specified" do
|
141
|
-
@connection.select_cursor(:table => 'scanner_records')
|
142
|
-
|
143
|
-
@connection.select_cursor(:table => 'scanner_records', :type_cast => true).
|
144
|
-
should be_an_instance_of(TypeCastingCursor)
|
135
|
+
fetcher = @connection.select_cursor(:table => 'scanner_records', :type_cast => true)
|
136
|
+
fetcher.should be_an_instance_of(TypeCastingCursor)
|
145
137
|
end
|
146
138
|
|
147
139
|
it "table_select_query should handle queries without any conditions" do
|
@@ -64,6 +64,13 @@ describe ReplicationRun do
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
+
it "run should not create the replicator if there are no pending changes" do
|
68
|
+
session = Session.new
|
69
|
+
run = ReplicationRun.new session
|
70
|
+
run.should_not_receive(:replicator)
|
71
|
+
run.run
|
72
|
+
end
|
73
|
+
|
67
74
|
it "run should only replicate real differences" do
|
68
75
|
session = Session.new
|
69
76
|
session.left.begin_db_transaction
|
@@ -121,8 +128,22 @@ describe ReplicationRun do
|
|
121
128
|
it "run should not catch exceptions raised during replicator initialization" do
|
122
129
|
config = deep_copy(standard_config)
|
123
130
|
config.options[:logged_replication_events] = [:invalid_option]
|
124
|
-
|
125
|
-
|
131
|
+
session = Session.new config
|
132
|
+
session.left.begin_db_transaction
|
133
|
+
begin
|
134
|
+
|
135
|
+
session.left.insert_record 'rr_pending_changes', {
|
136
|
+
'change_table' => 'extender_no_record',
|
137
|
+
'change_key' => 'id|1',
|
138
|
+
'change_type' => 'D',
|
139
|
+
'change_time' => Time.now
|
140
|
+
}
|
141
|
+
|
142
|
+
run = ReplicationRun.new session
|
143
|
+
lambda {run.run}.should raise_error(ArgumentError)
|
144
|
+
ensure
|
145
|
+
session.left.rollback_db_transaction
|
146
|
+
end
|
126
147
|
end
|
127
148
|
|
128
149
|
it "run should process trigger created change log records" do
|
@@ -34,6 +34,15 @@ describe TableSpecResolver do
|
|
34
34
|
it "resolve should complain about non-existing tables" do
|
35
35
|
lambda {@resolver.resolve(['dummy, scanner_records'])}.
|
36
36
|
should raise_error(/non-existing.*dummy/)
|
37
|
+
lambda {@resolver.resolve(['left_table, left_table'])}.
|
38
|
+
should raise_error(/non-existing.*left_table/)
|
39
|
+
lambda {@resolver.resolve(['left_table'])}.
|
40
|
+
should raise_error(/non-existing.*left_table/)
|
41
|
+
end
|
42
|
+
|
43
|
+
it "resolve should not complain about regexp specified tables not existing in right database" do
|
44
|
+
@resolver.resolve([/^scanner_records$/, /left_table/]).
|
45
|
+
should == [{:left => 'scanner_records', :right => 'scanner_records'}]
|
37
46
|
end
|
38
47
|
|
39
48
|
it "resolve should not check for non-existing tables if that is disabled" do
|
@@ -85,7 +94,7 @@ describe TableSpecResolver do
|
|
85
94
|
@resolver.non_existing_tables(table_pairs).should == {}
|
86
95
|
end
|
87
96
|
|
88
|
-
it "non_existing_tables should return
|
97
|
+
it "non_existing_tables should return a hash of non-existing tables" do
|
89
98
|
table_pairs = [{:left => 'scanner_records', :right => 'bla'}]
|
90
99
|
@resolver.non_existing_tables(table_pairs).should == {:right => ['bla']}
|
91
100
|
|
data/tasks/database.rake
CHANGED
@@ -102,11 +102,13 @@ def drop_postgres_schema(config)
|
|
102
102
|
end
|
103
103
|
|
104
104
|
# Creates the sample schema in the database specified by the given
|
105
|
-
# configuration
|
106
|
-
|
107
|
-
|
105
|
+
# configuration.
|
106
|
+
# * :+database+: either :+left+ or +:right+
|
107
|
+
# * :+config+: the Configuration object
|
108
|
+
def create_sample_schema(database, config)
|
109
|
+
create_postgres_schema config.send(database)
|
108
110
|
|
109
|
-
ActiveRecord::Base.establish_connection config
|
111
|
+
ActiveRecord::Base.establish_connection config.send(database)
|
110
112
|
|
111
113
|
ActiveRecord::Schema.define do
|
112
114
|
create_table :scanner_text_key, :id => false do |t|
|
@@ -252,11 +254,11 @@ def create_sample_schema(config)
|
|
252
254
|
|
253
255
|
create_table :left_table do |t|
|
254
256
|
t.column :name, :string
|
255
|
-
end
|
257
|
+
end if database == :left
|
256
258
|
|
257
259
|
create_table :right_table do |t|
|
258
260
|
t.column :name, :string
|
259
|
-
end
|
261
|
+
end if database == :right
|
260
262
|
end
|
261
263
|
end
|
262
264
|
|
@@ -410,8 +412,8 @@ namespace :db do
|
|
410
412
|
|
411
413
|
desc "Create the sample schemas"
|
412
414
|
task :create_schema do
|
413
|
-
create_sample_schema RR::Initializer.configuration
|
414
|
-
create_sample_schema RR::Initializer.configuration
|
415
|
+
create_sample_schema :left, RR::Initializer.configuration rescue nil
|
416
|
+
create_sample_schema :right, RR::Initializer.configuration rescue nil
|
415
417
|
end
|
416
418
|
|
417
419
|
desc "Writes the sample data"
|
data/tasks/java.rake
CHANGED
@@ -23,7 +23,8 @@ EOS
|
|
23
23
|
pkg_name = "rubyrep-#{RR::VERSION::STRING}"
|
24
24
|
|
25
25
|
system "rm -rf /tmp/#{pkg_name}"
|
26
|
-
system "
|
26
|
+
system "mkdir /tmp/#{pkg_name}"
|
27
|
+
system "git archive master |tar -x -C /tmp/#{pkg_name}"
|
27
28
|
system "mkdir -p /tmp/#{pkg_name}/jruby"
|
28
29
|
system "cp -r #{JRUBY_HOME}/* /tmp/#{pkg_name}/jruby/"
|
29
30
|
system "cd /tmp/#{pkg_name}/jruby; rm -rf samples share/ri lib/ruby/gems/1.8/doc"
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rubyrep
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arndt Lehmann
|
@@ -30,7 +30,7 @@ cert_chain:
|
|
30
30
|
NwT26VZnE2nr8g==
|
31
31
|
-----END CERTIFICATE-----
|
32
32
|
|
33
|
-
date: 2009-
|
33
|
+
date: 2009-06-13 00:00:00 +09:00
|
34
34
|
default_executable:
|
35
35
|
dependencies:
|
36
36
|
- !ruby/object:Gem::Dependency
|
metadata.gz.sig
CHANGED
Binary file
|