sq-dbsync 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +6 -0
- data/lib/sq/dbsync/manager.rb +0 -7
- data/lib/sq/dbsync/version.rb +1 -1
- data/sq-dbsync.gemspec +2 -2
- metadata +5 -6
- data/lib/sq/dbsync/consistency_verifier.rb +0 -70
- data/spec/integration/consistency_verifier_spec.rb +0 -54
data/README.md
CHANGED
data/lib/sq/dbsync/manager.rb
CHANGED
@@ -4,7 +4,6 @@ require 'sq/dbsync/incremental_load_action'
|
|
4
4
|
require 'sq/dbsync/refresh_recent_load_action'
|
5
5
|
require 'sq/dbsync/pipeline'
|
6
6
|
require 'sq/dbsync/table_registry'
|
7
|
-
require 'sq/dbsync/consistency_verifier'
|
8
7
|
require 'sq/dbsync/database/connection'
|
9
8
|
require 'sq/dbsync/error_handler'
|
10
9
|
|
@@ -98,8 +97,6 @@ class Sq::Dbsync::Manager
|
|
98
97
|
# No need to do this every cycle, 100 is chosen to be as good as any
|
99
98
|
# other number. It should run on the very first cycle however so that
|
100
99
|
# our specs will cover it.
|
101
|
-
verifier.check_consistency!(tables_to_load)
|
102
|
-
|
103
100
|
purge_registry
|
104
101
|
end
|
105
102
|
|
@@ -215,10 +212,6 @@ class Sq::Dbsync::Manager
|
|
215
212
|
TableRegistry.new(target)
|
216
213
|
end
|
217
214
|
|
218
|
-
def verifier
|
219
|
-
@verifier ||= ConsistencyVerifier.new(target, registry)
|
220
|
-
end
|
221
|
-
|
222
215
|
def logger
|
223
216
|
config[:logger]
|
224
217
|
end
|
data/lib/sq/dbsync/version.rb
CHANGED
data/sq-dbsync.gemspec
CHANGED
@@ -2,8 +2,8 @@
|
|
2
2
|
require File.expand_path('../lib/sq/dbsync/version', __FILE__)
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
|
-
gem.authors = ["Xavier Shay"]
|
6
|
-
gem.email = ["xavier@squareup.com"]
|
5
|
+
gem.authors = ["Xavier Shay", "Damon McCormick"]
|
6
|
+
gem.email = ["xavier@squareup.com", "damon@squaerup.com"]
|
7
7
|
gem.description =
|
8
8
|
%q{Column based, timestamp replication of MySQL and Postgres databases.}
|
9
9
|
gem.summary = %q{
|
metadata
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sq-dbsync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Xavier Shay
|
9
|
+
- Damon McCormick
|
9
10
|
autorequire:
|
10
11
|
bindir: bin
|
11
12
|
cert_chain: []
|
12
|
-
date: 2013-
|
13
|
+
date: 2013-03-27 00:00:00.000000000 Z
|
13
14
|
dependencies:
|
14
15
|
- !ruby/object:Gem::Dependency
|
15
16
|
name: rspec
|
@@ -94,6 +95,7 @@ dependencies:
|
|
94
95
|
description: Column based, timestamp replication of MySQL and Postgres databases.
|
95
96
|
email:
|
96
97
|
- xavier@squareup.com
|
98
|
+
- damon@squaerup.com
|
97
99
|
executables: []
|
98
100
|
extensions: []
|
99
101
|
extra_rdoc_files: []
|
@@ -103,7 +105,6 @@ files:
|
|
103
105
|
- spec/database_helper.rb
|
104
106
|
- spec/integration/all_tables_plan_spec.rb
|
105
107
|
- spec/integration/batch_load_action_spec.rb
|
106
|
-
- spec/integration/consistency_verifier_spec.rb
|
107
108
|
- spec/integration/database_connection_spec.rb
|
108
109
|
- spec/integration/incremental_load_action_spec.rb
|
109
110
|
- spec/integration/manager_spec.rb
|
@@ -118,7 +119,6 @@ files:
|
|
118
119
|
- lib/sq/dbsync/all_tables_plan.rb
|
119
120
|
- lib/sq/dbsync/batch_load_action.rb
|
120
121
|
- lib/sq/dbsync/config.rb
|
121
|
-
- lib/sq/dbsync/consistency_verifier.rb
|
122
122
|
- lib/sq/dbsync/database/common.rb
|
123
123
|
- lib/sq/dbsync/database/connection.rb
|
124
124
|
- lib/sq/dbsync/database/mysql.rb
|
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
163
163
|
version: '0'
|
164
164
|
requirements: []
|
165
165
|
rubyforge_project:
|
166
|
-
rubygems_version: 1.8.
|
166
|
+
rubygems_version: 1.8.24
|
167
167
|
signing_key:
|
168
168
|
specification_version: 3
|
169
169
|
summary: Column based, timestamp replication of MySQL and Postgres databases. Uses
|
@@ -174,7 +174,6 @@ test_files:
|
|
174
174
|
- spec/database_helper.rb
|
175
175
|
- spec/integration/all_tables_plan_spec.rb
|
176
176
|
- spec/integration/batch_load_action_spec.rb
|
177
|
-
- spec/integration/consistency_verifier_spec.rb
|
178
177
|
- spec/integration/database_connection_spec.rb
|
179
178
|
- spec/integration/incremental_load_action_spec.rb
|
180
179
|
- spec/integration/manager_spec.rb
|
@@ -1,70 +0,0 @@
|
|
1
|
-
require 'sq/dbsync/load_action' # For overlap, not ideal
|
2
|
-
|
3
|
-
module Sq::Dbsync
|
4
|
-
|
5
|
-
# Performs a cheap check to verify that the number of records present for a
|
6
|
-
# recent time slice are the same across source and target tables.
|
7
|
-
#
|
8
|
-
# This checks consistency on the current tables, not the new_ set.
|
9
|
-
class ConsistencyVerifier
|
10
|
-
def initialize(target, registry)
|
11
|
-
@target = target
|
12
|
-
@registry = registry
|
13
|
-
end
|
14
|
-
|
15
|
-
def check_consistency!(tables)
|
16
|
-
tables.each do |tplan|
|
17
|
-
next unless tplan[:consistency]
|
18
|
-
verify_consistency!(tplan)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def verify_consistency!(tplan)
|
23
|
-
last_row_at = registry.get(tplan[:table_name])[:last_row_at]
|
24
|
-
return unless last_row_at
|
25
|
-
|
26
|
-
now = registry.get(tplan[:table_name])[:last_row_at] - LoadAction.overlap
|
27
|
-
|
28
|
-
counts = [
|
29
|
-
tplan[:source_db],
|
30
|
-
target
|
31
|
-
].map do |x|
|
32
|
-
x.consistency_check(tplan[:table_name], now)
|
33
|
-
end
|
34
|
-
|
35
|
-
delta = counts.reduce(:-)
|
36
|
-
|
37
|
-
unless delta == 0
|
38
|
-
raise ConsistencyError.new(
|
39
|
-
tplan[:table_name],
|
40
|
-
delta,
|
41
|
-
"source: #{tplan[:source_db].name} (count: #{counts[0]}), " +
|
42
|
-
"sink: #{target.name} (count: #{counts[1]})"
|
43
|
-
)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
attr_reader :target, :registry
|
48
|
-
|
49
|
-
# Used to signal an observed error in the number of records between source
|
50
|
-
# and target tables. There are no current known situations in which this
|
51
|
-
# occurs, though in the past buggy handling of replication lag was normally
|
52
|
-
# the culprit.
|
53
|
-
#
|
54
|
-
# If it does occur, a good first response is to set `last_sync_time` to the
|
55
|
-
# last batch time (usually within 24 hours) which will force the
|
56
|
-
# incremental load to reconsider all recent records.
|
57
|
-
class ConsistencyError < RuntimeError
|
58
|
-
def initialize(table_name, delta, description="")
|
59
|
-
@table_name = table_name
|
60
|
-
@delta = delta
|
61
|
-
@description = description
|
62
|
-
end
|
63
|
-
|
64
|
-
def message
|
65
|
-
output = "%s had a count difference of %i" % [@table_name, @delta]
|
66
|
-
output = output + "; " + @description if !@description.empty?
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
require 'integration_helper'
|
2
|
-
|
3
|
-
require 'sq/dbsync/consistency_verifier'
|
4
|
-
require 'sq/dbsync/static_table_plan'
|
5
|
-
require 'sq/dbsync/table_registry'
|
6
|
-
|
7
|
-
describe SQD::ConsistencyVerifier do
|
8
|
-
let(:overlap) { SQD::LoadAction.overlap }
|
9
|
-
let(:now) { Date.new(2012, 4, 4).to_time.utc }
|
10
|
-
let(:source) { test_source(:source) }
|
11
|
-
let(:target) { test_target }
|
12
|
-
let(:tables) {[{
|
13
|
-
table_name: :test_table,
|
14
|
-
columns: [:id, :col1, :updated_at],
|
15
|
-
consistency: true,
|
16
|
-
source_db: source,
|
17
|
-
indexes: {}
|
18
|
-
}]}
|
19
|
-
let(:registry) { SQD::TableRegistry.new(target) }
|
20
|
-
let(:verifier) { SQD::ConsistencyVerifier.new(target, registry) }
|
21
|
-
|
22
|
-
before do
|
23
|
-
create_source_table_with(
|
24
|
-
id: 1,
|
25
|
-
col1: 'old record',
|
26
|
-
created_at: now - overlap
|
27
|
-
)
|
28
|
-
setup_target_table(now)
|
29
|
-
end
|
30
|
-
|
31
|
-
it 'raises if counts do not match up' do
|
32
|
-
error_string =
|
33
|
-
"test_table had a count difference of 1; " +
|
34
|
-
"source: #{source.name} (count: 1), " +
|
35
|
-
"sink: #{target.name} (count: 0)"
|
36
|
-
|
37
|
-
lambda {
|
38
|
-
verifier.check_consistency!(tables)
|
39
|
-
}.should raise_error(
|
40
|
-
SQD::ConsistencyVerifier::ConsistencyError,
|
41
|
-
error_string
|
42
|
-
)
|
43
|
-
end
|
44
|
-
|
45
|
-
it 'uses last_row_at rather than last_synced_at' do
|
46
|
-
registry.update(:test_table, now,
|
47
|
-
last_row_at: now - 3
|
48
|
-
)
|
49
|
-
|
50
|
-
lambda {
|
51
|
-
verifier.check_consistency!(tables)
|
52
|
-
}.should_not raise_error(SQD::ConsistencyVerifier::ConsistencyError)
|
53
|
-
end
|
54
|
-
end
|