sq-dbsync 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -18,7 +18,13 @@ but `sq-dbsync` can tide you over until you get there.
18
18
  Usage
19
19
  -----
20
20
 
21
+ ```
22
+ gem install sq-dbsync
23
+ ```
24
+
21
25
  ``` Ruby
26
+ require 'sq/dbsync'
27
+
22
28
  include Sq::Dbsync
23
29
 
24
30
  # Config will typically differ per environment.
@@ -4,7 +4,6 @@ require 'sq/dbsync/incremental_load_action'
4
4
  require 'sq/dbsync/refresh_recent_load_action'
5
5
  require 'sq/dbsync/pipeline'
6
6
  require 'sq/dbsync/table_registry'
7
- require 'sq/dbsync/consistency_verifier'
8
7
  require 'sq/dbsync/database/connection'
9
8
  require 'sq/dbsync/error_handler'
10
9
 
@@ -98,8 +97,6 @@ class Sq::Dbsync::Manager
98
97
  # No need to do this every cycle, 100 is chosen to be as good as any
99
98
  # other number. It should run on the very first cycle however so that
100
99
  # our specs will cover it.
101
- verifier.check_consistency!(tables_to_load)
102
-
103
100
  purge_registry
104
101
  end
105
102
 
@@ -215,10 +212,6 @@ class Sq::Dbsync::Manager
215
212
  TableRegistry.new(target)
216
213
  end
217
214
 
218
- def verifier
219
- @verifier ||= ConsistencyVerifier.new(target, registry)
220
- end
221
-
222
215
  def logger
223
216
  config[:logger]
224
217
  end
@@ -1,5 +1,5 @@
1
1
  module Sq
2
2
  module Dbsync
3
- VERSION = '1.0.0'
3
+ VERSION = '1.0.1'
4
4
  end
5
5
  end
data/sq-dbsync.gemspec CHANGED
@@ -2,8 +2,8 @@
2
2
  require File.expand_path('../lib/sq/dbsync/version', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
- gem.authors = ["Xavier Shay"]
6
- gem.email = ["xavier@squareup.com"]
5
+ gem.authors = ["Xavier Shay", "Damon McCormick"]
6
+ gem.email = ["xavier@squareup.com", "damon@squaerup.com"]
7
7
  gem.description =
8
8
  %q{Column based, timestamp replication of MySQL and Postgres databases.}
9
9
  gem.summary = %q{
metadata CHANGED
@@ -1,15 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sq-dbsync
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Xavier Shay
9
+ - Damon McCormick
9
10
  autorequire:
10
11
  bindir: bin
11
12
  cert_chain: []
12
- date: 2013-02-23 00:00:00.000000000 Z
13
+ date: 2013-03-27 00:00:00.000000000 Z
13
14
  dependencies:
14
15
  - !ruby/object:Gem::Dependency
15
16
  name: rspec
@@ -94,6 +95,7 @@ dependencies:
94
95
  description: Column based, timestamp replication of MySQL and Postgres databases.
95
96
  email:
96
97
  - xavier@squareup.com
98
+ - damon@squaerup.com
97
99
  executables: []
98
100
  extensions: []
99
101
  extra_rdoc_files: []
@@ -103,7 +105,6 @@ files:
103
105
  - spec/database_helper.rb
104
106
  - spec/integration/all_tables_plan_spec.rb
105
107
  - spec/integration/batch_load_action_spec.rb
106
- - spec/integration/consistency_verifier_spec.rb
107
108
  - spec/integration/database_connection_spec.rb
108
109
  - spec/integration/incremental_load_action_spec.rb
109
110
  - spec/integration/manager_spec.rb
@@ -118,7 +119,6 @@ files:
118
119
  - lib/sq/dbsync/all_tables_plan.rb
119
120
  - lib/sq/dbsync/batch_load_action.rb
120
121
  - lib/sq/dbsync/config.rb
121
- - lib/sq/dbsync/consistency_verifier.rb
122
122
  - lib/sq/dbsync/database/common.rb
123
123
  - lib/sq/dbsync/database/connection.rb
124
124
  - lib/sq/dbsync/database/mysql.rb
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
163
163
  version: '0'
164
164
  requirements: []
165
165
  rubyforge_project:
166
- rubygems_version: 1.8.23
166
+ rubygems_version: 1.8.24
167
167
  signing_key:
168
168
  specification_version: 3
169
169
  summary: Column based, timestamp replication of MySQL and Postgres databases. Uses
@@ -174,7 +174,6 @@ test_files:
174
174
  - spec/database_helper.rb
175
175
  - spec/integration/all_tables_plan_spec.rb
176
176
  - spec/integration/batch_load_action_spec.rb
177
- - spec/integration/consistency_verifier_spec.rb
178
177
  - spec/integration/database_connection_spec.rb
179
178
  - spec/integration/incremental_load_action_spec.rb
180
179
  - spec/integration/manager_spec.rb
@@ -1,70 +0,0 @@
1
- require 'sq/dbsync/load_action' # For overlap, not ideal
2
-
3
- module Sq::Dbsync
4
-
5
- # Performs a cheap check to verify that the number of records present for a
6
- # recent time slice are the same across source and target tables.
7
- #
8
- # This checks consistency on the current tables, not the new_ set.
9
- class ConsistencyVerifier
10
- def initialize(target, registry)
11
- @target = target
12
- @registry = registry
13
- end
14
-
15
- def check_consistency!(tables)
16
- tables.each do |tplan|
17
- next unless tplan[:consistency]
18
- verify_consistency!(tplan)
19
- end
20
- end
21
-
22
- def verify_consistency!(tplan)
23
- last_row_at = registry.get(tplan[:table_name])[:last_row_at]
24
- return unless last_row_at
25
-
26
- now = registry.get(tplan[:table_name])[:last_row_at] - LoadAction.overlap
27
-
28
- counts = [
29
- tplan[:source_db],
30
- target
31
- ].map do |x|
32
- x.consistency_check(tplan[:table_name], now)
33
- end
34
-
35
- delta = counts.reduce(:-)
36
-
37
- unless delta == 0
38
- raise ConsistencyError.new(
39
- tplan[:table_name],
40
- delta,
41
- "source: #{tplan[:source_db].name} (count: #{counts[0]}), " +
42
- "sink: #{target.name} (count: #{counts[1]})"
43
- )
44
- end
45
- end
46
-
47
- attr_reader :target, :registry
48
-
49
- # Used to signal an observed error in the number of records between source
50
- # and target tables. There are no current known situations in which this
51
- # occurs, though in the past buggy handling of replication lag was normally
52
- # the culprit.
53
- #
54
- # If it does occur, a good first response is to set `last_sync_time` to the
55
- # last batch time (usually within 24 hours) which will force the
56
- # incremental load to reconsider all recent records.
57
- class ConsistencyError < RuntimeError
58
- def initialize(table_name, delta, description="")
59
- @table_name = table_name
60
- @delta = delta
61
- @description = description
62
- end
63
-
64
- def message
65
- output = "%s had a count difference of %i" % [@table_name, @delta]
66
- output = output + "; " + @description if !@description.empty?
67
- end
68
- end
69
- end
70
- end
@@ -1,54 +0,0 @@
1
- require 'integration_helper'
2
-
3
- require 'sq/dbsync/consistency_verifier'
4
- require 'sq/dbsync/static_table_plan'
5
- require 'sq/dbsync/table_registry'
6
-
7
- describe SQD::ConsistencyVerifier do
8
- let(:overlap) { SQD::LoadAction.overlap }
9
- let(:now) { Date.new(2012, 4, 4).to_time.utc }
10
- let(:source) { test_source(:source) }
11
- let(:target) { test_target }
12
- let(:tables) {[{
13
- table_name: :test_table,
14
- columns: [:id, :col1, :updated_at],
15
- consistency: true,
16
- source_db: source,
17
- indexes: {}
18
- }]}
19
- let(:registry) { SQD::TableRegistry.new(target) }
20
- let(:verifier) { SQD::ConsistencyVerifier.new(target, registry) }
21
-
22
- before do
23
- create_source_table_with(
24
- id: 1,
25
- col1: 'old record',
26
- created_at: now - overlap
27
- )
28
- setup_target_table(now)
29
- end
30
-
31
- it 'raises if counts do not match up' do
32
- error_string =
33
- "test_table had a count difference of 1; " +
34
- "source: #{source.name} (count: 1), " +
35
- "sink: #{target.name} (count: 0)"
36
-
37
- lambda {
38
- verifier.check_consistency!(tables)
39
- }.should raise_error(
40
- SQD::ConsistencyVerifier::ConsistencyError,
41
- error_string
42
- )
43
- end
44
-
45
- it 'uses last_row_at rather than last_synced_at' do
46
- registry.update(:test_table, now,
47
- last_row_at: now - 3
48
- )
49
-
50
- lambda {
51
- verifier.check_consistency!(tables)
52
- }.should_not raise_error(SQD::ConsistencyVerifier::ConsistencyError)
53
- end
54
- end