sq-dbsync 1.0.0 → 1.0.1

data/README.md

@@ -18,7 +18,13 @@ but `sq-dbsync` can tide you over until you get there.
 Usage
 -----
 
+```
+gem install sq-dbsync
+```
+
 ``` Ruby
+require 'sq/dbsync'
+
 include Sq::Dbsync
 
 # Config will typically differ per environment.

data/lib/sq/dbsync/manager.rb

@@ -4,7 +4,6 @@ require 'sq/dbsync/incremental_load_action'
 require 'sq/dbsync/refresh_recent_load_action'
 require 'sq/dbsync/pipeline'
 require 'sq/dbsync/table_registry'
-require 'sq/dbsync/consistency_verifier'
 require 'sq/dbsync/database/connection'
 require 'sq/dbsync/error_handler'
 
@@ -98,8 +97,6 @@ class Sq::Dbsync::Manager
     # No need to do this every cycle, 100 is chosen to be as good as any
     # other number. It should run on the very first cycle however so that
     # our specs will cover it.
-    verifier.check_consistency!(tables_to_load)
-
     purge_registry
   end
 
@@ -215,10 +212,6 @@ class Sq::Dbsync::Manager
     TableRegistry.new(target)
   end
 
-  def verifier
-    @verifier ||= ConsistencyVerifier.new(target, registry)
-  end
-
   def logger
     config[:logger]
   end

data/lib/sq/dbsync/version.rb

@@ -1,5 +1,5 @@
 module Sq
   module Dbsync
-    VERSION = '1.0.0'
+    VERSION = '1.0.1'
   end
 end

data/sq-dbsync.gemspec

@@ -2,8 +2,8 @@
 require File.expand_path('../lib/sq/dbsync/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Xavier Shay"]
-  gem.email         = ["xavier@squareup.com"]
+  gem.authors       = ["Xavier Shay", "Damon McCormick"]
+  gem.email         = ["xavier@squareup.com", "damon@squaerup.com"]
   gem.description   =
     %q{Column based, timestamp replication of MySQL and Postgres databases.}
   gem.summary       = %q{

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: sq-dbsync
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
 - Xavier Shay
+- Damon McCormick
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-23 00:00:00.000000000 Z
+date: 2013-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -94,6 +95,7 @@ dependencies:
 description: Column based, timestamp replication of MySQL and Postgres databases.
 email:
 - xavier@squareup.com
+- damon@squaerup.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -103,7 +105,6 @@ files:
 - spec/database_helper.rb
 - spec/integration/all_tables_plan_spec.rb
 - spec/integration/batch_load_action_spec.rb
-- spec/integration/consistency_verifier_spec.rb
 - spec/integration/database_connection_spec.rb
 - spec/integration/incremental_load_action_spec.rb
 - spec/integration/manager_spec.rb
@@ -118,7 +119,6 @@ files:
 - lib/sq/dbsync/all_tables_plan.rb
 - lib/sq/dbsync/batch_load_action.rb
 - lib/sq/dbsync/config.rb
-- lib/sq/dbsync/consistency_verifier.rb
 - lib/sq/dbsync/database/common.rb
 - lib/sq/dbsync/database/connection.rb
 - lib/sq/dbsync/database/mysql.rb
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Column based, timestamp replication of MySQL and Postgres databases. Uses
@@ -174,7 +174,6 @@ test_files:
 - spec/database_helper.rb
 - spec/integration/all_tables_plan_spec.rb
 - spec/integration/batch_load_action_spec.rb
-- spec/integration/consistency_verifier_spec.rb
 - spec/integration/database_connection_spec.rb
 - spec/integration/incremental_load_action_spec.rb
 - spec/integration/manager_spec.rb

data/lib/sq/dbsync/consistency_verifier.rb

@@ -1,70 +0,0 @@
-require 'sq/dbsync/load_action' # For overlap, not ideal
-
-module Sq::Dbsync
-
-  # Performs a cheap check to verify that the number of records present for a
-  # recent time slice are the same across source and target tables.
-  #
-  # This checks consistency on the current tables, not the new_ set.
-  class ConsistencyVerifier
-    def initialize(target, registry)
-      @target   = target
-      @registry = registry
-    end
-
-    def check_consistency!(tables)
-      tables.each do |tplan|
-        next unless tplan[:consistency]
-        verify_consistency!(tplan)
-      end
-    end
-
-    def verify_consistency!(tplan)
-      last_row_at = registry.get(tplan[:table_name])[:last_row_at]
-      return unless last_row_at
-
-      now = registry.get(tplan[:table_name])[:last_row_at] - LoadAction.overlap
-
-      counts = [
-        tplan[:source_db],
-        target
-      ].map do |x|
-        x.consistency_check(tplan[:table_name], now)
-      end
-
-      delta = counts.reduce(:-)
-
-      unless delta == 0
-        raise ConsistencyError.new(
-          tplan[:table_name],
-          delta,
-          "source: #{tplan[:source_db].name} (count: #{counts[0]}), " +
-          "sink: #{target.name} (count: #{counts[1]})"
-        )
-      end
-    end
-
-    attr_reader :target, :registry
-
-    # Used to signal an observed error in the number of records between source
-    # and target tables. There are no current known situations in which this
-    # occurs, though in the past buggy handling of replication lag was normally
-    # the culprit.
-    #
-    # If it does occur, a good first response is to set `last_sync_time` to the
-    # last batch time (usually within 24 hours) which will force the
-    # incremental load to reconsider all recent records.
-    class ConsistencyError < RuntimeError
-      def initialize(table_name, delta, description="")
-        @table_name  = table_name
-        @delta       = delta
-        @description = description
-      end
-
-      def message
-        output = "%s had a count difference of %i" % [@table_name, @delta]
-        output = output + "; " + @description if !@description.empty?
-      end
-    end
-  end
-end

data/spec/integration/consistency_verifier_spec.rb

@@ -1,54 +0,0 @@
-require 'integration_helper'
-
-require 'sq/dbsync/consistency_verifier'
-require 'sq/dbsync/static_table_plan'
-require 'sq/dbsync/table_registry'
-
-describe SQD::ConsistencyVerifier do
-  let(:overlap) { SQD::LoadAction.overlap }
-  let(:now)     { Date.new(2012, 4, 4).to_time.utc }
-  let(:source)  { test_source(:source) }
-  let(:target)  { test_target }
-  let(:tables) {[{
-    table_name: :test_table,
-    columns: [:id, :col1, :updated_at],
-    consistency: true,
-    source_db: source,
-    indexes: {}
-  }]}
-  let(:registry) { SQD::TableRegistry.new(target) }
-  let(:verifier) { SQD::ConsistencyVerifier.new(target, registry) }
-
-  before do
-    create_source_table_with(
-      id:         1,
-      col1:       'old record',
-      created_at: now - overlap
-    )
-    setup_target_table(now)
-  end
-
-  it 'raises if counts do not match up' do
-    error_string =
-      "test_table had a count difference of 1; " +
-      "source: #{source.name} (count: 1), " +
-      "sink: #{target.name} (count: 0)"
-
-    lambda {
-      verifier.check_consistency!(tables)
-    }.should raise_error(
-      SQD::ConsistencyVerifier::ConsistencyError,
-      error_string
-    )
-  end
-
-  it 'uses last_row_at rather than last_synced_at' do
-    registry.update(:test_table, now,
-      last_row_at: now - 3
-    )
-
-    lambda {
-      verifier.check_consistency!(tables)
-    }.should_not raise_error(SQD::ConsistencyVerifier::ConsistencyError)
-  end
-end