sq-dbsync 1.0.0

data/spec/integration/batch_load_action_spec.rb

@@ -0,0 +1,229 @@
+require 'integration_helper'
+
+require 'sq/dbsync/database/connection'
+require 'sq/dbsync/loggers'
+require 'sq/dbsync/batch_load_action'
+require 'sq/dbsync/table_registry'
+require 'sq/dbsync/static_table_plan'
+require 'sq/dbsync/all_tables_plan'
+
+describe SQD::BatchLoadAction do
+  let(:overlap) { described_class.overlap }
+  let!(:now)    { @now = Time.now.utc }
+  let(:last_synced_at) { now - 10 }
+  let(:target) { test_target }
+  let(:table_plan) {{
+    table_name: :test_table,
+    source_table_name: :test_table,
+    columns: [:id, :col1, :updated_at],
+    source_db: source,
+    indexes: index
+  }}
+  let(:index) {{
+    index_on_col1: { columns: [:col1], unique: false }
+  } }
+  let(:registry) { SQD::TableRegistry.new(target) }
+  let(:action) { SQD::BatchLoadAction.new(
+    target,
+    table_plan,
+    registry,
+    SQD::Loggers::Null.new,
+    ->{ @now }
+  ) }
+
+  shared_examples_for 'a batch load' do
+    before do
+      create_source_table_with(
+        id:         1,
+        col1:       'hello',
+        pii:        'don alias',
+        updated_at: now - 10
+      )
+
+      registry.ensure_storage_exists
+    end
+
+    describe ':all columns options' do
+      let(:table_plan) {{
+        table_name: :test_table,
+        source_table_name: :test_table,
+        columns: :all,
+        source_db: source,
+      }}
+
+      it 'copies all columns to target' do
+        action.call
+
+        target.hash_schema(:test_table).keys.should ==
+          source.hash_schema(:test_table).keys
+      end
+    end
+
+    it 'copies source tables to target with matching schemas' do
+      start_time = now.to_f
+
+      action.call
+
+      verify_schema
+      verify_data
+      verify_metadata(start_time)
+    end
+
+    it 'handles column that does not exist in source' do
+      source.alter_table :test_table do
+        drop_column :col1
+      end
+
+      table_plan[:indexes] = {}
+      action.call
+
+      target[:test_table].map {|x| x.values_at(:id)}.
+        should == [[1]]
+    end
+
+    it 'handles table that does not exist in source' do
+      source.drop_table :test_table
+
+      action.call
+
+      target.table_exists?(:test_table).should_not be
+    end
+
+    it 'ignores duplicates when loading data' do
+      source[:test_table].insert(id: 2, col1: 'hello')
+      source[:test_table].insert(id: 3, col1: 'hello')
+
+      table_plan[:indexes][:unique_index] = {columns: [:col1], unique: true}
+
+      action.call
+
+      target[:test_table].count.should == 1
+    end
+
+    it 'clears partial load if a new_ table already exists' do
+      setup_target_table(now)
+      target.switch_table(:new_test_table, :test_table)
+
+      source[:test_table].insert(
+        id: 7,
+        col1: 'old',
+        updated_at: now - 600
+      )
+
+      target[:new_test_table].insert(
+        id:         2,
+        col1:       'already loaded',
+        updated_at: now - 200
+      )
+
+      action.call
+
+      target[:test_table].all.map {|x| x[:col1] }.sort.should ==
+        ['hello', 'old'].sort
+    end
+
+    it 'catches up from last_row_at' do
+      action.do_prepare
+      action.extract_data
+      action.load_data
+
+      source[:test_table].insert(id: 2, col1: 'new', updated_at: now)
+
+      @now += 600
+
+      action.post_load
+
+      target[:test_table].all.map {|x| x[:col1] }.sort.should ==
+        ['hello', 'new'].sort
+    end
+
+    def test_tables
+      {
+        test_table: source,
+      }
+    end
+
+    def verify_schema
+      test_tables.each do |table_name, source_db|
+        target_table_name = table_name
+        target.tables.should include(target_table_name)
+        source_test_table_schema =
+          source_db.schema(table_name).map do |column, hash|
+            # Auto-increment is not copied, since it isn't relevant for
+            # replicated tables and would be more complicated to support.
+            # Primary key status is copied, however.
+            hash.delete(:auto_increment)
+            hash.delete(:ruby_default)
+            [column, hash]
+          end
+
+        extract_common_db_column_info = ->(e) {
+          [e[0], {
+            type: e[1][:type],
+            primary_key: e[1][:primary_key]
+          }]
+        }
+
+        source_test_table_schema = source_test_table_schema.map do |e|
+          extract_common_db_column_info.call(e)
+        end
+
+        target.schema(target_table_name).each do |column_arr|
+          column_arr = extract_common_db_column_info.call(column_arr)
+          source_test_table_schema.should include(column_arr)
+        end
+        target.indexes(target_table_name).should == index
+      end
+    end
+
+    def verify_data
+      test_tables.each do |table_name, _|
+        data = target[table_name].all
+        data.count.should == 1
+        data = data[0]
+        data.keys.length.should == 3
+        data[:id].should == 1
+        data[:col1].should == 'hello'
+        data[:updated_at].to_i.should == (now - 10).to_i
+      end
+    end
+
+    def verify_metadata(start_time)
+      test_tables.each do |table_name, _|
+        meta = registry.get(table_name)
+        meta[:last_synced_at].should_not be_nil
+        meta[:last_batch_synced_at].should_not be_nil
+        meta[:last_batch_synced_at].to_i.should == start_time.to_i
+        meta[:last_row_at].to_i.should == (now - 10).to_i
+      end
+    end
+  end
+
+  describe 'with MySQL source' do
+    let(:source) { test_source(:source) }
+
+    it_should_behave_like 'a batch load'
+
+    it 'loads records with bodgy timestamps' do
+      create_source_table_with(
+        id:         1,
+        col1:       'hello',
+        pii:        'don alias',
+        created_at: '0000-00-00 00:00:00',
+        updated_at: '0000-00-00 00:00:00'
+      )
+      registry.ensure_storage_exists
+
+      action.call
+
+      target[:test_table].count.should == 1
+    end
+  end
+
+  describe 'with PG source' do
+    let(:source) { test_source(:postgres) }
+
+    it_should_behave_like 'a batch load'
+  end
+
+end

data/spec/integration/consistency_verifier_spec.rb

@@ -0,0 +1,54 @@
+require 'integration_helper'
+
+require 'sq/dbsync/consistency_verifier'
+require 'sq/dbsync/static_table_plan'
+require 'sq/dbsync/table_registry'
+
+describe SQD::ConsistencyVerifier do
+  let(:overlap) { SQD::LoadAction.overlap }
+  let(:now)     { Date.new(2012, 4, 4).to_time.utc }
+  let(:source)  { test_source(:source) }
+  let(:target)  { test_target }
+  let(:tables) {[{
+    table_name: :test_table,
+    columns: [:id, :col1, :updated_at],
+    consistency: true,
+    source_db: source,
+    indexes: {}
+  }]}
+  let(:registry) { SQD::TableRegistry.new(target) }
+  let(:verifier) { SQD::ConsistencyVerifier.new(target, registry) }
+
+  before do
+    create_source_table_with(
+      id:         1,
+      col1:       'old record',
+      created_at: now - overlap
+    )
+    setup_target_table(now)
+  end
+
+  it 'raises if counts do not match up' do
+    error_string =
+      "test_table had a count difference of 1; " +
+      "source: #{source.name} (count: 1), " +
+      "sink: #{target.name} (count: 0)"
+
+    lambda {
+      verifier.check_consistency!(tables)
+    }.should raise_error(
+      SQD::ConsistencyVerifier::ConsistencyError,
+      error_string
+    )
+  end
+
+  it 'uses last_row_at rather than last_synced_at' do
+    registry.update(:test_table, now,
+      last_row_at: now - 3
+    )
+
+    lambda {
+      verifier.check_consistency!(tables)
+    }.should_not raise_error(SQD::ConsistencyVerifier::ConsistencyError)
+  end
+end

data/spec/integration/database_connection_spec.rb

@@ -0,0 +1,61 @@
+require 'integration_helper'
+
+require 'sq/dbsync/database/connection'
+
+shared_examples_for 'a decorated database adapter' do
+  let(:path)   { @file.path }
+
+  before { @file = Tempfile.new('bogus') }
+
+  describe '#extract_sql_to_file' do
+    it 'should raise when it fails' do
+      lambda {
+        db.extract_to_file('some_table', [], path)
+      }.should raise_error(SQD::Database::ExtractError)
+    end
+  end
+end
+
+describe SQD::Database::Postgres do
+  let(:source) { test_source(:postgres) }
+  let(:db) { SQD::Database::Postgres.new(source) }
+
+  it_should_behave_like 'a decorated database adapter'
+end
+
+describe SQD::Database::Mysql do
+  let(:source) { test_source(:source) }
+  let(:db)     { SQD::Database::Mysql.new(source) }
+
+  it_should_behave_like 'a decorated database adapter'
+
+  describe '#load_incrementally_from_file' do
+    let(:path)   { @file.path }
+
+    before { @file = Tempfile.new('bogus') }
+
+    def source_with_exception(exception_message)
+      source.stub(:run).and_raise(
+        Sequel::DatabaseError.new(exception_message)
+      )
+    end
+
+    it 're-raises deadlock related exceptions as TransientError' do
+      source_with_exception("Deadlock found when trying to get lock")
+      -> { db.load_incrementally_from_file('bogus', ['bogus'], path) }.
+        should raise_error(SQD::Database::TransientError)
+    end
+
+    it 're-raises lock wait timeout exceptions as TransientError' do
+      source_with_exception("Lock wait timeout exceeded")
+      -> { db.load_incrementally_from_file('bogus', ['bogus'], path) }.
+        should raise_error(SQD::Database::TransientError)
+    end
+
+    it 'does not translate unknown errors' do
+      source_with_exception("Unknown")
+      -> { db.load_incrementally_from_file('bogus', ['bogus'], path) }.
+        should raise_error(Sequel::DatabaseError)
+    end
+  end
+end

data/spec/integration/incremental_load_action_spec.rb

@@ -0,0 +1,196 @@
+require 'integration_helper'
+
+require 'sq/dbsync/database/connection'
+require 'sq/dbsync/incremental_load_action'
+require 'sq/dbsync/table_registry'
+require 'sq/dbsync/loggers'
+
+describe SQD::IncrementalLoadAction do
+  let(:overlap)        { described_class.overlap }
+  let(:now)            { Date.new(2012, 4, 4).to_time.utc }
+  let(:last_synced_at) { now - 10 }
+  let(:source)         { test_source(:source) }
+  let(:target) { test_target }
+  let(:table_plan) {{
+    table_name: :test_table,
+    source_table_name: :test_table,
+    columns: [:id, :col1, :updated_at],
+    source_db: source,
+    indexes: {}
+  }}
+  let(:registry) { SQD::TableRegistry.new(target) }
+  let(:action) { SQD::IncrementalLoadAction.new(
+    target,
+    table_plan,
+    registry,
+    SQD::Loggers::Null.new,
+    ->{ now }
+  )}
+
+  shared_examples_for 'an incremental load' do
+    before :each do
+      create_source_table_with({
+        id:         1,
+        col1:       'old record',
+        updated_at:  last_synced_at - overlap - 1,
+        imported_at: last_synced_at - overlap - 1,
+      }, {
+        id:         2,
+        col1:       'new record',
+        updated_at:  last_synced_at - overlap + 1,
+        imported_at: last_synced_at - overlap + 1,
+      })
+
+      setup_target_table(last_synced_at)
+    end
+
+    describe ':all columns options' do
+      let(:table_plan) {{
+        table_name: :test_table,
+        source_table_name: :test_table,
+        columns: :all,
+        source_db: source,
+      }}
+
+      it 'copies all columns to target' do
+        action.call
+
+        target[:test_table].map { |row| row.values_at(:id, :col1) }.
+          should == [[2, 'new record']]
+      end
+    end
+
+
+    it 'copies null data to the target' do
+      source[:test_table].update(col1: nil)
+
+      action.call
+
+      target[:test_table].map {|row| row[:col1] }.
+        should == [nil]
+    end
+
+    it 'copies source data to the target since the last synced row' do
+      registry.update(:test_table, last_synced_at,
+        last_synced_at: last_synced_at + 2,
+        last_row_at:    last_synced_at
+      )
+
+      action.call
+
+      target[:test_table].map { |row| row.values_at(:id, :col1) }.
+        should == [[2, 'new record']]
+
+      metadata = registry.get(:test_table)
+      metadata[:last_synced_at].to_i.should == now.to_i
+      metadata[:last_row_at].to_i.should == (last_synced_at - overlap + 1).to_i
+    end
+
+    it 'should replace any records found within the overlap' do
+      target[:test_table].insert(
+        id:   2,
+        col1: 'old record'
+      )
+
+      action.call
+
+      target[:test_table].map { |row| row.values_at(:id, :col1) }.
+        should == [[2, 'new record']]
+    end
+
+    it 'should handle table that does not exist in source but does in target' do
+      source.drop_table :test_table
+
+      action.call
+
+      registry.get(:test_table).should_not be_nil
+      target.table_exists?(:test_table).should be
+    end
+
+    it 'should handle table that does not exist in target but in source' do
+      target.drop_table :test_table
+      registry.delete(:test_table)
+
+      action.call
+
+      target.table_exists?(:test_table).should_not be
+    end
+
+    it 'should handle column that does not exist in source' do
+      table_plan[:columns] += [:bogus]
+
+      action.call
+
+      target[:test_table].map { |row| row.values_at(:col1) }.
+        should == [['new record']]
+    end
+
+
+    describe 'with custom timestamp' do
+      let(:table_plan) {{
+        table_name: :test_table,
+        source_table_name: :test_table,
+        columns: [:id, :col1, :imported_at],
+        timestamp: :imported_at,
+        source_db: source,
+        indexes: {}
+      }}
+
+      it 'copies source data to the target since the last synced row' do
+        registry.update(:test_table, last_synced_at,
+          last_synced_at: last_synced_at + 2,
+          last_row_at:    last_synced_at
+        )
+
+        action.call
+
+        target[:test_table].map { |row| row.values_at(:id, :col1) }.
+          should == [[2, 'new record']]
+
+        metadata = registry.get(:test_table)
+        metadata[:last_synced_at].to_i.should == now.to_i
+        metadata[:last_row_at].to_i.should == (last_synced_at-overlap+1).to_i
+      end
+    end
+
+    context 'always_sync = true' do
+      it 'handles table that does not exist in source but does in target' do
+        source.drop_table :test_table
+        table_plan[:always_sync] = true
+
+        action.call
+
+        registry.get(:test_table).should be_nil
+        target.table_exists?(:test_table).should_not be
+      end
+
+      it 'handles table that does not exist in target with always_sync' do
+        table_plan[:always_sync] = true
+        target.drop_table :test_table
+        registry.delete(:test_table)
+
+        action.call
+
+        target[:test_table].map { |row| row.values_at(:id, :col1) }.
+          should == [[1, 'old record'], [2, 'new record']]
+
+        metadata = registry.get(:test_table)
+        metadata[:last_synced_at].to_i.should == now.to_i
+        metadata[:last_row_at].to_i.should ==
+          (last_synced_at - overlap + 1).to_i
+      end
+    end
+  end
+
+  describe 'with MySQL source' do
+    let(:source) { test_source(:source) }
+
+    it_should_behave_like 'an incremental load'
+  end
+
+  describe 'with PG source' do
+    let(:source) { test_source(:postgres) }
+
+    it_should_behave_like 'an incremental load'
+  end
+end

data/spec/integration/manager_spec.rb

@@ -0,0 +1,109 @@
+require 'integration_helper'
+
+require 'sq/dbsync/manager'
+require 'sq/dbsync/static_table_plan'
+require 'sq/dbsync/loggers'
+
+describe SQD::Manager do
+  let(:now) { Time.now.utc }
+  let(:config) {{
+    sources: TEST_SOURCES,
+    target:  TEST_TARGET,
+    logger:  SQD::Loggers::Null.new,
+    clock:   ->{ now }
+  }}
+  let(:manager)    { SQD::Manager.new(config, [
+    [SQD::StaticTablePlan.new(plan), :source]
+  ]) }
+  let(:source)     { manager.sources.fetch(:source) }
+  let(:alt_source) { manager.sources.fetch(:alt_source) }
+  let(:target)     { manager.target }
+  let(:registry)   { SQD::TableRegistry.new(target) }
+  let(:plan) {[{
+    table_name: :test_table,
+    columns:    [:id, :updated_at]
+  }] }
+  let(:now) { Time.now.utc }
+
+  before do
+    create_source_table_with(
+      id:         1,
+      col1:       'hello',
+      pii:        'don alias',
+      updated_at: now - 10
+    )
+  end
+
+  it 'handles duplicate table names by selecting the first one' do
+    create_source_table_with(alt_source, {
+      col1:       'hello',
+      pii:        'don alias'
+    }, {
+      col1:       'hello again',
+      pii:        'don alias'
+    })
+
+    manager = SQD::Manager.new(config,
+      [
+        [SQD::StaticTablePlan.new(plan), :source],
+        [SQD::StaticTablePlan.new(plan), :alt_source]
+      ]
+    )
+    manager.batch_nonactive
+
+    target[:test_table].count.should == 1
+  end
+
+  it 'does not purge old tables from the database' do
+    setup_target_table(now)
+
+    manager = SQD::Manager.new(config, [])
+    manager.batch_nonactive
+    target.table_exists?(:test_table).should be
+  end
+
+  it 'removes old tables from the registry' do
+    setup_target_table(now)
+
+    manager = SQD::Manager.new(config, [])
+    manager.increment_checkpoint
+    registry.get(:test_table).should_not be
+
+    # Dropping tables must be done manually
+    target.table_exists?(:test_table).should be
+  end
+
+  it 'only batch loads the given tables, even when batch load disabled' do
+    plan[0][:batch_load] = false
+
+    manager = SQD::Manager.new(config, [
+      [SQD:: StaticTablePlan.new(plan), :source],
+    ])
+    manager.batch_nonactive([:bogus])
+    target.table_exists?(:test_table).should_not be
+
+    manager.batch_nonactive([:test_table])
+    target.table_exists?(:test_table).should be
+  end
+
+  it 'does not purge tables excluded from batch load' do
+    plan[0][:batch_load] = false
+    setup_target_table(now)
+
+    manager = SQD::Manager.new(config, [
+      [SQD::StaticTablePlan.new(plan), :source],
+    ])
+
+    manager.batch_nonactive([:bogus])
+    target.table_exists?(:test_table).should be
+  end
+
+  it 'does not purge old tables when doing a partial load' do
+    setup_target_table(now)
+
+    manager = SQD::Manager.new(config, [])
+    manager.batch_nonactive([:bogus])
+
+    target.table_exists?(:test_table).should be
+  end
+end