sq-dbsync 1.0.0

data/lib/sq/dbsync/schema_maker.rb

@@ -0,0 +1,87 @@
+module Sq::Dbsync
+  # Service class for mapping table plans to DDL.
+  class SchemaMaker
+
+    # Creates a table in the database based on the given plan. If the table
+    # already exists, it will be recreated and any data will be lost.
+    def self.create_table(db, table_plan)
+      new(db, table_plan).run
+    end
+
+    def run
+      table_plan = @table_plan
+
+      db.create_table!(table_name,
+         engine: 'InnoDB',
+         charset: table_plan.charset || 'utf8'
+      ) do
+        extend Helpers
+
+        add_columns!(table_plan)
+        add_indexes!(table_plan)
+        add_primary_key!(table_plan)
+      end
+    end
+
+    protected
+
+    def initialize(db, table_plan)
+      @db = db
+      @table_plan = table_plan
+    end
+
+    attr_reader :db, :table_plan
+
+    def table_name
+      table_plan.prefixed_table_name
+    end
+
+    module Helpers
+      def add_columns!(table_plan)
+        columns  = table_plan.columns
+        db_types = table_plan.db_types || {}
+        schema   = table_plan.schema
+        columns.each do |column_name|
+          add_column(column_name, db_types, schema)
+        end
+      end
+
+      def add_indexes!(table_plan)
+        indexes = table_plan.indexes || []
+
+        indexes.each do |index_name, index_metadata|
+          index_columns = index_metadata[:columns]
+          unique        = index_metadata[:unique] || false
+          send(:index, index_columns, name: index_name, unique: unique)
+        end
+      end
+
+      def add_primary_key!(table_plan)
+        columns = if table_plan.primary_key
+          table_plan.primary_key
+        else
+          table_plan.schema.select {|col, schema|
+            schema[:primary_key]
+          }.map(&:first)
+        end
+
+        columns = [:id] if columns.empty?
+
+        primary_key(columns)
+      end
+
+
+      def add_column(column_name, db_types, schema)
+        db_type = db_types[column_name] || [schema[column_name][:db_type]]
+
+        extra = if db_type[0] == :enum
+          { elements: db_type[1] }
+        else
+          {}
+        end
+
+        send db_type[0], column_name, extra
+      end
+    end
+  end
+end

data/lib/sq/dbsync/static_table_plan.rb

@@ -0,0 +1,42 @@
+# Generates a plan given a static "spec" of tables, columns, optional data
+# types, and indexes. This is useful for partial replication of tables, for
+# instance when some columns contain sensitive or large information that is not
+# to be replicated.
+#
+# Simple Example:
+#
+#  spec = [{
+#    table_name: :users,
+#    columns: [:id, :updated_at],
+#    indexes: {
+#      index_users_on_updated_at: {columns: [:updated_at]}
+#    }
+#  }]
+class Sq::Dbsync::StaticTablePlan
+  def initialize(spec)
+    @spec = format_spec(spec)
+  end
+
+  def tables(source)
+    deep_clone(@spec).map do |tplan|
+      tplan.update(source_db: source)
+    end
+  end
+
+  def deep_clone(object)
+    Marshal.load(Marshal.dump(object))
+  end
+
+  def format_spec(spec)
+    # Support copying from different relations of a Postgres DB, but to the
+    # same target database in MySQL.
+    spec.map do |table_def|
+      unless table_def[:source_table_name]
+        table_def[:source_table_name] = table_def[:table_name]
+        table_def[:table_name] = table_def[:source_table_name].
+          to_s.gsub('__', '_').to_sym
+      end
+      table_def
+    end
+  end
+end

data/lib/sq/dbsync/table_registry.rb

@@ -0,0 +1,75 @@
+module Sq::Dbsync
+
+  # A key-value abstraction that is used to store metadata about loads on a
+  # per-table basis.
+  class TableRegistry
+    def initialize(db)
+      @db    = db
+      @table = db[table_name]
+    end
+
+    def delete(key)
+      table.filter(table_name: key.to_s).delete
+    end
+
+    # Set a value if an existing value does not already exist.
+    def set(key, values)
+      unless exists?(key)
+        table.insert(values.merge(table_name: key.to_s))
+      end
+    end
+
+    # Set a value, overriding any existing.
+    def set!(key, values)
+      db.transaction do
+        delete(key)
+        set(key, values)
+      end
+    end
+
+    def update(key, lock, values)
+      table.
+        filter(
+          table_name:           key.to_s,
+          last_batch_synced_at: lock
+        ).
+        update(values)
+    end
+
+    def get(key)
+      table.
+        select(:last_synced_at, :last_row_at, :last_batch_synced_at).
+        filter(table_name: key.to_s).
+        first
+    end
+
+    def purge_except(keys)
+      query = table
+      if keys.any?
+        query = query.where('table_name NOT IN ?', keys.map(&:to_s))
+      end
+      query.delete
+    end
+
+    def ensure_storage_exists
+      db.create_table?(table_name, charset: 'utf8') do
+        String   :table_name, primary_key: true
+        DateTime :last_synced_at
+        DateTime :last_batch_synced_at
+        DateTime :last_row_at
+      end
+    end
+
+    private
+
+    attr_reader :table, :db
+
+    def table_name
+      :meta_last_sync_times
+    end
+
+    def exists?(key)
+      table.filter(table_name: key.to_s).count > 0
+    end
+  end
+end

data/lib/sq/dbsync/tempfile_factory.rb

@@ -0,0 +1,41 @@
+require 'tempfile'
+
+module Sq::Dbsync
+  # Provide extra functionality on top of the standard tempfile API.
+  class TempfileFactory
+
+    # ENV['TMPDIR'] is explicitly referenced here, since a change to JRuby in
+    # 1.7.0 makes `Dir.tmpdir` preference non-world writable directories first,
+    # of which `.` is a member. This makes it impossible to configure a world
+    # writable directory solely via the environment.
+    def self.make(name)
+      Tempfile.new(name, ENV['TMPDIR'] || Dir.tmpdir)
+    end
+
+    def self.make_with_content(name, content)
+      file = make(name)
+      file.write(content)
+      file.flush
+      file
+    end
+
+    # A world writable file is necessary if it is being used as a communication
+    # mechanism with other processes (such as MySQL `LOAD DATA INFILE`).
+    def self.make_world_writable(name)
+      file = make(name)
+      file.chmod(0666)
+      file
+    end
+
+    def self.split(file, n, logger, &block)
+      `split -l #{n} #{file.path} #{file.path}.`
+      files = Dir[file.path + '.*']
+      files.each_with_index do |tempfile, i|
+        logger.log("Loading chunk #{i+1}/#{files.length}")
+        block.call(tempfile)
+        FileUtils.rm(tempfile)
+      end
+    end
+
+  end
+end

data/lib/sq/dbsync/version.rb

@@ -0,0 +1,5 @@
+module Sq
+  module Dbsync
+    VERSION = '1.0.0'
+  end
+end

data/lib/sq/dbsync.rb

@@ -0,0 +1,9 @@
+module Sq
+  module Dbsync
+  end
+end
+
+require 'sq/dbsync/manager'
+require 'sq/dbsync/static_table_plan'
+require 'sq/dbsync/all_tables_plan'
+require 'sq/dbsync/loggers'

data/spec/acceptance/loading_spec.rb

@@ -0,0 +1,237 @@
+require 'acceptance_helper'
+
+require 'sq/dbsync'
+
+describe 'Syncing source databases to a target' do
+  let(:config) {{
+    sources: TEST_SOURCES,
+    target:  TEST_TARGET,
+    logger:  SQD::Loggers::Composite.new([logger]),
+    clock:   ->{ @now }
+  }}
+  let(:logger)  { SQD::Loggers::NullWithCallbacks.new }
+  let(:manager) {
+    SQD::Manager.new(config, [[SQD::StaticTablePlan.new(plan), :source]])
+  }
+  let(:source)     { manager.sources.fetch(:source) }
+  let(:alt_source) { manager.sources.fetch(:alt_source) }
+  let(:target) { manager.target }
+  let(:plan) {[{
+    table_name: :test_table,
+    source_table_name: :test_table,
+    refresh_recent: true,
+    columns:    [:id, :updated_at]
+  }] }
+
+  MINUTE = 60
+  WEEK = MINUTE * 60 * 24 * 7
+
+  before do
+    @now = Time.now.utc
+
+    setup_source_table
+  end
+
+  context 'batch loads' do
+    it 'batch loads the nonactive database and switches it to active' do
+      row = source[:test_table].insert(updated_at: @now)
+
+      manager.batch_nonactive
+
+      target[:test_table].map {|x| x[:id] }.should include(row)
+      target[:meta_last_sync_times].count.should == 1
+    end
+
+    it 'catches up missed rows with an incremental update' do
+      new_row_id = nil
+
+      logger.callbacks = {
+        'batch.load.test_table' => ->{
+          @now += SQD::BatchLoadAction::MAX_LAG
+          new_row_id = source[:test_table].insert(updated_at: @now - 1)
+        }
+      }
+
+      row = source[:test_table].insert(updated_at: @now)
+
+      manager.batch_nonactive
+
+      target[:test_table].map {|x| x[:id] }.should include(row)
+      target[:test_table].map {|x| x[:id] }.should include(new_row_id)
+    end
+
+    it 'loads from two distinct sources' do
+      manager = SQD::Manager.new(config, [
+        [SQD::StaticTablePlan.new(plan), :source],
+        [SQD::AllTablesPlan.new,         :alt_source]
+      ])
+
+      row     =     source[:test_table    ].insert(updated_at: @now)
+      alt_row = alt_source[:alt_test_table].insert(updated_at: @now)
+
+      manager.batch_nonactive
+
+      target[:test_table     ].map {|x| x[:id] }.should include(row)
+      target[:alt_test_table ].map {|x| x[:id] }.should include(alt_row)
+    end
+  end
+
+  context 'refresh recent load' do
+    before do
+      manager.batch_nonactive
+    end
+
+    it 'reloads all recent data' do
+      deleted = 1
+      to_keep = 2
+      new_row = 3
+
+      target[:test_table].insert(id: deleted, updated_at: @now)
+      target[:test_table].insert(id: to_keep, updated_at: @now - WEEK)
+      source[:test_table  ].insert(id: new_row, updated_at: @now - MINUTE)
+
+      x = target[:test_table].map {|x| x[:id] }
+
+      manager.refresh_recent
+
+      target[:test_table].map {|x| x[:id] }.should include(new_row)
+      target[:test_table].map {|x| x[:id] }.should include(to_keep)
+      target[:test_table].map {|x| x[:id] }.should_not include(deleted)
+    end
+
+    describe 'when a column is provided' do
+      let(:plan) {[{
+        table_name: :test_table,
+        refresh_recent: :reporting_date,
+        source_table_name: :test_table,
+        columns:    [:id, :updated_at, :reporting_date]
+      }] }
+
+      it 'adds an extra filter when a column is provided' do
+        deleted = 1
+        to_keep = 2
+        new_row = 3
+        to_keep2 = 4
+
+        target[:test_table].insert(
+          id:             deleted,
+          updated_at:     @now,
+          reporting_date: @now
+        )
+        target[:test_table].insert(
+          id:             to_keep,
+          updated_at:     @now - WEEK,
+          reporting_date: @now
+        )
+        target[:test_table].insert(
+          id:             to_keep2,
+          updated_at:     @now,
+          reporting_date: @now - WEEK
+        )
+        source[:test_table].insert(
+          id:         new_row,
+          updated_at: @now - MINUTE
+        )
+
+        x = target[:test_table].map {|x| x[:id] }
+        manager.refresh_recent
+
+        target[:test_table].map {|x| x[:id] }.should include(new_row)
+        target[:test_table].map {|x| x[:id] }.should include(to_keep)
+        target[:test_table].map {|x| x[:id] }.should include(to_keep2)
+        target[:test_table].map {|x| x[:id] }.should_not include(deleted)
+      end
+    end
+  end
+
+  context 'incremental loads' do
+    let(:worker) { @worker }
+    before do
+      manager.batch_nonactive
+      @worker = background do
+        manager.incremental
+      end
+    end
+
+    after do
+      manager.stop!
+      worker.wait_until_finished!
+    end
+
+    def background(&block)
+      worker = Thread.new(&block)
+      worker.instance_eval do
+        def wait_until_finished!; join rescue nil; end
+      end
+      worker
+    end
+
+    it 'updates the active database' do
+      2.times do |t|
+        insert_and_verify_row
+      end
+    end
+
+    it 'does not continually retry consistent failures' do
+      manager.
+        stub!(:incremental_once).
+        and_raise(SQD::Database::ExtractError.new)
+
+      ->{
+        worker.value
+      }.should raise_error(SQD::Database::ExtractError)
+    end
+
+    context 'with an all tables plan' do
+      let(:manager) { SQD::Manager.new(config, [[
+        SQD::AllTablesPlan.new, :source
+      ]]) }
+
+      it 'adds a database that is newly in the source but not in the target' do
+        source.create_table! :new_table do
+          primary_key  :id
+          DateTime :updated_at
+        end
+
+        insert_and_verify_row(:new_table)
+      end
+    end
+  end
+
+  def insert_and_verify_row(table = :test_table)
+    row = source[table].insert(updated_at: Time.now.utc)
+    spin_for(1) do
+      target.table_exists?(table) &&
+        target[table].map {|x| x[:id] }.include?(row)
+    end
+  end
+
+  def setup_source_table
+    source.create_table! :test_table do
+      primary_key  :id
+      DateTime :reporting_date
+      DateTime :updated_at
+    end
+    source.create_table! :test_table_2 do
+      primary_key  :id
+      DateTime :updated_at
+    end
+    alt_source.create_table! :alt_test_table do
+      primary_key  :id
+      DateTime :updated_at
+    end
+  end
+
+  def spin_for(timeout = 1)
+    result     = false
+    start_time = Time.now
+
+    while start_time + timeout > Time.now
+      result = yield
+      break if result
+      sleep 0.001
+    end
+
+    raise "timed out" unless result
+  end
+end

data/spec/acceptance_helper.rb

@@ -0,0 +1,2 @@
+require 'spec_helper'
+require 'database_helper'

data/spec/database_helper.rb

@@ -0,0 +1,86 @@
+require 'sq/dbsync/database/connection'
+
+def db_options(opts)
+  opts = {
+    user: 'root',
+    host: 'localhost',
+    brand: 'mysql',
+    port: opts[:brand] == 'postgresql' ? 5432 : 3306
+  }.merge(opts)
+
+  if RUBY_PLATFORM == 'java'
+    opts.merge(
+      adapter: "jdbc",
+      uri: begin
+        base = 'jdbc:%s://%s:%i/%s?user=%s' % [
+          opts.fetch(:brand),
+          opts.fetch(:host),
+          opts.fetch(:port),
+          opts.fetch(:database),
+          opts.fetch(:user)
+        ]
+        if opts[:password]
+          base += '&password=%s' % opts[:password]
+        end
+        base
+      end
+    )
+  else
+    {
+      adapter: opts[:brand] == 'postgresql' ? 'postgres' : 'mysql2',
+    }.merge(opts)
+  end
+end
+
+TEST_SOURCES = {
+  source:     db_options(database: 'sq_dbsync_test_source'),
+  alt_source: db_options(database: 'sq_dbsync_test_source_alt'),
+  postgres:   db_options(
+    user:     `whoami`.chomp,
+    brand:    'postgresql',
+    host:     'localhost',
+    database: 'sq_dbsync_pg_test_source'
+  )
+}
+TEST_TARGET = db_options(database: 'sq_dbsync_test_target')
+
+$target = nil
+def test_target
+  $target ||= SQD::Database::Connection.create(TEST_TARGET)
+end
+
+$sources = {}
+def test_source(name)
+  $sources[name] ||= SQD::Database::Connection.create(TEST_SOURCES.fetch(name))
+end
+
+RSpec.configure do |config|
+  db_specs_only = {
+    example_group: {file_path: /spec\/(integration|acceptance)/}
+  }
+
+  config.before(:suite, db_specs_only) do
+    (TEST_SOURCES.values + [TEST_TARGET]).each do |opts|
+      db = opts.fetch(:database)
+
+      case opts.fetch(:brand)
+      when 'mysql'
+        `mysql -u root -e "drop database if exists #{db}"`
+        `mysql -u root -e "create database #{db}"`
+      when 'postgresql'
+        `dropdb #{db}`
+        `createdb #{db}`
+      else
+        raise "Unknown database: #{opts.inspect}"
+      end
+    end
+  end
+
+  config.before(:each, db_specs_only) do
+    (TEST_SOURCES.keys.map {|x| test_source(x) } + [test_target]).each do |db|
+      db.tables.each do |name|
+        db.drop_table(name)
+      end
+    end
+  end
+end

data/spec/integration/all_tables_plan_spec.rb

@@ -0,0 +1,36 @@
+require 'integration_helper'
+
+require 'sq/dbsync/all_tables_plan'
+require 'sq/dbsync/database/connection'
+
+describe SQD::AllTablesPlan do
+  let(:source) { test_source(:source) }
+
+  it 'does not return tables with no PK' do
+    source.create_table :test_table do
+      Integer :col1
+      DateTime :updated_at
+    end
+
+    SQD::AllTablesPlan.new.tables(source).should == []
+  end
+
+  it 'does not return tables with no timestamps' do
+    source.create_table :test_table do
+      primary_key :id
+    end
+
+    SQD::AllTablesPlan.new.tables(source).should == []
+  end
+
+  it 'handles table dropped after select' do
+    source.create_table :test_table do
+      primary_key :id
+      DateTime :updated_at
+    end
+
+    source.should_receive(:schema).and_raise(Sequel::DatabaseError)
+
+    SQD::AllTablesPlan.new.tables(source).should == []
+  end
+end