sq-dbsync 1.0.0

data/spec/integration/schema_maker_spec.rb

@@ -0,0 +1,119 @@
+require 'integration_helper'
+
+require 'sq/dbsync/schema_maker'
+require 'ostruct'
+
+describe SQD::SchemaMaker do
+  let(:target) { test_target }
+
+  it 'creates a table with a compound index' do
+    index = {
+        index_on_col1: { columns: [:col1, :col2], unique: false }
+    }
+
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:col1, :col2],
+      indexes: index,
+      schema: {
+        col1: { db_type: 'varchar(255)', primary_key: true },
+        col2: { db_type: 'varchar(255)', primary_key: false }
+      }
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.indexes(:test_table).should == index
+  end
+
+  it 'defaults primary key to id' do
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:id],
+      schema: { id: {db_type: 'varchar(255)', primary_key: false }}
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.schema(:test_table)[0][1][:primary_key].should == true
+  end
+
+  it 'allows primary key override' do
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:id, :col1],
+      primary_key: [:id, :col1],
+      schema: {
+        id:   {db_type: 'varchar(255)', primary_key: false },
+        col1: {db_type: 'varchar(255)', primary_key: false }
+      }
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.schema(:test_table)[0][1][:primary_key].should == true
+    target.schema(:test_table)[1][1][:primary_key].should == true
+  end
+
+
+  it 'creates a table with an enum column' do
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:col1],
+      db_types: {:col1 => [:enum, %w(a b)]},
+      schema: { col1: { primary_key: true }}
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.schema(:test_table)[0][1][:db_type].should == "enum('a','b')"
+  end
+
+  it 'creates a table with non-id primary key' do
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:col1],
+      schema: { col1: { db_type: 'varchar(255)', primary_key: true }}
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.schema(:test_table)[0][1][:primary_key].should == true
+  end
+
+  it 'creates a table with a not-null column' do
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:col1],
+      db_types: {:col1 => ['int(1) not null']},
+      schema: { col1: { primary_key: true }}
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.schema(:test_table)[0][1][:allow_null].should == false
+  end
+
+  it 'creates a table with a composite primary key' do
+    plan = {
+      prefixed_table_name: :test_table,
+      table_name: :test_table,
+      columns: [:a, :b],
+      schema: {
+        a: { db_type: 'int', primary_key: true },
+        b: { db_type: 'int', primary_key: true }
+      }
+    }
+
+    described_class.create_table(target, OpenStruct.new(plan))
+
+    target.schema(:test_table)[0][1][:primary_key].should == true
+    target.schema(:test_table)[1][1][:primary_key].should == true
+  end
+end

data/spec/integration_helper.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+require 'database_helper'
+
+def create_source_table_with(*rows)
+  # Total hack to allow source db to be passed as optional first argument.
+  if rows[0].is_a?(Hash)
+    source_db  = source
+  else
+    source_db  = rows.shift
+  end
+  table_name = :test_table
+
+  source_db.create_table! table_name do
+    primary_key  :id
+    String       :col1
+    String       :pii
+    DateTime     :updated_at
+    DateTime     :created_at
+    DateTime     :imported_at
+  end
+
+  rows.each do |row|
+    source_db[table_name].insert(row)
+  end
+end
+
+def setup_target_table(last_synced_at)
+  target.create_table! :test_table do
+    Integer  :id
+    String   :col1
+    DateTime :updated_at
+    DateTime :created_at
+  end
+
+  target.add_index :test_table, :id, :unique => true
+
+  registry.ensure_storage_exists
+  registry.set(:test_table,
+    last_synced_at:       last_synced_at,
+    last_row_at:          last_synced_at,
+    last_batch_synced_at: last_synced_at
+  )
+end

data/spec/spec_helper.rb

@@ -0,0 +1,27 @@
+ENV['APP_ENV'] = 'test'
+
+if ENV['COVERAGE'] != "0" && RUBY_PLATFORM != 'java'
+  require 'simplecov'
+
+  class SimpleCov::Formatter::MergedFormatter
+    def format(result)
+      SimpleCov::Formatter::HTMLFormatter.new.format(result)
+      File.open("coverage/covered_percent", "w") do |f|
+        f.puts result.source_files.covered_percent.to_i
+      end
+    end
+  end
+
+  SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
+  SimpleCov.start do
+    add_filter "/config/"
+    add_filter "/spec/"
+  end
+end
+
+module Sq
+  module Dbsync
+  end
+end
+
+SQD = Sq::Dbsync

data/spec/unit/config_spec.rb

@@ -0,0 +1,18 @@
+require 'unit_helper'
+
+require 'sq/dbsync/config'
+
+describe Sq::Dbsync::Config do
+  it 'provides a default error handler' do
+    described_class.make({})[:error_handler].should respond_to(:call)
+  end
+
+  it 'provides a default clock' do
+    described_class.make({})[:clock].().should be_instance_of(Time)
+  end
+
+  it 'provides a default logger' do
+    described_class.make({})[:logger].should \
+      be_a_kind_of(Sq::Dbsync::Loggers::Abstract)
+  end
+end

data/spec/unit/error_handler_spec.rb

@@ -0,0 +1,52 @@
+require 'unit_helper'
+
+require 'sq/dbsync/error_handler'
+
+describe Sq::Dbsync::ErrorHandler do
+  let(:config) { {
+    sources: {
+      db_a: { password: 'redactme' },
+      db_b: { password: 'alsome' },
+      db_c: {}
+    },
+    target: { password: 'thistoo'},
+  }}
+
+  describe '#wrap' do
+    it 'redacts message' do
+      called = nil
+      config[:error_handler] = ->(ex) { called = ex }
+      handler = described_class.new(config)
+      ->{
+        handler.wrap do
+          raise "redactme alsome thistoo notthis"
+        end
+      }.should raise_error("REDACTED REDACTED REDACTED notthis")
+      called.message.should == "REDACTED REDACTED REDACTED notthis"
+    end
+  end
+
+  describe '#notify_error' do
+    it 'includes tag in exception message' do
+      called = nil
+      config[:error_handler] = ->(ex) { called = ex }
+      handler = described_class.new(config)
+
+      handler.notify_error(:test_table, RuntimeError.new('hello'))
+
+      called.message.should include('[test_table]')
+      called.message.should include('hello')
+    end
+
+    it 'redacts message' do
+      called = nil
+      config[:error_handler] = ->(ex) { called = ex }
+      handler = described_class.new(config)
+
+      handler.notify_error(:test_table,
+        RuntimeError.new("redactme alsome thistoo notthis"))
+
+      called.message.should include("REDACTED REDACTED REDACTED notthis")
+    end
+  end
+end

data/spec/unit/pipeline_spec.rb

@@ -0,0 +1,42 @@
+require 'unit_helper'
+
+require 'sq/dbsync/pipeline'
+
+shared_examples_for 'a pipeline' do
+  it 'passes tasks through each stage' do
+    ret = SQD::Pipeline.new([3, 4],
+      ->(x) { x * x },
+      ->(x) { x + x }
+    ).run(described_class)
+    ret.should == [18, 32]
+  end
+
+  it 'returns errors' do
+    ret = SQD::Pipeline.new([1],
+      ->(x) { raise("fail") }
+    ).run(described_class)
+    ret.length.should == 1
+    ret = ret[0]
+    ret.should be_instance_of(SQD::Pipeline::Failure)
+    ret.wrapped_exception.should be_instance_of(RuntimeError)
+    ret.wrapped_exception.message.should == "fail"
+    ret.task.should == 1
+  end
+
+  it 'handles errors in the middle of a pipeline' do
+    ret = SQD::Pipeline.new([1, 2],
+      ->(x) { x == 1 ? 10 : raise("fail") },
+      ->(x) { x + 1 }
+    ).run(described_class)
+    ret[0].should == 11
+    ret[1].should be_instance_of(SQD::Pipeline::Failure)
+  end
+end
+
+describe SQD::Pipeline::ThreadedContext do
+  it_should_behave_like 'a pipeline'
+end
+
+describe SQD::Pipeline::SimpleContext do
+  it_should_behave_like 'a pipeline'
+end

data/spec/unit/stream_logger_spec.rb

@@ -0,0 +1,33 @@
+require 'unit_helper'
+
+require 'sq/dbsync/loggers'
+
+describe SQD::Loggers::Stream do
+  let(:buffer) { "" }
+  let(:logger) { described_class.new(StringIO.new(buffer)) }
+
+  it 'logs :finished when no exception is raised' do
+    logger.measure(:ok) {}
+    buffer.should include('finished')
+  end
+
+  it 'logs :failed when exception is raised' do
+    lambda {
+      logger.measure(:fail) { raise("fail") }
+    }.should raise_error("fail")
+    buffer.should include('failed')
+  end
+
+  it 'logs error message when exception is raised' do
+    lambda {
+      logger.measure(:fail) { raise("oh no") }
+    }.should raise_error("oh no")
+    buffer.should include('oh no')
+  end
+
+
+  it 'logs specified strings' do
+    logger.log('logging is good')
+    buffer.should include('logging is good')
+  end
+end

data/spec/unit_helper.rb

@@ -0,0 +1 @@
+require 'spec_helper'

data/sq-dbsync.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/sq/dbsync/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Xavier Shay"]
+  gem.email         = ["xavier@squareup.com"]
+  gem.description   =
+    %q{Column based, timestamp replication of MySQL and Postgres databases.}
+  gem.summary       = %q{
+    Column based, timestamp replication of MySQL and Postgres databases. Uses
+    Ruby for the glue code but pushes the heavy lifting on to the database.
+  }
+  gem.homepage      = "http://github.com/square/sq-dbsync"
+
+  gem.executables   = []
+  gem.files         = Dir.glob("{spec,lib}/**/*.rb") + %w(
+                        README.md
+                        HISTORY.md
+                        LICENSE
+                        sq-dbsync.gemspec
+                      )
+  gem.test_files    = Dir.glob("spec/**/*.rb")
+  gem.name          = "sq-dbsync"
+  gem.require_paths = ["lib"]
+  gem.version       = Sq::Dbsync::VERSION
+  gem.has_rdoc      = false
+  gem.add_development_dependency 'rspec', '~> 2.0'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'cane'
+  gem.add_dependency 'sequel'
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: sq-dbsync
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Xavier Shay
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cane
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Column based, timestamp replication of MySQL and Postgres databases.
+email:
+- xavier@squareup.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- spec/acceptance/loading_spec.rb
+- spec/acceptance_helper.rb
+- spec/database_helper.rb
+- spec/integration/all_tables_plan_spec.rb
+- spec/integration/batch_load_action_spec.rb
+- spec/integration/consistency_verifier_spec.rb
+- spec/integration/database_connection_spec.rb
+- spec/integration/incremental_load_action_spec.rb
+- spec/integration/manager_spec.rb
+- spec/integration/schema_maker_spec.rb
+- spec/integration_helper.rb
+- spec/spec_helper.rb
+- spec/unit/config_spec.rb
+- spec/unit/error_handler_spec.rb
+- spec/unit/pipeline_spec.rb
+- spec/unit/stream_logger_spec.rb
+- spec/unit_helper.rb
+- lib/sq/dbsync/all_tables_plan.rb
+- lib/sq/dbsync/batch_load_action.rb
+- lib/sq/dbsync/config.rb
+- lib/sq/dbsync/consistency_verifier.rb
+- lib/sq/dbsync/database/common.rb
+- lib/sq/dbsync/database/connection.rb
+- lib/sq/dbsync/database/mysql.rb
+- lib/sq/dbsync/database/postgres.rb
+- lib/sq/dbsync/error_handler.rb
+- lib/sq/dbsync/example_record_destroyer.rb
+- lib/sq/dbsync/incremental_load_action.rb
+- lib/sq/dbsync/load_action.rb
+- lib/sq/dbsync/loggers.rb
+- lib/sq/dbsync/manager.rb
+- lib/sq/dbsync/pipeline/simple_context.rb
+- lib/sq/dbsync/pipeline/threaded_context.rb
+- lib/sq/dbsync/pipeline.rb
+- lib/sq/dbsync/refresh_recent_load_action.rb
+- lib/sq/dbsync/schema_maker.rb
+- lib/sq/dbsync/static_table_plan.rb
+- lib/sq/dbsync/table_registry.rb
+- lib/sq/dbsync/tempfile_factory.rb
+- lib/sq/dbsync/version.rb
+- lib/sq/dbsync.rb
+- README.md
+- HISTORY.md
+- LICENSE
+- sq-dbsync.gemspec
+homepage: http://github.com/square/sq-dbsync
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Column based, timestamp replication of MySQL and Postgres databases. Uses
+  Ruby for the glue code but pushes the heavy lifting on to the database.
+test_files:
+- spec/acceptance/loading_spec.rb
+- spec/acceptance_helper.rb
+- spec/database_helper.rb
+- spec/integration/all_tables_plan_spec.rb
+- spec/integration/batch_load_action_spec.rb
+- spec/integration/consistency_verifier_spec.rb
+- spec/integration/database_connection_spec.rb
+- spec/integration/incremental_load_action_spec.rb
+- spec/integration/manager_spec.rb
+- spec/integration/schema_maker_spec.rb
+- spec/integration_helper.rb
+- spec/spec_helper.rb
+- spec/unit/config_spec.rb
+- spec/unit/error_handler_spec.rb
+- spec/unit/pipeline_spec.rb
+- spec/unit/stream_logger_spec.rb
+- spec/unit_helper.rb