sq-dbsync 1.0.7 → 1.0.8

data/lib/sq/dbsync/database/common.rb

@@ -5,6 +5,17 @@ module Sq::Dbsync::Database
 
     SQD = ::Sq::Dbsync
 
+    def initialize(opts, source_or_target)
+      db = Sequel.connect(opts)
+      super(db)
+      @db, @source_or_target = db, source_or_target
+      @charset = opts[:charset] if opts[:charset]
+    end
+
+    def inspect
+      "#<Database::#{self.class.name} #{source_or_target} #{opts[:database]}>"
+    end
+
     def extract_to_file(table_name, columns, file_name)
       extract_sql_to_file("SELECT %s FROM %s" % [
         columns.join(', '),
@@ -34,7 +45,11 @@ module Sq::Dbsync::Database
 
     def hash_schema(plan)
       ensure_connection
-      Hash[schema(plan.source_table_name)]
+      Hash[schema(source? ? plan.source_table_name : plan.table_name)]
+    end
+
+    def source?
+      source_or_target == :source
     end
 
     def name
@@ -59,6 +74,8 @@ module Sq::Dbsync::Database
 
     protected
 
+    attr_reader :db, :source_or_target, :charset
+
     def execute!(cmd)
       # psql doesn't return a non-zero error code when executing commands from
       # a file. The best way I can come up with is to raise if anything is

data/lib/sq/dbsync/database/connection.rb

@@ -9,14 +9,12 @@ module Sq::Dbsync::Database
   # Factory class to abstract selection of a decorator to faciliate databases
   # other than MySQL.
   class Connection
-    def self.create(opts)
+    def self.create(opts, direction)
       case opts[:brand]
       when 'mysql'
-        db = Sq::Dbsync::Database::Mysql.new(Sequel.connect(opts))
-        db.charset = opts[:charset] if opts[:charset]
-        db
+        Sq::Dbsync::Database::Mysql.new(opts, direction)
       when 'postgresql'
-        Sq::Dbsync::Database::Postgres.new(Sequel.connect(opts))
+        Sq::Dbsync::Database::Postgres.new(opts, direction)
       else
         raise "Unsupported database: #{opts.inspect}"
       end

data/lib/sq/dbsync/database/mysql.rb

@@ -13,28 +13,17 @@ module Sq::Dbsync::Database
   # Decorator around a Sequel database object, providing some non-standard
   # extensions required for effective ETL with MySQL.
   class Mysql < Delegator
+    # 2 days is chosen as an arbitrary buffer
+    AUX_TIME_BUFFER = 60 * 60 * 24 * 2 # 2 days
+    LOAD_SQL = "LOAD DATA INFILE '%s' %s INTO TABLE %s %s (%s)"
 
     include Common
 
-    attr_accessor :charset
-
-    def initialize(db)
-      super
-      @db = db
-    end
-
-    def inspect; "#<Database::Mysql #{opts[:database]}>"; end
-
     def load_from_file(table_name, columns, file_name)
       ensure_connection
-      character_set = self.charset ? " character set #{self.charset}" : ""
-      sql = "LOAD DATA INFILE '%s' IGNORE INTO TABLE %s %s (%s)" % [
-        file_name,
-        table_name,
-        character_set,
-        escape_columns(columns)
-      ]
-      db.run sql
+      db.run(LOAD_SQL % [
+        file_name, 'IGNORE', table_name, character_set, escape_columns(columns)
+      ])
     end
 
     def set_lock_timeout(seconds)
@@ -46,13 +35,9 @@ module Sq::Dbsync::Database
       # Very low lock wait timeout, since we don't want loads to be blocked
       # waiting for long queries.
       set_lock_timeout(10)
-      character_set = self.charset ? " character set #{self.charset}" : ""
-      db.run "LOAD DATA INFILE '%s' REPLACE INTO TABLE %s %s (%s)" % [
-        file_name,
-        table_name,
-        character_set,
-        escape_columns(columns)
-      ]
+      db.run(LOAD_SQL % [
+        file_name, 'REPLACE', table_name, character_set, escape_columns(columns)
+      ])
     rescue Sequel::DatabaseError => e
       transient_regex =
         /Lock wait timeout exceeded|Deadlock found when trying to get lock/
@@ -64,9 +49,6 @@ module Sq::Dbsync::Database
       end
     end
 
-    # 2 days is chosen as an arbitrary buffer
-    AUX_TIME_BUFFER = 60 * 60 * 24 * 2 # 2 days
-
     # Deletes recent rows based on timestamp, but also allows filtering by an
     # auxilary timestamp column for the case where the primary one is not
     # indexed on the target (such as the DFR reports, where imported_at is not
@@ -93,21 +75,6 @@ module Sq::Dbsync::Database
         count
     end
 
-    # Overriden because the Sequel implementation does not work with partial
-    # permissions on a table. See:
-    # https://github.com/jeremyevans/sequel/issues/422
-    def table_exists?(table_name)
-      begin
-        !!db.schema(table_name, reload: true)
-      rescue Sequel::DatabaseError
-        false
-      end
-    end
-
-    def drop_table(table_name)
-      db.drop_table(table_name)
-    end
-
     def switch_table(to_replace, new_table)
       ensure_connection
 
@@ -131,8 +98,6 @@ module Sq::Dbsync::Database
 
     protected
 
-    attr_reader :db
-
     def extract_sql_to_file(sql, file_name)
       file = sql_to_file(connection_settings + sql)
       cmd = "set -o pipefail; mysql --skip-column-names"
@@ -147,7 +112,7 @@ module Sq::Dbsync::Database
         ]
       end
 
-      cmd += " --default-character-set %s" % opts[:charset] if opts[:charset]
+      cmd += " --default-character-set %s" % charset if charset
 
       cmd += " %s"      % opts.fetch(:database)
 
@@ -170,9 +135,10 @@ module Sq::Dbsync::Database
       lock_timeout_sql(10)
     end
 
+    def character_set; charset ? " character set #{charset}" : "" end
+
     def lock_timeout_sql(seconds)
       "SET SESSION innodb_lock_wait_timeout = %i;" % seconds
     end
-
   end
 end

data/lib/sq/dbsync/database/postgres.rb

@@ -29,13 +29,6 @@ module Sq::Dbsync::Database
 
     include Sq::Dbsync::Database::Common
 
-    def initialize(db)
-      super
-      @db = db
-    end
-
-    def inspect; "#<Database::Postgres #{opts[:database]}>"; end
-
     def set_lock_timeout(seconds)
       # Unimplemented
     end
@@ -44,7 +37,8 @@ module Sq::Dbsync::Database
       type_casts = plan.type_casts || {}
       ensure_connection
 
-      result = schema(plan.source_table_name).each do |col, metadata|
+      table_name = source? ? plan.source_table_name : plan.table_name
+      result = schema(table_name).each do |col, metadata|
         metadata[:source_db_type] ||= metadata[:db_type]
         metadata[:db_type] = cast_psql_to_mysql(
           metadata[:db_type], type_casts[col.to_s]
@@ -56,8 +50,6 @@ module Sq::Dbsync::Database
 
     protected
 
-    attr_reader :db
-
     def cast_psql_to_mysql(db_type, cast=nil)
       CASTS.fetch(db_type, cast || db_type)
     end

data/lib/sq/dbsync/manager.rb

@@ -105,7 +105,8 @@ class Sq::Dbsync::Manager
   end
 
   def target
-    @target ||= Sq::Dbsync::Database::Connection.create(config[:target])
+    opts = config[:target]
+    @target ||= Sq::Dbsync::Database::Connection.create(opts, :target)
   end
 
   def tables_to_load
@@ -124,7 +125,7 @@ class Sq::Dbsync::Manager
 
   def sources
     @sources ||= Hash[config[:sources].map do |name, opts|
-      [name, Sq::Dbsync::Database::Connection.create(opts)]
+      [name, Sq::Dbsync::Database::Connection.create(opts, :source)]
     end]
   end
 
@@ -217,7 +218,7 @@ class Sq::Dbsync::Manager
   end
 
   def db
-    @db ||= Database::Connection.create(config[:target])
+    @db ||= Database::Connection.create(config[:target], :target)
   end
 
   def transient_exceptions

data/lib/sq/dbsync/version.rb

@@ -1,5 +1,5 @@
 module Sq
   module Dbsync
-    VERSION = '1.0.7'
+    VERSION = '1.0.8'
   end
 end

data/spec/database_helper.rb

@@ -48,12 +48,14 @@ MB4_TEST_TARGET = db_options(database: 'sq_dbsync_test_target', charset:"utf8mb4
 
 $target = nil
 def test_target
-  $target ||= SQD::Database::Connection.create(TEST_TARGET)
+  $target ||= SQD::Database::Connection.create(TEST_TARGET, :target)
 end
 
 $sources = {}
 def test_source(name)
-  $sources[name] ||= SQD::Database::Connection.create(TEST_SOURCES.fetch(name))
+  $sources[name] ||= SQD::Database::Connection.create(
+    TEST_SOURCES.fetch(name), :source
+  )
 end
 
 RSpec.configure do |config|

data/spec/integration/batch_load_action_spec.rb

@@ -11,8 +11,9 @@ describe SQD::BatchLoadAction do
   let!(:now)    { @now = Time.now.utc }
   let(:last_synced_at) { now - 10 }
   let(:target) { test_target }
+  let(:target_table_name) { :test_table }
   let(:table_plan) {{
-    table_name: :test_table,
+    table_name: target_table_name,
     source_table_name: :test_table,
     columns: [:id, :col1, :updated_at],
     source_db: source,
@@ -20,7 +21,7 @@ describe SQD::BatchLoadAction do
   }}
   let(:index) {{
     index_on_col1: { columns: [:col1], unique: false }
-  } }
+  }}
   let(:registry) { SQD::TableRegistry.new(target) }
   let(:action) { SQD::BatchLoadAction.new(
     target,
@@ -59,14 +60,18 @@ describe SQD::BatchLoadAction do
       end
     end
 
-    it 'copies source tables to target with matching schemas' do
-      start_time = now.to_f
+    describe 'when the source and destination table names differ' do
+      let(:target_table_name) { :target_test_table }
 
-      action.call
+      it 'copies source tables to target with matching schemas' do
+        start_time = now.to_f
 
-      verify_schema
-      verify_data
-      verify_metadata(start_time)
+        action.call
+
+        verify_schema
+        verify_data
+        verify_metadata(start_time)
+      end
     end
 
     it 'handles column that does not exist in source' do
@@ -139,13 +144,12 @@ describe SQD::BatchLoadAction do
 
     def test_tables
       {
-        test_table: source,
+        test_table: [source, :target_test_table],
       }
     end
 
     def verify_schema
-      test_tables.each do |table_name, source_db|
-        target_table_name = table_name
+      test_tables.each do |table_name, (source_db, target_table_name)|
         target.tables.should include(target_table_name)
         source_test_table_schema =
           source_db.schema(table_name).map do |column, hash|
@@ -177,8 +181,8 @@ describe SQD::BatchLoadAction do
     end
 
     def verify_data
-      test_tables.each do |table_name, _|
-        data = target[table_name].all
+      test_tables.each do |table_name, (source_db, target_table_name)|
+        data = target[target_table_name].all
         data.count.should == 1
         data = data[0]
         data.keys.length.should == 3
@@ -189,8 +193,8 @@ describe SQD::BatchLoadAction do
     end
 
     def verify_metadata(start_time)
-      test_tables.each do |table_name, _|
-        meta = registry.get(table_name)
+      test_tables.each do |table_name, (source_db, target_table_name)|
+        meta = registry.get(target_table_name)
         meta[:last_synced_at].should_not be_nil
         meta[:last_batch_synced_at].should_not be_nil
         meta[:last_batch_synced_at].to_i.should == start_time.to_i

data/spec/integration/database_connection_spec.rb

@@ -17,15 +17,13 @@ shared_examples_for 'a decorated database adapter' do
 end
 
 describe SQD::Database::Postgres do
-  let(:source) { test_source(:postgres) }
-  let(:db) { SQD::Database::Postgres.new(source) }
+  let(:db) { test_source(:postgres) }
 
   it_should_behave_like 'a decorated database adapter'
 end
 
 describe SQD::Database::Mysql do
-  let(:source) { test_source(:source) }
-  let(:db)     { SQD::Database::Mysql.new(source) }
+  let(:db) { test_source(:source) }
 
   it_should_behave_like 'a decorated database adapter'
 
@@ -34,26 +32,26 @@ describe SQD::Database::Mysql do
 
     before { @file = Tempfile.new('bogus') }
 
-    def source_with_exception(exception_message)
-      source.stub(:run).and_raise(
+    def sequel_with_exception(exception_message)
+      db.send(:db).stub(:run).and_raise(
         Sequel::DatabaseError.new(exception_message)
       )
     end
 
     it 're-raises deadlock related exceptions as TransientError' do
-      source_with_exception("Deadlock found when trying to get lock")
+      sequel_with_exception("Deadlock found when trying to get lock")
       -> { db.load_incrementally_from_file('bogus', ['bogus'], path) }.
         should raise_error(SQD::Database::TransientError)
     end
 
     it 're-raises lock wait timeout exceptions as TransientError' do
-      source_with_exception("Lock wait timeout exceeded")
+      sequel_with_exception("Lock wait timeout exceeded")
       -> { db.load_incrementally_from_file('bogus', ['bogus'], path) }.
         should raise_error(SQD::Database::TransientError)
     end
 
     it 'does not translate unknown errors' do
-      source_with_exception("Unknown")
+      sequel_with_exception("Unknown")
       -> { db.load_incrementally_from_file('bogus', ['bogus'], path) }.
         should raise_error(Sequel::DatabaseError)
     end

data/spec/integration/incremental_load_action_spec.rb

@@ -11,8 +11,9 @@ describe SQD::IncrementalLoadAction do
   let(:last_synced_at) { now - 10 }
   let(:source)         { test_source(:source) }
   let(:target) { test_target }
+  let(:target_table_name) { :test_table }
   let(:table_plan) {{
-    table_name: :test_table,
+    table_name: target_table_name,
     source_table_name: :test_table,
     columns: [:id, :col1, :updated_at],
     source_db: source,
@@ -60,6 +61,18 @@ describe SQD::IncrementalLoadAction do
       end
     end
 
+    describe 'when source and target are differently named' do
+      let(:target_table_name) { :target_test_table }
+
+      it 'copies all columns to the correctly named target' do
+        setup_target_table(last_synced_at, target_table_name)
+
+        action.call
+
+        target[target_table_name].map { |row| row.values_at(:id, :col1) }.
+          should == [[2, 'new record']]
+      end
+    end
 
     it 'copies null data to the target' do
       source[:test_table].update(col1: nil)

data/spec/integration_helper.rb

@@ -48,18 +48,18 @@ def create_pg_source_table_with(*rows)
   end
 end
 
-def setup_target_table(last_synced_at)
-  target.create_table! :test_table do
+def setup_target_table(last_synced_at, name=:test_table)
+  target.create_table! name do
     Integer  :id
     String   :col1
     DateTime :updated_at
     DateTime :created_at
   end
 
-  target.add_index :test_table, :id, :unique => true
+  target.add_index name, :id, :unique => true
 
   registry.ensure_storage_exists
-  registry.set(:test_table,
+  registry.set(name,
     last_synced_at:       last_synced_at,
     last_row_at:          last_synced_at,
     last_batch_synced_at: last_synced_at

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sq-dbsync
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-29 00:00:00.000000000 Z
+date: 2013-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec