lhm 1.0.0.rc2 → 1.0.0.rc3

data/spec/bootstrap.rb

@@ -1,7 +1,5 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require 'minitest/spec'
 require 'minitest/autorun'
@@ -13,4 +11,3 @@ $spec = $project.join("spec")
 $fixtures = $spec.join("fixtures")
 
 $: << $project.join("lib").to_s
-

data/spec/config/.config

@@ -0,0 +1,3 @@
+basedir=/opt/lhm-cluster
+master_port=3306
+slave_port=3307

data/spec/config/clobber

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+set -u
+
+source .config
+
+lhmkill() {
+  ps -ef | gsed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
+  sleep 5
+}
+
+echo stopping other running mysql instance
+launchctl remove com.mysql.mysqld || { echo launchctl did not remove mysqld; }
+mysqladmin shutdown || { echo mysqladmin did not shut down anything; }
+
+echo killing lhm-cluster
+lhmkill
+
+echo removing $basedir
+rm -rf "$basedir"
+
+echo setting up cluster
+spec/config/setup-cluster
+
+echo staring instances
+mysqld --defaults-file="$basedir/master/my.cnf" 2>&1 >$basedir/master/lhm.log &
+mysqld --defaults-file="$basedir/slave/my.cnf" 2>&1 >$basedir/slave/lhm.log &
+sleep 5
+
+echo running grants
+spec/config/grants
+
+trap lhmkill SIGTERM SIGINT
+
+wait

data/spec/config/grants

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+source .config
+
+master() { mysql --protocol=TCP -P $master_port -uroot; }
+slave()  { mysql --protocol=TCP -P $slave_port -uroot; }
+
+# set up master
+
+echo "create user 'slave'@'localhost' identified by 'slave'" | master
+echo "grant replication slave on *.* to 'slave'@'localhost'" | master
+
+# set up slave
+
+echo "change master to master_user = 'slave', master_password = 'slave', master_port = 3306, master_host = 'localhost'" | slave
+echo "start slave" | slave
+echo "show slave status \G" | slave
+
+# setup for test
+
+echo "grant all privileges on *.* to ''@'localhost'" | master
+echo "grant all privileges on *.* to ''@'localhost'" | slave
+
+echo "create database lhm" | master
+echo "create database lhm" | slave

data/spec/config/setup-cluster

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+#
+# Set up master slave cluster for lhm specs
+#
+
+set -e
+set -u
+
+source .config
+
+#
+# Main
+#
+
+mkdir -p "$basedir/master/data" "$basedir/slave/data"
+
+cat <<-CNF > $basedir/master/my.cnf
+[mysqld]
+pid-file = $basedir/master/mysqld.pid
+socket = $basedir/master/mysqld.sock
+port = $master_port
+log_output = FILE
+log-error = $basedir/master/error.log
+datadir = $basedir/master/data
+log-bin = master-bin
+log-bin-index = master-bin.index
+server-id = 1
+CNF
+
+cat <<-CNF > $basedir/slave/my.cnf
+[mysqld]
+pid-file = $basedir/slave/mysqld.pid
+socket = $basedir/slave/mysqld.sock
+port = $slave_port
+log_output = FILE
+log-error = $basedir/slave/error.log
+datadir = $basedir/slave/data
+relay-log = slave-relay-bin
+relay-log-index = slave-relay-bin.index
+server-id = 2
+
+# replication (optional filters)
+
+replicate-do-table = lhm.users
+replicate-do-table = lhm.lhmn_users
+replicate-wild-do = lhm.lhma_%_users
+
+replicate-do-table = lhm.origin
+replicate-do-table = lhm.lhmn_origin
+replicate-wild-do = lhm.lhma_%_origin
+
+replicate-do-table = lhm.destination
+replicate-do-table = lhm.lhmn_destination
+replicate-wild-do = lhm.lhma_%_destination
+CNF
+
+# build system tables
+
+mysql_install_db --datadir="$basedir/master/data"
+mysql_install_db --datadir="$basedir/slave/data"

data/spec/fixtures/destination.ddl

@@ -4,4 +4,3 @@ CREATE TABLE `destination` (
   `common` varchar(255) DEFAULT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8
-

data/spec/fixtures/origin.ddl

@@ -4,4 +4,3 @@ CREATE TABLE `origin` (
   `common` varchar(255) DEFAULT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8
-

data/spec/fixtures/users.ddl

@@ -2,10 +2,10 @@ CREATE TABLE `users` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `reference` int(11) DEFAULT NULL,
   `username` varchar(255) DEFAULT NULL,
+  `group` varchar(255) DEFAULT NULL,
   `created_at` datetime DEFAULT NULL,
   `comment` varchar(20) DEFAULT NULL,
   PRIMARY KEY (`id`),
   UNIQUE KEY `index_users_on_reference` (`reference`),
   KEY `index_users_on_username_and_created_at` (`username`,`created_at`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8
-

data/spec/integration/chunker_spec.rb

@@ -1,7 +1,5 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 
@@ -12,7 +10,7 @@ require 'lhm/migration'
 describe Lhm::Chunker do
   include IntegrationHelper
 
-  before(:each) { connect! }
+  before(:each) { connect_master! }
 
   describe "copying" do
     before(:each) do
@@ -22,10 +20,13 @@ describe Lhm::Chunker do
     end
 
     it "should copy 23 rows from origin to destination" do
-      23.times { |n| execute("insert into origin set common = '#{ n }'") }
-      Lhm::Chunker.new(@migration, limit = 23, connection).run
-      count_all(@destination.name).must_equal(23)
+      23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
+
+      Lhm::Chunker.new(@migration, connection, { :stride => 100 }).run
+
+      slave do
+        count_all(@destination.name).must_equal(23)
+      end
     end
   end
 end
-

data/spec/integration/entangler_spec.rb

@@ -1,7 +1,5 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 
@@ -12,7 +10,7 @@ require 'lhm/entangler'
 describe Lhm::Entangler do
   include IntegrationHelper
 
-  before(:each) { connect! }
+  before(:each) { connect_master! }
 
   describe "entanglement" do
     before(:each) do
@@ -27,7 +25,9 @@ describe Lhm::Entangler do
         execute("insert into origin (common) values ('inserted')")
       end
 
-      count(:destination, "common", "inserted").must_equal(1)
+      slave do
+        count(:destination, "common", "inserted").must_equal(1)
+      end
     end
 
     it "should replay deletes from origin into destination" do
@@ -37,7 +37,9 @@ describe Lhm::Entangler do
         execute("delete from origin where common = 'inserted'")
       end
 
-      count(:destination, "common", "inserted").must_equal(0)
+      slave do
+        count(:destination, "common", "inserted").must_equal(0)
+      end
     end
 
     it "should replay updates from origin into destination" do
@@ -46,15 +48,19 @@ describe Lhm::Entangler do
         execute("update origin set common = 'updated'")
       end
 
-      count(:destination, "common", "updated").must_equal(1)
+      slave do
+        count(:destination, "common", "updated").must_equal(1)
+      end
     end
 
     it "should remove entanglement" do
       @entangler.run {}
 
       execute("insert into origin (common) values ('inserted')")
-      count(:destination, "common", "inserted").must_equal(0)
+
+      slave do
+        count(:destination, "common", "inserted").must_equal(0)
+      end
     end
   end
 end
-

data/spec/integration/integration_helper.rb

@@ -1,31 +1,36 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + "/../bootstrap"
 
 require 'active_record'
 require 'lhm/table'
+require 'lhm/sql_helper'
 
 module IntegrationHelper
+  attr_accessor :connection
 
   #
   # Connectivity
   #
 
-  def connect!
+  def connect_master!
+    @connection = connect!(3306)
+  end
+
+  def connect_slave!
+    @connection = connect!(3307)
+  end
+
+  def connect!(port)
     ActiveRecord::Base.establish_connection(
       :adapter => 'mysql',
+      :host => '127.0.0.1',
       :database => 'lhm',
       :username => '',
-      :host => 'localhost'
+      :port => port
     )
 
-    ActiveRecord::Migration.verbose = !!ENV["VERBOSE"]
-  end
-
-  def connection
     ActiveRecord::Base.connection
   end
 
@@ -41,6 +46,23 @@ module IntegrationHelper
     connection.execute(*args)
   end
 
+  def slave(&block)
+    if master_slave_mode?
+      connect_slave!
+
+      # need to wait for the slave to catch up. a better method would be to
+      # check the master binlog position and wait for the slave to catch up
+      # to that position.
+      sleep 1
+    end
+
+    yield block
+
+    if master_slave_mode?
+      connect_master!
+    end
+  end
+
   #
   # Test Data
   #
@@ -77,9 +99,18 @@ module IntegrationHelper
     select_value(query).to_i
   end
 
-  def key?(table, cols, type = :non_unique)
+  def key?(table_name, cols, type = :non_unique)
     non_unique = type == :non_unique ? 1 : 0
-    query = "show indexes in #{ table.name } where key_name = '#{ table.idx_name(cols) }' and non_unique = #{ non_unique }"
+    key_name = Lhm::SqlHelper.idx_name(table_name, cols)
+    query = "show indexes in #{ table_name } where key_name = '#{ key_name }' and non_unique = #{ non_unique }"
     !!select_value(query)
   end
+
+  #
+  # Environment
+  #
+
+  def master_slave_mode?
+    !!ENV["MASTER_SLAVE"]
+  end
 end

data/spec/integration/lhm_spec.rb

@@ -1,7 +1,5 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 
@@ -9,9 +7,8 @@ require 'lhm'
 
 describe Lhm do
   include IntegrationHelper
-  include Lhm
 
-  before(:each) { connect! }
+  before(:each) { connect_master! }
 
   describe "changes" do
     before(:each) do
@@ -19,67 +16,91 @@ describe Lhm do
     end
 
     it "should add a column" do
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
-      table_read("users").columns["logins"].must_equal({
-        :type => "int(12)",
-        :metadata => "DEFAULT '0'"
-      })
+      slave do
+        table_read(:users).columns["logins"].must_equal({
+          :type => "int(12)",
+          :metadata => "DEFAULT '0'"
+        })
+      end
     end
 
     it "should copy all rows" do
       23.times { |n| execute("insert into users set reference = '#{ n }'") }
 
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
-      count_all("users").must_equal(23)
+      slave do
+        count_all(:users).must_equal(23)
+      end
     end
 
     it "should remove a column" do
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.remove_column(:comment)
       end
 
-      table_read("users").columns["comment"].must_equal nil
+      slave do
+        table_read(:users).columns["comment"].must_equal nil
+      end
     end
 
     it "should add an index" do
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.add_index([:comment, :created_at])
       end
 
-      key?(table_read("users"), ["comment", "created_at"]).must_equal(true)
+      slave do
+        key?(:users, [:comment, :created_at]).must_equal(true)
+      end
+    end
+
+    it "should add an index on a column with a reserved name" do
+      Lhm.change_table(:users) do |t|
+        t.add_index(:group)
+      end
+
+      slave do
+        key?(:users, :group).must_equal(true)
+      end
     end
 
     it "should add a unqiue index" do
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.add_unique_index(:comment)
       end
 
-      key?(table_read(:users), :comment, :unique).must_equal(true)
+      slave do
+        key?(:users, :comment, :unique).must_equal(true)
+      end
     end
 
     it "should remove an index" do
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.remove_index([:username, :created_at])
       end
 
-      key?(table_read("users"), ["username", "created_at"]).must_equal(false)
+      slave do
+        key?(:users, [:username, :created_at]).must_equal(false)
+      end
     end
 
     it "should apply a ddl statement" do
-      hadron_change_table("users") do |t|
+      Lhm.change_table(:users) do |t|
         t.ddl("alter table %s add column flag tinyint(1)" % t.name)
       end
 
-      table_read("users").columns["flag"].must_equal({
-        :type => "tinyint(1)",
-        :metadata => "DEFAULT NULL"
-      })
+      slave do
+        table_read(:users).columns["flag"].must_equal({
+          :type => "tinyint(1)",
+          :metadata => "DEFAULT NULL"
+        })
+      end
     end
 
     describe "parallel" do
@@ -93,34 +114,37 @@ describe Lhm do
           end
         end
 
-        hadron_change_table("users", :stride => 10, :throttle => 97) do |t|
+        Lhm.change_table(:users, :stride => 10, :throttle => 97) do |t|
           t.add_column(:parallel, "INT(10) DEFAULT '0'")
         end
 
         insert.join
 
-        count_all("users").must_equal(60)
+        slave do
+          count_all(:users).must_equal(60)
+        end
       end
-    end
 
-    it "should perserve deletes during migration" do
-      50.times { |n| execute("insert into users set reference = '#{ n }'") }
+      it "should perserve deletes during migration" do
+        50.times { |n| execute("insert into users set reference = '#{ n }'") }
 
-      insert = Thread.new do
-        10.times do |n|
-          execute("delete from users where id = '#{ n + 1 }'")
-          sleep(0.17)
+        delete = Thread.new do
+          10.times do |n|
+            execute("delete from users where id = '#{ n + 1 }'")
+            sleep(0.17)
+          end
         end
-      end
 
-      hadron_change_table("users", :stride => 10, :throttle => 97) do |t|
-        t.add_column(:parallel, "INT(10) DEFAULT '0'")
-      end
+        Lhm.change_table(:users, :stride => 10, :throttle => 97) do |t|
+          t.add_column(:parallel, "INT(10) DEFAULT '0'")
+        end
 
-      insert.join
+        delete.join
 
-      count_all("users").must_equal(40)
+        slave do
+          count_all(:users).must_equal(40)
+        end
+      end
     end
   end
 end
-