sbader-lhm 1.1.0

data/lib/lhm/sql_helper.rb

@@ -0,0 +1,85 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  module SqlHelper
+    extend self
+
+    def annotation
+      "/* large hadron migration */"
+    end
+
+    def idx_name(table_name, cols)
+      column_names = column_definition(cols).map(&:first)
+      "index_#{ table_name }_on_#{ column_names.join("_and_") }"
+    end
+
+    def idx_spec(cols)
+      column_definition(cols).map do |name, length|
+        "`#{ name }`#{ length }"
+      end.join(', ')
+    end
+
+    def table?(table_name)
+      connection.table_exists?(table_name)
+    end
+
+    def sql(statements)
+      [statements].flatten.each do |statement|
+        connection.execute(tagged(statement))
+      end
+    rescue ActiveRecord::StatementInvalid => e
+      error e.message
+    end
+
+    def update(statements)
+      [statements].flatten.inject(0) do |memo, statement|
+        memo += connection.update(tagged(statement))
+      end
+    rescue ActiveRecord::StatementInvalid => e
+      error e.message
+    end
+
+    def version_string
+      connection.select_one("show variables like 'version'")["Value"]
+    end
+
+  private
+
+    def tagged(statement)
+      "#{ statement } #{ SqlHelper.annotation }"
+    end
+
+    def column_definition(cols)
+      Array(cols).map do |column|
+        column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
+      end
+    end
+
+    # Older versions of MySQL contain an atomic rename bug affecting bin
+    # log order. Affected versions extracted from bug report:
+    #
+    #   http://bugs.mysql.com/bug.php?id=39675
+    #
+    # More Info: http://dev.mysql.com/doc/refman/5.5/en/metadata-locking.html
+    def supports_atomic_switch?
+      major, minor, tiny = version_string.split('.').map(&:to_i)
+
+      case major
+      when 4 then return false if minor and minor < 2
+      when 5
+        case minor
+        when 0 then return false if tiny and tiny < 52
+        when 1 then return false
+        when 4 then return false if tiny and tiny < 4
+        when 5 then return false if tiny and tiny < 3
+        end
+      when 6
+        case minor
+        when 0 then return false if tiny and tiny < 11
+        end
+      end
+      return true
+    end
+  end
+end

data/lib/lhm/table.rb

@@ -0,0 +1,97 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/sql_helper'
+
+module Lhm
+  class Table
+    attr_reader :name, :columns, :indices, :pk, :ddl
+
+    def initialize(name, pk = "id", ddl = nil)
+      @name = name
+      @columns = {}
+      @indices = {}
+      @pk = pk
+      @ddl = ddl
+    end
+
+    def satisfies_primary_key?
+      @pk == "id"
+    end
+
+    def destination_name
+      "lhmn_#{ @name }"
+    end
+
+    def self.parse(table_name, connection)
+      Parser.new(table_name, connection).parse
+    end
+
+    class Parser
+      include SqlHelper
+
+      def initialize(table_name, connection)
+        @table_name = table_name.to_s
+        @schema_name = connection.current_database
+        @connection = connection
+      end
+
+      def ddl
+        sql = "show create table `#{ @table_name }`"
+        specification = nil
+        @connection.execute(sql).each { |row| specification = row.last }
+        specification
+      end
+
+      def parse
+        schema = read_information_schema
+
+        Table.new(@table_name, extract_primary_key(schema), ddl).tap do |table|
+          schema.each do |defn|
+            table.columns[defn["COLUMN_NAME"]] = {
+              :type => defn["COLUMN_TYPE"],
+              :is_nullable => defn["IS_NULLABLE"],
+              :column_default => defn["COLUMN_DEFAULT"]
+            }
+          end
+
+          extract_indices(read_indices).each do |idx, columns|
+            table.indices[idx] = columns
+          end
+        end
+      end
+
+    private
+
+      def read_information_schema
+        @connection.select_all %Q{
+          select *
+            from information_schema.columns
+           where table_name = "#{ @table_name }"
+             and table_schema = "#{ @schema_name }"
+        }
+      end
+
+      def read_indices
+        @connection.select_all %Q{
+          show indexes from `#{ @schema_name }`.`#{ @table_name }`
+         where key_name != "PRIMARY"
+        }
+      end
+
+      def extract_indices(indices)
+        indices.map { |row| [row["Key_name"], row["Column_name"]] }.
+          inject(Hash.new { |h, k| h[k] = []}) do |memo, (idx, column)|
+            memo[idx] << column
+            memo
+          end
+      end
+
+      def extract_primary_key(schema)
+        cols = schema.select { |defn| defn["COLUMN_KEY"] == "PRI" }
+        keys = cols.map { |defn| defn["COLUMN_NAME"] }
+        keys.length == 1 ? keys.first : keys
+      end
+    end
+  end
+end

data/lib/lhm/version.rb

@@ -0,0 +1,6 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  VERSION = "1.1.0"
+end

data/spec/.lhm.example

@@ -0,0 +1,4 @@
+mysqldir=/usr/local/mysql
+basedir=/opt/lhm-cluster
+master_port=3306
+slave_port=3307

data/spec/README.md

@@ -0,0 +1,51 @@
+Preparing for master slave integration tests
+--------------------------------------------
+
+# configuration
+
+create ~/.lhm:
+
+    mysqldir=/usr/local/mysql
+    basedir=/opt/lhm-cluster
+    master_port=3306
+    slave_port=3307
+
+mysqldir specifies the location of your mysql install. basedir is the
+directory master and slave databases will get installed into.
+
+# setup
+
+You can set the integration specs up to run against a master slave setup by
+running the included `bin/lhm-spec-clobber.sh` script. this deletes the configured
+lhm master slave setup and reinstalls and configures a master slave setup.
+
+Follow the manual instructions if you want more control over this process.
+
+# manual setup
+
+## set up instances
+
+    bin/lhm-spec-setup-cluster.sh
+
+## start instances
+
+    basedir=/opt/lhm-luster
+    mysqld --defaults-file="$basedir/master/my.cnf"
+    mysqld --defaults-file="$basedir/slave/my.cnf"
+
+## run the grants
+
+    bin/lhm-spec-grants.sh
+
+## run specs
+
+To run specs in slave mode, set the SLAVE=1 when running tests:
+
+    MASTER_SLAVE=1 rake specs
+
+# connecting
+
+you can connect by running (with the respective ports):
+
+    mysql --protocol=TCP -p3307
+

data/spec/bootstrap.rb

@@ -0,0 +1,13 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'minitest/mock'
+require "pathname"
+
+$project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
+$spec = $project.join("spec")
+$fixtures = $spec.join("fixtures")
+
+$: << $project.join("lib").to_s

data/spec/fixtures/destination.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `destination` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `destination` int(11) DEFAULT NULL,
+  `common` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/origin.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `origin` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `origin` int(11) DEFAULT NULL,
+  `common` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/small_table.ddl

@@ -0,0 +1,4 @@
+CREATE TABLE `small_table` (
+  `id` INT(11),
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/users.ddl

@@ -0,0 +1,12 @@
+CREATE TABLE `users` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `reference` int(11) DEFAULT NULL,
+  `username` varchar(255) DEFAULT NULL,
+  `group` varchar(255) DEFAULT NULL,
+  `created_at` datetime DEFAULT NULL,
+  `comment` varchar(20) DEFAULT NULL,
+  `description` text,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_users_on_reference` (`reference`),
+  KEY `index_users_on_username_and_created_at` (`username`,`created_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/integration/atomic_switcher_spec.rb

@@ -0,0 +1,42 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+
+describe Lhm::AtomicSwitcher do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe "switching" do
+    before(:each) do
+      @origin = table_create("origin")
+      @destination = table_create("destination")
+      @migration = Lhm::Migration.new(@origin, @destination)
+    end
+
+    it "rename origin to archive" do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        table_exists?(@origin).must_equal true
+        table_read(@migration.archive_name).columns.keys.must_include "origin"
+      end
+    end
+
+    it "rename destination to origin" do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        table_exists?(@destination).must_equal false
+        table_read(@origin.name).columns.keys.must_include "destination"
+      end
+    end
+  end
+end

data/spec/integration/chunker_spec.rb

@@ -0,0 +1,32 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm'
+require 'lhm/table'
+require 'lhm/migration'
+
+describe Lhm::Chunker do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe "copying" do
+    before(:each) do
+      @origin = table_create(:origin)
+      @destination = table_create(:destination)
+      @migration = Lhm::Migration.new(@origin, @destination)
+    end
+
+    it "should copy 23 rows from origin to destination" do
+      23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
+
+      Lhm::Chunker.new(@migration, connection, { :stride => 100 }).run
+
+      slave do
+        count_all(@destination.name).must_equal(23)
+      end
+    end
+  end
+end

data/spec/integration/entangler_spec.rb

@@ -0,0 +1,66 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/entangler'
+
+describe Lhm::Entangler do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe "entanglement" do
+    before(:each) do
+      @origin = table_create("origin")
+      @destination = table_create("destination")
+      @migration = Lhm::Migration.new(@origin, @destination)
+      @entangler = Lhm::Entangler.new(@migration, connection)
+    end
+
+    it "should replay inserts from origin into destination" do
+      @entangler.run do |entangler|
+        execute("insert into origin (common) values ('inserted')")
+      end
+
+      slave do
+        count(:destination, "common", "inserted").must_equal(1)
+      end
+    end
+
+    it "should replay deletes from origin into destination" do
+      execute("insert into origin (common) values ('inserted')")
+
+      @entangler.run do |entangler|
+        execute("delete from origin where common = 'inserted'")
+      end
+
+      slave do
+        count(:destination, "common", "inserted").must_equal(0)
+      end
+    end
+
+    it "should replay updates from origin into destination" do
+      @entangler.run do |entangler|
+        execute("insert into origin (common) values ('inserted')")
+        execute("update origin set common = 'updated'")
+      end
+
+      slave do
+        count(:destination, "common", "updated").must_equal(1)
+      end
+    end
+
+    it "should remove entanglement" do
+      @entangler.run {}
+
+      execute("insert into origin (common) values ('inserted')")
+
+      slave do
+        count(:destination, "common", "inserted").must_equal(0)
+      end
+    end
+  end
+end

data/spec/integration/integration_helper.rb

@@ -0,0 +1,140 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + "/../bootstrap"
+
+require 'active_record'
+begin
+  require 'mysql2'
+rescue LoadError
+  require 'mysql'
+end
+require 'lhm/table'
+require 'lhm/sql_helper'
+
+module IntegrationHelper
+  #
+  # Connectivity
+  #
+
+  def connection
+    ActiveRecord::Base.connection
+  end
+
+  def connect_master!
+    connect!(3306)
+  end
+
+  def connect_slave!
+    connect!(3307)
+  end
+
+  def connect!(port)
+    ActiveRecord::Base.establish_connection(
+      :adapter => defined?(Mysql2) ? 'mysql2' : 'mysql',
+      :host => '127.0.0.1',
+      :database => 'lhm',
+      :username => '',
+      :port => port
+    )
+  end
+
+  def select_one(*args)
+    connection.select_one(*args)
+  end
+
+  def select_value(*args)
+    connection.select_value(*args)
+  end
+
+  def execute(*args)
+    retries = 10
+    begin
+      connection.execute(*args)
+    rescue ActiveRecord::StatementInvalid => e
+      if (retries -= 1) > 0 && e.message =~ /Table '.*?' doesn't exist/
+        sleep 0.1
+        retry
+      else
+        raise
+      end
+    end
+  end
+
+  def slave(&block)
+    if master_slave_mode?
+      connect_slave!
+
+      # need to wait for the slave to catch up. a better method would be to
+      # check the master binlog position and wait for the slave to catch up
+      # to that position.
+      sleep 1
+    end
+
+    yield block
+
+    if master_slave_mode?
+      connect_master!
+    end
+  end
+
+  #
+  # Test Data
+  #
+
+  def fixture(name)
+    File.read($fixtures.join("#{ name }.ddl"))
+  end
+
+  def table_create(fixture_name)
+    execute "drop table if exists `#{ fixture_name }`"
+    execute fixture(fixture_name)
+    table_read(fixture_name)
+  end
+
+  def table_read(fixture_name)
+    Lhm::Table.parse(fixture_name, connection)
+  end
+
+  def table_exists?(table)
+    connection.table_exists?(table.name)
+  end
+
+  #
+  # Database Helpers
+  #
+
+  def count(table, column, value)
+    query = "select count(*) from #{ table } where #{ column } = '#{ value }'"
+    select_value(query).to_i
+  end
+
+  def count_all(table)
+    query = "select count(*) from #{ table }"
+    select_value(query).to_i
+  end
+
+  def index_on_columns?(table_name, cols, type = :non_unique)
+    key_name = Lhm::SqlHelper.idx_name(table_name, cols)
+
+    index?(table_name, key_name, type)
+  end
+
+  def index?(table_name, key_name, type = :non_unique)
+    non_unique = type == :non_unique ? 1 : 0
+
+    !!select_one(%Q<
+      show indexes in #{ table_name }
+     where key_name = '#{ key_name }'
+       and non_unique = #{ non_unique }
+    >)
+  end
+
+  #
+  # Environment
+  #
+
+  def master_slave_mode?
+    !!ENV["MASTER_SLAVE"]
+  end
+end