sbader-lhm 1.1.0

data/bin/lhm-spec-clobber.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -e
+set -u
+
+source ~/.lhm
+
+lhmkill() {
+  ps -ef | gsed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
+  sleep 5
+}
+
+echo stopping other running mysql instance
+launchctl remove com.mysql.mysqld || { echo launchctl did not remove mysqld; }
+"$mysqldir"/bin/mysqladmin shutdown || { echo mysqladmin did not shut down anything; }
+
+echo killing lhm-cluster
+lhmkill
+
+echo removing $basedir
+rm -rf "$basedir"
+
+echo setting up cluster
+bin/lhm-spec-setup-cluster.sh
+
+echo staring instances
+"$mysqldir"/bin/mysqld --defaults-file="$basedir/master/my.cnf" 2>&1 >$basedir/master/lhm.log &
+"$mysqldir"/bin/mysqld --defaults-file="$basedir/slave/my.cnf" 2>&1 >$basedir/slave/lhm.log &
+sleep 5
+
+echo running grants
+bin/lhm-spec-grants.sh
+
+trap lhmkill SIGTERM SIGINT
+
+wait

data/bin/lhm-spec-grants.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+source ~/.lhm
+
+master() { "$mysqldir"/bin/mysql --protocol=TCP -P $master_port -uroot; }
+slave()  { "$mysqldir"/bin/mysql --protocol=TCP -P $slave_port -uroot; }
+
+# set up master
+
+echo "create user 'slave'@'localhost' identified by 'slave'" | master
+echo "grant replication slave on *.* to 'slave'@'localhost'" | master
+
+# set up slave
+
+echo "change master to master_user = 'slave', master_password = 'slave', master_port = 3306, master_host = 'localhost'" | slave
+echo "start slave" | slave
+echo "show slave status \G" | slave
+
+# setup for test
+
+echo "grant all privileges on *.* to ''@'localhost'" | master
+echo "grant all privileges on *.* to ''@'localhost'" | slave
+
+echo "create database lhm" | master
+echo "create database if not exists lhm" | slave

data/bin/lhm-spec-setup-cluster.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+#
+# Set up master slave cluster for lhm specs
+#
+
+set -e
+set -u
+
+source ~/.lhm
+
+#
+# Main
+#
+
+install_bin="$(echo ./*/mysql_install_db)"
+
+mkdir -p "$basedir/master/data" "$basedir/slave/data"
+
+cat <<-CNF > $basedir/master/my.cnf
+[mysqld]
+pid-file = $basedir/master/mysqld.pid
+socket = $basedir/master/mysqld.sock
+port = $master_port
+log_output = FILE
+log-error = $basedir/master/error.log
+datadir = $basedir/master/data
+log-bin = master-bin
+log-bin-index = master-bin.index
+server-id = 1
+CNF
+
+cat <<-CNF > $basedir/slave/my.cnf
+[mysqld]
+pid-file = $basedir/slave/mysqld.pid
+socket = $basedir/slave/mysqld.sock
+port = $slave_port
+log_output = FILE
+log-error = $basedir/slave/error.log
+datadir = $basedir/slave/data
+relay-log = slave-relay-bin
+relay-log-index = slave-relay-bin.index
+server-id = 2
+
+# replication (optional filters)
+
+# replicate-do-table = lhm.users
+# replicate-do-table = lhm.lhmn_users
+# replicate-wild-do-table = lhm.lhma_%_users
+
+# replicate-do-table = lhm.origin
+# replicate-do-table = lhm.lhmn_origin
+# replicate-wild-do-table = lhm.lhma_%_origin
+
+# replicate-do-table = lhm.destination
+# replicate-do-table = lhm.lhmn_destination
+# replicate-wild-do-table = lhm.lhma_%_destination
+CNF
+
+# build system tables
+
+(
+  cd "$mysqldir"
+  $install_bin --datadir="$basedir/master/data"
+  $install_bin --datadir="$basedir/slave/data"
+
+)

data/bin/lhm-test-all.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+for gemfile in gemfiles/*.gemfile
+do
+  if !(BUNDLE_GEMFILE=$gemfile bundle install &&
+       BUNDLE_GEMFILE=$gemfile bundle exec rake)
+  then
+    exit 1
+  fi
+done

data/gemfiles/ar-2.3_mysql.gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gem "mysql", "~> 2.8.1"
+gem "activerecord", "~> 2.3.14"
+gemspec :path=>"../"

data/gemfiles/ar-3.2_mysql.gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gem "mysql", "~> 2.8.1"
+gem "activerecord", "~> 3.2.2"
+gemspec :path=>"../"

data/gemfiles/ar-3.2_mysql2.gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+gem "mysql2", "~> 0.3.11"
+gem "activerecord", "~> 3.2.2"
+gemspec :path=>"../"

data/lhm.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib', __FILE__)
+$:.unshift(lib) unless $:.include?(lib)
+
+require 'lhm/version'
+
+Gem::Specification.new do |s|
+  s.name          = "sbader-lhm"
+  s.version       = Lhm::VERSION
+  s.platform      = Gem::Platform::RUBY
+  s.authors       = ["SoundCloud", "Rany Keddo", "Tobias Bielohlawek", "Tobias Schmidt"]
+  s.email         = %q{rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com}
+  s.summary       = %q{online schema changer for mysql}
+  s.description   = %q{Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.}
+  s.homepage      = %q{http://github.com/soundcloud/large-hadron-migrator}
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+  s.executables   = ["lhm-kill-queue"]
+
+  s.add_development_dependency "minitest", "= 2.10.0"
+  s.add_development_dependency "rake"
+
+  s.add_dependency "activerecord"
+end
+

data/lib/lhm.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'active_record'
+require 'lhm/table'
+require 'lhm/invoker'
+require 'lhm/version'
+
+# Large hadron migrator - online schema change tool
+#
+# @example
+#
+#   Lhm.change_table(:users) do |m|
+#     m.add_column(:arbitrary, "INT(12)")
+#     m.add_index([:arbitrary, :created_at])
+#     m.ddl("alter table %s add column flag tinyint(1)" % m.name)
+#   end
+#
+module Lhm
+
+  # Alters a table with the changes described in the block
+  #
+  # @param [String, Symbol] table_name Name of the table
+  # @param [Hash] options Optional options to alter the chunk / switch behavior
+  # @option options [Fixnum] :stride
+  #   Size of a chunk (defaults to: 40,000)
+  # @option options [Fixnum] :throttle
+  #   Time to wait between chunks in milliseconds (defaults to: 100)
+  # @option options [Boolean] :atomic_switch
+  #   Use atomic switch to rename tables (defaults to: true)
+  #   If using a version of mysql affected by atomic switch bug, LHM forces user
+  #   to set this option (see SqlHelper#supports_atomic_switch?)
+  # @yield [Migrator] Yielded Migrator object records the changes
+  # @return [Boolean] Returns true if the migration finishes
+  # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
+  def self.change_table(table_name, options = {}, &block)
+    connection = ActiveRecord::Base.connection
+    origin = Table.parse(table_name, connection)
+    invoker = Invoker.new(origin, connection)
+    block.call(invoker.migrator)
+    invoker.run(options)
+
+    true
+  end
+end

data/lib/lhm/atomic_switcher.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+  # Switches origin with destination table using an atomic rename.
+  #
+  # It should only be used if the MySQL server version is not affected by the
+  # bin log affecting bug #39675. This can be verified using
+  # Lhm::SqlHelper.supports_atomic_switch?.
+  class AtomicSwitcher
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def statements
+      atomic_switch
+    end
+
+    def atomic_switch
+      [
+        "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+        "`#{ @destination.name }` to `#{ @origin.name }`"
+      ]
+    end
+
+    def validate
+      unless table?(@origin.name) && table?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+  private
+    def execute
+      sql statements
+    end
+  end
+end

data/lib/lhm/chunker.rb

@@ -0,0 +1,114 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/sql_helper'
+
+module Lhm
+  class Chunker
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    # Copy from origin to destination in chunks of size `stride`. Sleeps for
+    # `throttle` milliseconds between each stride.
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
+      @stride = options[:stride] || 40_000
+      @throttle = options[:throttle] || 100
+      @start = options[:start] || select_start
+      @limit = options[:limit] || select_limit
+    end
+
+    # Copies chunks of size `stride`, starting from `start` up to id `limit`.
+    def up_to(&block)
+      1.upto(traversable_chunks_size) do |n|
+        yield(bottom(n), top(n))
+      end
+    end
+
+    def traversable_chunks_size
+      @limit && @start ? ((@limit - @start + 1) / @stride.to_f).ceil : 0
+    end
+
+    def bottom(chunk)
+      (chunk - 1) * @stride + @start
+    end
+
+    def top(chunk)
+      [chunk * @stride + @start - 1, @limit].min
+    end
+
+    def copy(lowest, highest)
+      "insert ignore into `#{ destination_name }` (#{ columns }) " +
+      "select #{ columns_with_joins } from `#{ origin_name }` " +
+      "#{ joins } " +
+      "where #{origin_name}.`id` between #{ lowest } and #{ highest }"
+    end
+
+    def joins
+      @migration.insert_joins.map{|j| "join #{ j[:table] } on #{ j[:statement] }"}.join(" ")
+    end
+
+    def join_fields_typed
+      @migration.insert_joins.map {|j| "#{j[:table]}.`#{j[:origin_field]}`"}
+    end
+
+    def join_fields
+      @migration.insert_joins.map {|j| "`#{ j[:destination_field] }`"}
+    end
+
+    def select_start
+      start = connection.select_value("select min(id) from #{ origin_name }")
+      start ? start.to_i : nil
+    end
+
+    def select_limit
+      limit = connection.select_value("select max(id) from #{ origin_name }")
+      limit ? limit.to_i : nil
+    end
+
+    def throttle_seconds
+      @throttle / 1000.0
+    end
+
+  private
+
+    def destination_name
+      @migration.destination.name
+    end
+
+    def origin_name
+      @migration.origin.name
+    end
+
+    def columns
+      @columns ||= (@migration.intersection.escaped + join_fields).join(", ")
+    end
+
+    def columns_with_joins
+      @columns_with_joins ||= (@migration.intersection.typed_unjoined(origin_name) + join_fields_typed).join(", ")
+    end
+
+    def validate
+      if @start && @limit && @start > @limit
+        error("impossible chunk options (limit must be greater than start)")
+      end
+    end
+
+    def execute
+      up_to do |lowest, highest|
+        affected_rows = update(copy(lowest, highest))
+
+        if affected_rows > 0
+          sleep(throttle_seconds)
+        end
+
+        print "."
+      end
+      print "\n"
+    end
+  end
+end

data/lib/lhm/command.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  class Error < StandardError
+  end
+
+  module Command
+    def run(&block)
+      validate
+
+      if(block_given?)
+        before
+        block.call(self)
+        after
+      else
+        execute
+      end
+    rescue
+      revert
+      raise
+    end
+
+  private
+
+    def validate
+    end
+
+    def revert
+    end
+
+    def execute
+      raise NotImplementedError.new(self.class.name)
+    end
+
+    def before
+    end
+
+    def after
+    end
+
+    def error(msg)
+      raise Error.new(msg)
+    end
+  end
+end