lhm 1.2.0 → 2.2.0

data/lib/lhm/table.rb

@@ -7,7 +7,7 @@ module Lhm
   class Table
     attr_reader :name, :columns, :indices, :pk, :ddl
 
-    def initialize(name, pk = "id", ddl = nil)
+    def initialize(name, pk = 'id', ddl = nil)
       @name = name
       @columns = {}
       @indices = {}
@@ -16,7 +16,7 @@ module Lhm
     end
 
     def satisfies_primary_key?
-      @pk == "id"
+      @pk == 'id'
     end
 
     def destination_name
@@ -45,10 +45,10 @@ module Lhm
 
         Table.new(@table_name, extract_primary_key(schema), ddl).tap do |table|
           schema.each do |defn|
-            column_name    = struct_key(defn, "COLUMN_NAME")
-            column_type    = struct_key(defn, "COLUMN_TYPE")
-            is_nullable    = struct_key(defn, "IS_NULLABLE")
-            column_default = struct_key(defn, "COLUMN_DEFAULT")
+            column_name    = struct_key(defn, 'COLUMN_NAME')
+            column_type    = struct_key(defn, 'COLUMN_TYPE')
+            is_nullable    = struct_key(defn, 'IS_NULLABLE')
+            column_default = struct_key(defn, 'COLUMN_DEFAULT')
             table.columns[defn[column_name]] = {
               :type => defn[column_type],
               :is_nullable => defn[is_nullable],
@@ -83,11 +83,11 @@ module Lhm
       def extract_indices(indices)
         indices.
           map do |row|
-            key_name = struct_key(row, "Key_name")
-            column_name = struct_key(row, "COLUMN_NAME")
+            key_name = struct_key(row, 'Key_name')
+            column_name = struct_key(row, 'COLUMN_NAME')
             [row[key_name], row[column_name]]
           end.
-          inject(Hash.new { |h, k| h[k] = []}) do |memo, (idx, column)|
+          inject(Hash.new { |h, k| h[k] = [] }) do |memo, (idx, column)|
             memo[idx] << column
             memo
           end
@@ -95,12 +95,12 @@ module Lhm
 
       def extract_primary_key(schema)
         cols = schema.select do |defn|
-          column_key = struct_key(defn, "COLUMN_KEY")
-          defn[column_key] == "PRI"
+          column_key = struct_key(defn, 'COLUMN_KEY')
+          defn[column_key] == 'PRI'
         end
 
         keys = cols.map do |defn|
-          column_name = struct_key(defn, "COLUMN_NAME")
+          column_name = struct_key(defn, 'COLUMN_NAME')
           defn[column_name]
         end
 

data/lib/lhm/throttler/time.rb

@@ -0,0 +1,29 @@
+module Lhm
+  module Throttler
+    class Time
+      include Command
+
+      DEFAULT_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 40_000
+
+      attr_accessor :timeout_seconds
+      attr_accessor :stride
+
+      def initialize(options = {})
+        @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+      end
+
+      def execute
+        sleep timeout_seconds
+      end
+    end
+
+    class LegacyTime < Time
+      def initialize(timeout, stride)
+        @timeout_seconds = timeout / 1000.0
+        @stride = stride
+      end
+    end
+  end
+end

data/lib/lhm/throttler.rb

@@ -0,0 +1,32 @@
+require 'lhm/throttler/time'
+
+module Lhm
+  module Throttler
+    CLASSES = { :time_throttler => Throttler::Time }
+
+    def throttler
+      @throttler ||= Throttler::Time.new
+    end
+
+    def setup_throttler(type, options = {})
+      @throttler = Factory.create_throttler(type, options)
+    end
+
+    class Factory
+      def self.create_throttler(type, options = {})
+        case type
+        when Lhm::Command
+          type
+        when Symbol
+          CLASSES[type].new(options)
+        when String
+          CLASSES[type.to_sym].new(options)
+        when Class
+          type.new(options)
+        else
+          raise ArgumentError, 'type argument must be a Symbol, String or Class'
+        end
+      end
+    end
+  end
+end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = "1.2.0"
+  VERSION = '2.2.0'
 end

data/lib/lhm.rb

@@ -4,7 +4,9 @@
 require 'lhm/table'
 require 'lhm/invoker'
 require 'lhm/connection'
+require 'lhm/throttler'
 require 'lhm/version'
+require 'logger'
 
 # Large hadron migrator - online schema change tool
 #
@@ -17,6 +19,10 @@ require 'lhm/version'
 #   end
 #
 module Lhm
+  extend Throttler
+  extend self
+
+  DEFAULT_LOGGER_OPTIONS =  { level: Logger::INFO, file: STDOUT }
 
   # Alters a table with the changes described in the block
   #
@@ -26,6 +32,10 @@ module Lhm
   #   Size of a chunk (defaults to: 40,000)
   # @option options [Fixnum] :throttle
   #   Time to wait between chunks in milliseconds (defaults to: 100)
+  # @option options [Fixnum] :start
+  #   Primary Key position at which to start copying chunks
+  # @option options [Fixnum] :limit
+  #   Primary Key position at which to stop copying chunks
   # @option options [Boolean] :atomic_switch
   #   Use atomic switch to rename tables (defaults to: true)
   #   If using a version of mysql affected by atomic switch bug, LHM forces user
@@ -33,26 +43,81 @@ module Lhm
   # @yield [Migrator] Yielded Migrator object records the changes
   # @return [Boolean] Returns true if the migration finishes
   # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
-  def self.change_table(table_name, options = {}, &block)
-    connection = Connection.new(adapter)
-
+  def change_table(table_name, options = {}, &block)
     origin = Table.parse(table_name, connection)
     invoker = Invoker.new(origin, connection)
     block.call(invoker.migrator)
     invoker.run(options)
-
     true
   end
 
-  def self.setup(adapter)
+  # Cleanup tables and triggers
+  #
+  # @param [Boolean] run execute now or just display information
+  # @param [Hash] options Optional options to alter cleanup behaviour
+  # @option options [Time] :until
+  #   Filter to only remove tables up to specified time (defaults to: nil)
+  def cleanup(run = false, options = {})
+    lhm_tables = connection.select_values('show tables').select { |name| name =~ /^lhm(a|n)_/ }
+    if options[:until]
+      lhm_tables.select! { |table|
+        table_date_time = Time.strptime(table, 'lhma_%Y_%m_%d_%H_%M_%S')
+        table_date_time <= options[:until]
+      }
+    end
+
+    lhm_triggers = connection.select_values('show triggers').collect do |trigger|
+      trigger.respond_to?(:trigger) ? trigger.trigger : trigger
+    end.select { |name| name =~ /^lhmt/ }
+
+    if run
+      lhm_triggers.each do |trigger|
+        connection.execute("drop trigger if exists #{trigger}")
+      end
+      lhm_tables.each do |table|
+        connection.execute("drop table if exists #{table}")
+      end
+      true
+    elsif lhm_tables.empty? && lhm_triggers.empty?
+      puts 'Everything is clean. Nothing to do.'
+      true
+    else
+      puts "Existing LHM backup tables: #{lhm_tables.join(', ')}."
+      puts "Existing LHM triggers: #{lhm_triggers.join(', ')}."
+      puts 'Run Lhm.cleanup(true) to drop them all.'
+      false
+    end
+  end
+
+  def setup(adapter)
     @@adapter = adapter
   end
 
-  def self.adapter
+  def adapter
     @@adapter ||=
       begin
         raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
         ActiveRecord::Base.connection
       end
   end
+
+  def self.logger=(new_logger)
+    @@logger = new_logger
+  end
+
+  def self.logger
+    @@logger ||=
+      begin
+        logger = Logger.new(DEFAULT_LOGGER_OPTIONS[:file])
+        logger.level = DEFAULT_LOGGER_OPTIONS[:level]
+        logger.formatter = nil
+        logger
+      end
+  end
+
+  protected
+
+  def connection
+    Connection.new(adapter)
+  end
 end

data/spec/.lhm.example

@@ -1,4 +1,4 @@
 mysqldir=/usr/local/mysql
-basedir=/opt/lhm-cluster
+basedir=~/lhm-cluster
 master_port=3306
 slave_port=3307

data/spec/README.md

@@ -1,47 +1,54 @@
-Preparing for master slave integration tests
---------------------------------------------
+# Preparing for master slave integration tests
 
-# configuration
+## Configuration
 
 create ~/.lhm:
 
     mysqldir=/usr/local/mysql
-    basedir=/opt/lhm-cluster
+    basedir=~/lhm-cluster
     master_port=3306
     slave_port=3307
 
 mysqldir specifies the location of your mysql install. basedir is the
 directory master and slave databases will get installed into.
 
-# setup
+## Automatic setup
+
+### Run
+
+    bin/lhm-spec-clobber.sh
 
 You can set the integration specs up to run against a master slave setup by
-running the included `bin/lhm-spec-clobber.sh` script. this deletes the configured
-lhm master slave setup and reinstalls and configures a master slave setup.
+running the included that. This deletes the configured lhm master slave setup and reinstalls and configures a master slave setup.
 
 Follow the manual instructions if you want more control over this process.
 
-# manual setup
+## Manual setup
 
-## set up instances
+### set up instances
 
     bin/lhm-spec-setup-cluster.sh
 
-## start instances
+### start instances
 
     basedir=/opt/lhm-luster
     mysqld --defaults-file="$basedir/master/my.cnf"
     mysqld --defaults-file="$basedir/slave/my.cnf"
 
-## run the grants
+### run the grants
 
     bin/lhm-spec-grants.sh
 
 ## run specs
 
-To run specs in slave mode, set the SLAVE=1 when running tests:
+Setup the dependency gems
+
+    export BUNDLE_GEMFILE=gemfiles/ar-3.2_mysql2.gemfile
+    bundle install
+
+To run specs in slave mode, set the MASTER_SLAVE=1 when running tests:
 
-    MASTER_SLAVE=1 rake specs
+    MASTER_SLAVE=1 bundle exec rake specs
 
 # connecting
 

data/spec/fixtures/lines.ddl

@@ -0,0 +1,7 @@
+CREATE TABLE `lines` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `between` varchar(10),
+  `lines` int(11),
+  `key` varchar(10),
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/permissions.ddl

@@ -0,0 +1,5 @@
+CREATE TABLE `permissions` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `track_id` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/tracks.ddl

@@ -0,0 +1,5 @@
+CREATE TABLE `tracks` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `public` int(4) DEFAULT 0,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/users.ddl

@@ -2,11 +2,13 @@ CREATE TABLE `users` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `reference` int(11) DEFAULT NULL,
   `username` varchar(255) DEFAULT NULL,
-  `group` varchar(255) DEFAULT NULL,
+  `group` varchar(255) DEFAULT 'Superfriends',
   `created_at` datetime DEFAULT NULL,
   `comment` varchar(20) DEFAULT NULL,
   `description` text,
   PRIMARY KEY (`id`),
   UNIQUE KEY `index_users_on_reference` (`reference`),
-  KEY `index_users_on_username_and_created_at` (`username`,`created_at`)
+  KEY `index_users_on_username_and_created_at` (`username`,`created_at`),
+  KEY `index_with_a_custom_name` (`username`,`group`)
+
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/integration/atomic_switcher_spec.rb

@@ -12,30 +12,30 @@ describe Lhm::AtomicSwitcher do
 
   before(:each) { connect_master! }
 
-  describe "switching" do
+  describe 'switching' do
     before(:each) do
-      @origin      = table_create("origin")
-      @destination = table_create("destination")
+      @origin      = table_create('origin')
+      @destination = table_create('destination')
       @migration   = Lhm::Migration.new(@origin, @destination)
     end
 
-    it "rename origin to archive" do
+    it 'rename origin to archive' do
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
       switcher.run
 
       slave do
         table_exists?(@origin).must_equal true
-        table_read(@migration.archive_name).columns.keys.must_include "origin"
+        table_read(@migration.archive_name).columns.keys.must_include 'origin'
       end
     end
 
-    it "rename destination to origin" do
+    it 'rename destination to origin' do
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
       switcher.run
 
       slave do
         table_exists?(@destination).must_equal false
-        table_read(@origin.name).columns.keys.must_include "destination"
+        table_read(@origin.name).columns.keys.must_include 'destination'
       end
     end
   end

data/spec/integration/chunker_spec.rb

@@ -2,8 +2,6 @@
 # Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
-
-require 'lhm'
 require 'lhm/table'
 require 'lhm/migration'
 
@@ -12,21 +10,29 @@ describe Lhm::Chunker do
 
   before(:each) { connect_master! }
 
-  describe "copying" do
+  describe 'copying' do
     before(:each) do
       @origin = table_create(:origin)
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
     end
 
-    it "should copy 23 rows from origin to destination" do
+    it 'should copy 23 rows from origin to destination' do
       23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
 
-      Lhm::Chunker.new(@migration, connection, { :stride => 100 }).run
+      printer = MiniTest::Mock.new
+      5.times { printer.expect(:notify, :return_value, [Fixnum, Fixnum]) }
+      printer.expect(:end, :return_value, [])
+
+      Lhm::Chunker.new(
+        @migration, connection, { :throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer }
+      ).run
 
       slave do
         count_all(@destination.name).must_equal(23)
       end
+
+     printer.verify
     end
   end
 end

data/spec/integration/cleanup_spec.rb

@@ -0,0 +1,72 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+describe Lhm, 'cleanup' do
+  include IntegrationHelper
+  before(:each) { connect_master! }
+
+  describe 'changes' do
+    before(:each) do
+      table_create(:users)
+      simulate_failed_migration do
+        Lhm.change_table(:users, :atomic_switch => false) do |t|
+          t.add_column(:logins, "INT(12) DEFAULT '0'")
+          t.add_index(:logins)
+        end
+      end
+    end
+
+    after(:each) do
+      Lhm.cleanup(true)
+    end
+
+    it 'should show temporary tables' do
+      output = capture_stdout do
+        Lhm.cleanup
+      end
+      output.must_include('Existing LHM backup tables')
+      output.must_match(/lhma_[0-9_]*_users/)
+    end
+
+    it 'should show temporary tables within range' do
+      table = OpenStruct.new(:name => 'users')
+      table_name = Lhm::Migration.new(table, nil, nil, {}, Time.now - 172800).archive_name
+      table_rename(:users, table_name)
+
+      output = capture_stdout do
+        Lhm.cleanup false, { :until => Time.now - 86400 }
+      end
+      output.must_include('Existing LHM backup tables')
+      output.must_match(/lhma_[0-9_]*_users/)
+    end
+
+    it 'should exclude temporary tables outside range' do
+      table = OpenStruct.new(:name => 'users')
+      table_name = Lhm::Migration.new(table, nil, nil, {}, Time.now).archive_name
+      table_rename(:users, table_name)
+
+      output = capture_stdout do
+        Lhm.cleanup false, { :until => Time.now - 172800 }
+      end
+      output.must_include('Existing LHM backup tables')
+      output.wont_match(/lhma_[0-9_]*_users/)
+    end
+
+    it 'should show temporary triggers' do
+      output = capture_stdout do
+        Lhm.cleanup
+      end
+      output.must_include('Existing LHM triggers')
+      output.must_include('lhmt_ins_users')
+      output.must_include('lhmt_del')
+      output.must_include('lhmt_upd_users')
+    end
+
+    it 'should delete temporary tables' do
+      Lhm.cleanup(true).must_equal(true)
+      Lhm.cleanup.must_equal(true)
+    end
+  end
+end

data/spec/integration/entangler_spec.rb

@@ -12,25 +12,25 @@ describe Lhm::Entangler do
 
   before(:each) { connect_master! }
 
-  describe "entanglement" do
+  describe 'entanglement' do
     before(:each) do
-      @origin = table_create("origin")
-      @destination = table_create("destination")
+      @origin = table_create('origin')
+      @destination = table_create('destination')
       @migration = Lhm::Migration.new(@origin, @destination)
       @entangler = Lhm::Entangler.new(@migration, connection)
     end
 
-    it "should replay inserts from origin into destination" do
+    it 'should replay inserts from origin into destination' do
       @entangler.run do |entangler|
         execute("insert into origin (common) values ('inserted')")
       end
 
       slave do
-        count(:destination, "common", "inserted").must_equal(1)
+        count(:destination, 'common', 'inserted').must_equal(1)
       end
     end
 
-    it "should replay deletes from origin into destination" do
+    it 'should replay deletes from origin into destination' do
       execute("insert into origin (common) values ('inserted')")
 
       @entangler.run do |entangler|
@@ -38,28 +38,28 @@ describe Lhm::Entangler do
       end
 
       slave do
-        count(:destination, "common", "inserted").must_equal(0)
+        count(:destination, 'common', 'inserted').must_equal(0)
       end
     end
 
-    it "should replay updates from origin into destination" do
+    it 'should replay updates from origin into destination' do
       @entangler.run do |entangler|
         execute("insert into origin (common) values ('inserted')")
         execute("update origin set common = 'updated'")
       end
 
       slave do
-        count(:destination, "common", "updated").must_equal(1)
+        count(:destination, 'common', 'updated').must_equal(1)
       end
     end
 
-    it "should remove entanglement" do
+    it 'should remove entanglement' do
       @entangler.run {}
 
       execute("insert into origin (common) values ('inserted')")
 
       slave do
-        count(:destination, "common", "inserted").must_equal(0)
+        count(:destination, 'common', 'inserted').must_equal(0)
       end
     end
   end