lhm 1.0.0.rc2 → 1.0.0.rc3

data/.config

@@ -0,0 +1,3 @@
+basedir=/opt/lhm-cluster
+master_port=3306
+slave_port=3307

data/.gitignore

@@ -1,5 +1,6 @@
 *.gem
 .bundle
 Gemfile.lock
+gemfiles/*.lock
 pkg/*
 .rvmrc

data/.travis.yml

@@ -1,9 +1,15 @@
+before_script:
+  - "mysql -e 'create database lhm;'"
 rvm:
   - 1.8.7
-  - 1.9.2
   - 1.9.3
-before_script:
-  - "mysql -e 'create database lhm;'"
+gemfile:
+  - gemfiles/ar-2.3.gemfile
+  - gemfiles/ar-3.1.gemfile
+matrix:
+  exclude:
+    - rvm: 1.8.7
+      gemfile: gemfiles/ar-2.3.gemfile
 branches:
   only:
     - master

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# 1.0.0.rc3 (January 19, 2012)
+
+* Speedup migrations for tables with large minimum id
+* Add a bit yard documentation
+* Fix issues with index creation on reserved column names
+* Improve error handling
+* Add tests for replication
+* Rename public API method from `hadron_change_table` to `change_table`
+* Add tests for ActiveRecord 2.3 and 3.1 compatibility
+
 # 1.0.0.rc2 (January 18, 2012)
 
 * Speedup migrations for tables with large ids

data/README.md

@@ -16,7 +16,7 @@ access becomes just as difficult.
 
 There are few things that can be done at the server or engine level. It is
 possible to change default values in an `ALTER TABLE` without locking the
-table.  The InnoDB Plugin provides facilities for online index creation, which
+table. The InnoDB Plugin provides facilities for online index creation, which
 is great if you are using this engine, but only solves half the problem.
 
 At SoundCloud we started having migration pains quite a while ago, and after
@@ -42,40 +42,44 @@ twitter solution [1], it does not require the presence of an indexed
 
 ## Usage
 
-After extending Lhm, `hadron_change_table` becomes available
-with the following methods:
+You can invoke Lhm directly from a plain ruby file after connecting active
+record to your mysql instance:
 
-    class MigrateArbitrary < ActiveRecord::Migration
-      extend Lhm
+    require 'lhm'
+
+    ActiveRecord::Base.establish_connection(
+      :adapter => 'mysql',
+      :host => '127.0.0.1',
+      :database => 'lhm'
+    )
+
+    Lhm.change_table(:users) do |m|
+      m.add_column(:arbitrary, "INT(12)")
+      m.add_index([:arbitrary, :created_at])
+      m.ddl("alter table %s add column flag tinyint(1)" % m.name)
+    end
+
+To use Lhm from an ActiveRecord::Migration in a Rails project, add it to your
+Gemfile, then invoke as follows:
+
+    class MigrateUsers < ActiveRecord::Migration
 
       def self.up
-        hadron_change_table(:users) do |t|
-          t.add_column(:arbitrary, "INT(12)")
-          t.add_index([:arbitrary, :created_at])
-          t.ddl("alter table %s add column flag tinyint(1)" % t.name)
+        Lhm.change_table(:users) do |m|
+          m.add_column(:arbitrary, "INT(12)")
+          m.add_index([:arbitrary, :created_at])
+          m.ddl("alter table %s add column flag tinyint(1)" % m.name)
         end
       end
 
       def self.down
-        hadron_change_table(:users) do |t|
-          t.remove_index([:arbitrary, :created_at])
-          t.remove_column(:arbitrary)
+        Lhm.change_table(:users) do |m|
+          m.remove_index([:arbitrary, :created_at])
+          m.remove_column(:arbitrary)
         end
       end
     end
 
-## Migration phases
-
-_TODO_
-
-### When adding a column
-
-_TODO_
-
-### When removing a column
-
-_TODO_
-
 ## Contributing
 
 We'll check out your contribution if you:

data/Rakefile

@@ -15,5 +15,6 @@ Rake::TestTask.new("integration") do |t|
   t.verbose = true
 end
 
-task :default => [:unit, :integration]
+task :specs => [:unit, :integration]
+task :default => :specs
 

data/gemfiles/ar-2.3.gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem "activerecord", "~> 2.3.14"
+gemspec :path=>"../"

data/gemfiles/ar-3.1.gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem "activerecord", "~> 3.1.3"
+gemspec :path=>"../"

data/lhm.gemspec

@@ -3,7 +3,7 @@
 lib = File.expand_path('../lib', __FILE__)
 $:.unshift(lib) unless $:.include?(lib)
 
-require 'lhm'
+require 'lhm/version'
 
 Gem::Specification.new do |s|
   s.name          = "lhm"

data/lib/lhm.rb

@@ -1,20 +1,42 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
+require 'active_record'
 require 'lhm/table'
 require 'lhm/invoker'
-require 'lhm/migration'
+require 'lhm/version'
 
+# Large hadron migrator - online schema change tool
+#
+# @example
+#
+#   Lhm.change_table(:users) do |m|
+#     m.add_column(:arbitrary, "INT(12)")
+#     m.add_index([:arbitrary, :created_at])
+#     m.ddl("alter table %s add column flag tinyint(1)" % m.name)
+#   end
+#
 module Lhm
-  VERSION = "1.0.0.rc2"
+  extend self
 
-  def hadron_change_table(table_name, chunk_options = {}, &block)
+  # Alters a table with the changes described in the block
+  #
+  # @param [String, Symbol] table_name Name of the table
+  # @param [Hash] chunk_options Optional options to alter the chunk behavior
+  # @option chunk_options [Fixnum] :stride
+  #   Size of a chunk (defaults to: 40,000)
+  # @option chunk_options [Fixnum] :throttle
+  #   Time to wait between chunks in milliseconds (defaults to: 100)
+  # @yield [Migrator] Yielded Migrator object records the changes
+  # @return [Boolean] Returns true if the migration finishs
+  # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
+  def change_table(table_name, chunk_options = {}, &block)
+    connection = ActiveRecord::Base.connection
     origin = Table.parse(table_name, connection)
     invoker = Invoker.new(origin, connection)
     block.call(invoker.migrator)
     invoker.run(chunk_options)
+
+    true
   end
 end
-

data/lib/lhm/chunker.rb

@@ -1,64 +1,88 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
-require 'lhm/migration'
 require 'lhm/command'
+require 'lhm/sql_helper'
 
 module Lhm
   class Chunker
     include Command
+    include SqlHelper
+
+    attr_reader :connection
 
-    #
     # Copy from origin to destination in chunks of size `stride`. Sleeps for
     # `throttle` milliseconds between each stride.
-    #
-
-    def initialize(migration, limit = 1, connection = nil, options = {})
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
       @stride = options[:stride] || 40_000
       @throttle = options[:throttle] || 100
-      @limit = limit
-      @connection = connection
-      @migration = migration
+      @start = options[:start] || select_start
+      @limit = options[:limit] || select_limit
     end
 
-    #
-    # Copies chunks of size `stride`, starting from id 1 up to id `limit`.
-    #
-
-    def up_to(limit)
-      traversable_chunks_up_to(limit).times do |n|
-        yield(bottom(n + 1), top(n + 1, limit))
+    # Copies chunks of size `stride`, starting from `start` up to id `limit`.
+    def up_to(&block)
+      1.upto(traversable_chunks_size) do |n|
+        yield(bottom(n), top(n))
       end
     end
 
-    def traversable_chunks_up_to(limit)
-      (limit / @stride.to_f).ceil
+    def traversable_chunks_size
+      @limit && @start ? ((@limit - @start + 1) / @stride.to_f).ceil : 0
     end
 
     def bottom(chunk)
-      (chunk - 1) * @stride + 1
+      (chunk - 1) * @stride + @start
     end
 
-    def top(chunk, limit)
-      [chunk * @stride, limit].min
+    def top(chunk)
+      [chunk * @stride + @start - 1, @limit].min
     end
 
     def copy(lowest, highest)
-      "insert ignore into `#{ @migration.destination.name }` (#{ cols.joined  }) " +
-      "select #{ cols.joined } from `#{ @migration.origin.name }` " +
+      "insert ignore into `#{ destination_name }` (#{ columns }) " +
+      "select #{ columns } from `#{ origin_name }` " +
       "where `id` between #{ lowest } and #{ highest }"
     end
 
+    def select_start
+      start = connection.select_value("select min(id) from #{ origin_name }")
+      start ? start.to_i : nil
+    end
+
+    def select_limit
+      limit = connection.select_value("select max(id) from #{ origin_name }")
+      limit ? limit.to_i : nil
+    end
+
+    def throttle_seconds
+      @throttle / 1000.0
+    end
+
   private
 
-    def cols
-      @cols ||= @migration.intersection
+    def destination_name
+      @migration.destination.name
+    end
+
+    def origin_name
+      @migration.origin.name
+    end
+
+    def columns
+      @columns ||= @migration.intersection.joined
+    end
+
+    def validate
+      if @start && @limit && @start > @limit
+        error("impossible chunk options (limit must be greater than start)")
+      end
     end
 
     def execute
-      up_to(@limit) do |lowest, highest|
+      up_to do |lowest, highest|
         affected_rows = update(copy(lowest, highest))
 
         if affected_rows > 0
@@ -68,10 +92,5 @@ module Lhm
         print "."
       end
     end
-
-    def throttle_seconds
-      @throttle / 1000.0
-    end
   end
 end
-

data/lib/lhm/command.rb

@@ -1,24 +1,11 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
-#  Apply a change to the database.
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 module Lhm
-  module Command
-    def self.included(base)
-      base.send :attr_reader, :connection
-    end
-
-    #
-    # Command Interface
-    #
-
-    def validate; end
-
-    def revert; end
+  class Error < StandardError
+  end
 
+  module Command
     def run(&block)
       validate
 
@@ -29,45 +16,31 @@ module Lhm
       else
         execute
       end
+    rescue
+      revert
+      raise
     end
 
   private
 
-    def execute
-      raise NotImplementedError.new(self.class.name)
+    def validate
     end
 
-    def before
-      raise NotImplementedError.new(self.class.name)
+    def revert
     end
 
-    def after
+    def execute
       raise NotImplementedError.new(self.class.name)
     end
 
-    def table?(table_name)
-      @connection.table_exists?(table_name)
-    end
-
-    def error(msg)
-      raise Exception.new("#{ self.class }: #{ msg }")
+    def before
     end
 
-    def sql(statements)
-      [statements].flatten.each { |statement| @connection.execute(statement) }
-    rescue ActiveRecord::StatementInvalid, Mysql::Error => e
-      revert
-      error e.message
+    def after
     end
 
-    def update(statements)
-      [statements].flatten.inject(0) do |memo, statement|
-        memo += @connection.update(statement)
-      end
-    rescue ActiveRecord::StatementInvalid, Mysql::Error => e
-      revert
-      error e.message
+    def error(msg)
+      raise Error.new(msg)
     end
   end
 end
-

data/lib/lhm/entangler.rb

@@ -1,19 +1,18 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
-#  Creates entanglement between two tables. All creates, updates and deletes
-#  to origin will be repeated on the the destination table.
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require 'lhm/command'
+require 'lhm/sql_helper'
 
 module Lhm
   class Entangler
     include Command
+    include SqlHelper
 
-    attr_reader :epoch
+    attr_reader :connection
 
+    # Creates entanglement between two tables. All creates, updates and deletes
+    # to origin will be repeated on the destination table.
     def initialize(migration, connection = nil)
       @common = migration.intersection
       @origin = migration.origin
@@ -23,9 +22,9 @@ module Lhm
 
     def entangle
       [
-        create_trigger_del,
-        create_trigger_ins,
-        create_trigger_upd
+        create_delete_trigger,
+        create_insert_trigger,
+        create_update_trigger
       ]
     end
 
@@ -37,7 +36,7 @@ module Lhm
       ]
     end
 
-    def create_trigger_ins
+    def create_insert_trigger
       strip %Q{
         create trigger `#{ trigger(:ins) }`
         after insert on `#{ @origin.name }` for each row
@@ -46,7 +45,7 @@ module Lhm
       }
     end
 
-    def create_trigger_upd
+    def create_update_trigger
       strip %Q{
         create trigger `#{ trigger(:upd) }`
         after update on `#{ @origin.name }` for each row
@@ -55,7 +54,7 @@ module Lhm
       }
     end
 
-    def create_trigger_del
+    def create_delete_trigger
       strip %Q{
         create trigger `#{ trigger(:del) }`
         after delete on `#{ @origin.name }` for each row
@@ -68,10 +67,6 @@ module Lhm
       "lhmt_#{ type }_#{ @origin.name }"
     end
 
-    #
-    # Command implementation
-    #
-
     def validate
       unless table?(@origin.name)
         error("#{ @origin.name } does not exist")
@@ -84,7 +79,6 @@ module Lhm
 
     def before
       sql(entangle)
-      @epoch = connection.select_value("select max(id) from #{ @origin.name }").to_i
     end
 
     def after
@@ -102,4 +96,3 @@ module Lhm
     end
   end
 end
-