lhm 1.0.0.rc2 → 1.0.0.rc3

data/lib/lhm/intersection.rb

@@ -1,11 +1,8 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
-#  Determine and format columns common to origin and destination.
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 module Lhm
+  #  Determine and format columns common to origin and destination.
   class Intersection
     def initialize(origin, destination)
       @origin = origin
@@ -39,4 +36,3 @@ module Lhm
     end
   end
 end
-

data/lib/lhm/invoker.rb

@@ -1,21 +1,17 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
-#  Copies an origin table to an altered destination table. Live activity is
-#  synchronized into the destination table using triggers.
-#
-#  Once the origin and destination tables have converged, origin is archived
-#  and replaced by destination.
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require 'lhm/chunker'
 require 'lhm/entangler'
 require 'lhm/locked_switcher'
-require 'lhm/migration'
 require 'lhm/migrator'
 
 module Lhm
+  # Copies an origin table to an altered destination table. Live activity is
+  # synchronized into the destination table using triggers.
+  #
+  # Once the origin and destination tables have converged, origin is archived
+  # and replaced by destination.
   class Invoker
     attr_reader :migrator
 
@@ -27,11 +23,10 @@ module Lhm
     def run(chunk_options = {})
       migration = @migrator.run
 
-      Entangler.new(migration, @connection).run do |tangler|
-        Chunker.new(migration, tangler.epoch, @connection, chunk_options).run
+      Entangler.new(migration, @connection).run do
+        Chunker.new(migration, @connection, chunk_options).run
         LockedSwitcher.new(migration, @connection).run
       end
     end
   end
 end
-

data/lib/lhm/locked_switcher.rb

@@ -1,28 +1,30 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
-#  Switches origin with destination table with a write lock. Use this as a safe
-#  alternative to rename, which can cause slave inconsistencies:
-#
-#    http://bugs.mysql.com/bug.php?id=39675
-#
-#  LockedSwitcher adopts the Facebook strategy, with the following caveat:
-#
-#    "Since alter table causes an implicit commit in innodb, innodb locks get
-#    released after the first alter table. So any transaction that sneaks in
-#    after the first alter table and before the second alter table gets
-#    a 'table not found' error. The second alter table is expected to be very
-#    fast though because copytable is not visible to other transactions and so
-#    there is no need to wait."
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require 'lhm/command'
 require 'lhm/migration'
+require 'lhm/sql_helper'
 
 module Lhm
+  # Switches origin with destination table with a write lock. Use this as
+  # a safe alternative to rename, which can cause slave inconsistencies:
+  #
+  #   http://bugs.mysql.com/bug.php?id=39675
+  #
+  # LockedSwitcher adopts the Facebook strategy, with the following caveat:
+  #
+  #   "Since alter table causes an implicit commit in innodb, innodb locks get
+  #   released after the first alter table. So any transaction that sneaks in
+  #   after the first alter table and before the second alter table gets
+  #   a 'table not found' error. The second alter table is expected to be very
+  #   fast though because copytable is not visible to other transactions and so
+  #   there is no need to wait."
+  #
   class LockedSwitcher
     include Command
+    include SqlHelper
+
+    attr_reader :connection
 
     def initialize(migration, connection = nil)
       @migration = migration
@@ -51,29 +53,23 @@ module Lhm
         "set session autocommit = 0",
         yield,
         "set session autocommit = @lhm_auto_commit"
-
       ].flatten
     end
 
-    #
-    # Command interface
-    #
-
     def validate
       unless table?(@origin.name) && table?(@destination.name)
         error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
       end
     end
 
+  private
+
     def revert
       sql "unlock tables"
     end
 
-  private
-
     def execute
       sql statements
     end
   end
 end
-

data/lib/lhm/migration.rb

@@ -1,9 +1,6 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
-require 'lhm/table'
 require 'lhm/intersection'
 
 module Lhm
@@ -24,11 +21,8 @@ module Lhm
       Intersection.new(@origin, @destination)
     end
 
-  private
-
     def startstamp
       @start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
     end
   end
 end
-

data/lib/lhm/migrator.rb

@@ -1,19 +1,19 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
-#  Copies existing schema and applies changes using alter on the empty table.
-#  `run` returns a Migration which can be used for the remaining process.
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 require 'lhm/command'
 require 'lhm/migration'
+require 'lhm/sql_helper'
+require 'lhm/table'
 
 module Lhm
+  # Copies existing schema and applies changes using alter on the empty table.
+  # `run` returns a Migration which can be used for the remaining process.
   class Migrator
     include Command
+    include SqlHelper
 
-    attr_reader :name, :statements
+    attr_reader :name, :statements, :connection
 
     def initialize(table, connection = nil)
       @connection = connection
@@ -22,78 +22,103 @@ module Lhm
       @statements = []
     end
 
+    # Alter a table with a custom statement
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.ddl("ALTER TABLE #{m.name} ADD COLUMN age INT(11)")
+    #   end
+    #
+    # @param [String] statement SQL alter statement
+    # @note
+    #
+    #   Don't write the table name directly into the statement. Use the #name
+    #   getter instead, because the alter statement will be executed against a
+    #   temporary table.
+    #
     def ddl(statement)
       statements << statement
     end
 
+    # Add a column to a table
     #
-    # Add a column to a table:
+    # @example
     #
-    #   hadron_change_table("users") do |t|
-    #     t.add_column(:logins, "INT(12) DEFAULT '0'")
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_column(:comment, "VARCHAR(12) DEFAULT '0'")
     #   end
     #
-
-    def add_column(name, definition = "")
-      ddl = "alter table `%s` add column `%s` %s" % [@name, name, definition]
-      statements << ddl.strip
+    # @param [String] name Name of the column to add
+    # @param [String] definition Valid SQL column definition
+    def add_column(name, definition)
+      ddl("alter table `%s` add column `%s` %s" % [@name, name, definition])
     end
 
+    # Remove a column from a table
     #
-    # Remove a column from a table:
+    # @example
     #
-    #   hadron_change_table("users") do |t|
-    #     t.remove_column(:comment)
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_column(:comment)
     #   end
     #
-
+    # @param [String] name Name of the column to delete
     def remove_column(name)
-      ddl = "alter table `%s` drop `%s`" % [@name, name]
-      statements << ddl.strip
+      ddl("alter table `%s` drop `%s`" % [@name, name])
     end
 
+    # Add an index to a table
     #
-    # Add an index to a table:
+    # @example
     #
-    #  hadron_change_table("users") do |t|
-    #    t.add_index([:comment, :created_at])
-    #  end
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_index(:comment)
+    #     m.add_index([:username, :created_at])
+    #     m.add_index("comment(10)")
+    #   end
     #
-
-    def add_index(cols)
-      ddl = "create index `%s` on %s" % idx_parts(cols)
-      statements << ddl.strip
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    def add_index(columns)
+      ddl(index_ddl(columns))
     end
 
+    # Add a unique index to a table
     #
-    # Add a unique index to a table:
+    # @example
     #
-    #  hadron_change_table("users") do |t|
-    #    t.add_unique_index([:comment, :created_at])
-    #  end
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_unique_index(:comment)
+    #     m.add_unique_index([:username, :created_at])
+    #     m.add_unique_index("comment(10)")
+    #   end
     #
-
-    def add_unique_index(cols)
-      ddl = "create unique index `%s` on %s" % idx_parts(cols)
-      statements << ddl.strip
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    def add_unique_index(columns)
+      ddl(index_ddl(columns, :unique))
     end
 
-    #
     # Remove an index from a table
     #
-    #   hadron_change_table("users") do |t|
-    #     t.remove_index(:username, :created_at)
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_index(:comment)
+    #     m.remove_index([:username, :created_at])
     #   end
     #
-
-    def remove_index(cols)
-      ddl = "drop index `%s` on `%s`" % [@origin.idx_name(cols), @name]
-      statements << ddl.strip
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes.
+    def remove_index(columns)
+      ddl("drop index `%s` on `%s`" % [idx_name(@origin.name, columns), @name])
     end
 
-    #
-    # Command implementation
-    #
+  private
 
     def validate
       unless table?(@origin.name)
@@ -111,8 +136,6 @@ module Lhm
       end
     end
 
-  private
-
     def execute
       destination_create
       sql(@statements)
@@ -130,13 +153,10 @@ module Lhm
       Table.parse(@origin.destination_name, connection)
     end
 
-    def idx_spec(cols)
-      "`#{ @name }` (#{ Array(cols).map(&:to_s).join(', ') })"
-    end
-
-    def idx_parts(cols)
-      [@origin.idx_name(cols), idx_spec(cols)]
+    def index_ddl(cols, unique = nil)
+      type = unique ? "unique index" : "index"
+      parts = [type, idx_name(@origin.name, cols), @name, idx_spec(cols)]
+      "create %s `%s` on `%s` (%s)" % parts
     end
   end
 end
-

data/lib/lhm/sql_helper.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  module SqlHelper
+    extend self
+
+    def idx_name(table_name, cols)
+      column_names = column_definition(cols).map(&:first)
+      "index_#{ table_name }_on_#{ column_names.join("_and_") }"
+    end
+
+    def idx_spec(cols)
+      column_definition(cols).map do |name, length|
+        "`#{name}`#{length}"
+      end.join(', ')
+    end
+
+    def table?(table_name)
+      connection.table_exists?(table_name)
+    end
+
+    def sql(statements)
+      [statements].flatten.each { |statement| connection.execute(statement) }
+    rescue ActiveRecord::StatementInvalid, Mysql::Error => e
+      error e.message
+    end
+
+    def update(statements)
+      [statements].flatten.inject(0) do |memo, statement|
+        memo += connection.update(statement)
+      end
+    rescue ActiveRecord::StatementInvalid, Mysql::Error => e
+      error e.message
+    end
+
+  private
+
+    def column_definition(cols)
+      Array(cols).map do |column|
+        column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
+      end
+    end
+  end
+end

data/lib/lhm/table.rb

@@ -1,7 +1,5 @@
-#
-#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
-#  Schmidt
-#
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
 
 module Lhm
   class Table
@@ -23,11 +21,6 @@ module Lhm
       "lhmn_#{ @name }"
     end
 
-    def idx_name(cols)
-      column_part = Array(cols).map { |c| c.to_s.sub(/\(.*/, "") }.join("_and_")
-      "index_#{ @name }_on_#{ column_part }"
-    end
-
     def self.parse(table_name, connection)
       sql = "show create table `#{ table_name }`"
       ddl = connection.execute(sql).fetch_row.last
@@ -85,4 +78,3 @@ module Lhm
     end
   end
 end
-

data/lib/lhm/version.rb

@@ -0,0 +1,6 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  VERSION = "1.0.0.rc3"
+end

data/spec/README.md

@@ -0,0 +1,26 @@
+Preparing for master slave integration tests
+--------------------------------------------
+
+You can set the integration specs up to run against a master slave setup by
+running the included `setup-cluster` script.
+
+# set up instances
+
+    spec/config/setup-cluster
+
+# start instances
+
+    basedir=/opt/lhm-cluster
+    mysqld --defaults-file="$basedir/master/my.cnf"
+    mysqld --defaults-file="$basedir/slave/my.cnf"
+
+# run the grants
+
+    spec/config/grants
+
+# run specs
+
+To run specs in slave mode, set the SLAVE=1 when running tests:
+
+    MASTER_SLAVE=1 rake specs
+