lhm 1.0.0.rc.1

data/lib/lhm/intersection.rb

@@ -0,0 +1,42 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+#  Determine and format columns common to origin and destination.
+#
+
+module Lhm
+  class Intersection
+    def initialize(origin, destination)
+      @origin = origin
+      @destination = destination
+    end
+
+    def common
+      @origin.columns.keys & @destination.columns.keys
+    end
+
+    def escaped
+      common.map { |name| tick(name)  }
+    end
+
+    def joined
+      escaped.join(", ")
+    end
+
+    def typed(type)
+      common.map { |name| qualified(name, type)  }.join(", ")
+    end
+
+  private
+
+    def qualified(name, type)
+      "#{ type }.`#{ name }`"
+    end
+
+    def tick(name)
+      "`#{ name }`"
+    end
+  end
+end
+

data/lib/lhm/invoker.rb

@@ -0,0 +1,37 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+#  Copies an origin table to an altered destination table. Live activity is
+#  synchronized into the destination table using triggers.
+#
+#  Once the origin and destination tables have converged, origin is archived
+#  and replaced by destination.
+#
+
+require 'lhm/chunker'
+require 'lhm/entangler'
+require 'lhm/locked_switcher'
+require 'lhm/migration'
+require 'lhm/migrator'
+
+module Lhm
+  class Invoker
+    attr_reader :migrator
+
+    def initialize(origin, connection)
+      @connection = connection
+      @migrator = Migrator.new(origin, connection)
+    end
+
+    def run(chunk_options = {})
+      migration = @migrator.run
+
+      Entangler.new(migration, @connection).run do |tangler|
+        Chunker.new(migration, tangler.epoch, @connection, chunk_options).run
+        LockedSwitcher.new(migration, @connection).run
+      end
+    end
+  end
+end
+

data/lib/lhm/locked_switcher.rb

@@ -0,0 +1,78 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+#  Switches origin with destination table with a write lock. Use this as a safe
+#  alternative to rename, which can cause slave inconsistencies:
+#
+#    http://bugs.mysql.com/bug.php?id=39675
+#
+#  LockedSwitcher adopts the Facebook strategy, with the following caveat:
+#
+#    "Since alter table causes an implicit commit in innodb, innodb locks get
+#    released after the first alter table. So any transaction that sneaks in
+#    after the first alter table and before the second alter table gets
+#    a 'table not found' error. The second alter table is expected to be very
+#    fast though because copytable is not visible to other transactions and so
+#    there is no need to wait."
+#
+
+require 'lhm/command'
+require 'lhm/migration'
+
+module Lhm
+  class LockedSwitcher
+    include Command
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def statements
+      uncommitted { switch }
+    end
+
+    def switch
+      [
+        "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
+        "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
+        "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
+        "commit",
+        "unlock tables"
+      ]
+    end
+
+    def uncommitted(&block)
+      [
+        "set @lhm_auto_commit = @@session.autocommit",
+        "set session autocommit = 0",
+        *yield,
+        "set session autocommit = @lhm_auto_commit"
+      ]
+    end
+
+    #
+    # Command interface
+    #
+
+    def validate
+      unless table?(@origin.name) && table?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+    def revert
+      sql "unlock tables"
+    end
+
+  private
+
+    def execute
+      sql statements
+    end
+  end
+end
+

data/lib/lhm/migration.rb

@@ -0,0 +1,34 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+
+require 'lhm/table'
+require 'lhm/intersection'
+
+module Lhm
+  class Migration
+    attr_reader :origin, :destination
+
+    def initialize(origin, destination, time = Time.now)
+      @origin = origin
+      @destination = destination
+      @start = time
+    end
+
+    def archive_name
+      "lhma_#{ startstamp }_#{ @origin.name }"
+    end
+
+    def intersection
+      Intersection.new(@origin, @destination)
+    end
+
+  private
+
+    def startstamp
+      @start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
+    end
+  end
+end
+

data/lib/lhm/migrator.rb

@@ -0,0 +1,125 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+#  Copies existing schema and applies changes using alter on the empty table.
+#  `run` returns a Migration which can be used for the remaining process.
+#
+
+require 'lhm/command'
+require 'lhm/migration'
+
+module Lhm
+  class Migrator
+    include Command
+
+    attr_reader :name, :statements
+
+    def initialize(table, connection = nil)
+      @connection = connection
+      @origin = table
+      @name = table.destination_name
+      @statements = []
+    end
+
+    def ddl(statement)
+      statements << statement
+    end
+
+    #
+    # Add a column to a table:
+    #
+    #   hadron_change_table("users") do |t|
+    #     t.add_column(:logins, "INT(12) DEFAULT '0'")
+    #   end
+    #
+
+    def add_column(name, definition = "")
+      ddl = "alter table `%s` add column `%s` %s" % [@name, name, definition]
+      statements << ddl.strip
+    end
+
+    #
+    # Remove a column from a table:
+    #
+    #   hadron_change_table("users") do |t|
+    #     t.remove_column(:comment)
+    #   end
+    #
+
+    def remove_column(name)
+      ddl = "alter table `%s` drop `%s`" % [@name, name]
+      statements << ddl.strip
+    end
+
+    #
+    # Add an index to a table:
+    #
+    #  hadron_change_table("users") do |t|
+    #    t.add_index([:comment, :created_at])
+    #  end
+    #
+
+    def add_index(cols)
+      ddl = "create index `%s` on %s" % [@origin.idx_name(cols), idx_spec(cols)]
+      statements << ddl.strip
+    end
+
+    #
+    # Remove an index from a table
+    #
+    #   hadron_change_table("users") do |t|
+    #     t.remove_index(:username, :created_at)
+    #   end
+    #
+
+    def remove_index(*cols)
+      ddl = "drop index `%s` on `%s`" % [@origin.idx_name(cols), @name]
+      statements << ddl.strip
+    end
+
+    #
+    # Command implementation
+    #
+
+    def validate
+      unless table?(@origin.name)
+        error("could not find origin table #{ @origin.name }")
+      end
+
+      unless @origin.satisfies_primary_key?
+        error("origin does not satisfy primary key requirements")
+      end
+
+      dest = @origin.destination_name
+
+      if table?(dest)
+        error("#{ dest } should not exist; not cleaned up from previous run?")
+      end
+    end
+
+  private
+
+    def execute
+      destination_create
+      sql(@statements)
+      Migration.new(@origin, destination_read)
+    end
+
+    def destination_create
+      original = "CREATE TABLE `#{ @origin.name }`"
+      replacement = "CREATE TABLE `#{ @origin.destination_name }`"
+
+      sql(@origin.ddl.gsub(original, replacement))
+    end
+
+    def destination_read
+      Table.parse(@origin.destination_name, connection)
+    end
+
+    def idx_spec(cols)
+      "#{ @name }(#{ [*cols].map(&:to_s).join(', ') })"
+    end
+  end
+end
+

data/lib/lhm/table.rb

@@ -0,0 +1,87 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+
+module Lhm
+  class Table
+    attr_reader :name, :columns, :indices, :pk, :ddl
+
+    def initialize(name, pk = "id", ddl = nil)
+      @name = name
+      @columns = {}
+      @indices = {}
+      @pk = pk
+      @ddl = ddl
+    end
+
+    def satisfies_primary_key?
+      @pk == "id"
+    end
+
+    def destination_name
+      "lhmn_#{ @name }"
+    end
+
+    def idx_name(cols)
+      "index_#{ @name }_on_" + [*cols].join("_and_")
+    end
+
+    def self.parse(table_name, connection)
+      sql = "show create table `#{ table_name }`"
+      ddl = connection.execute(sql).fetch_row.last
+
+      Parser.new(ddl).parse
+    end
+
+    class Parser
+      def initialize(ddl)
+        @ddl = ddl
+      end
+
+      def lines
+        @ddl.lines.to_a.map(&:strip).reject(&:empty?)
+      end
+
+      def create_definitions
+        lines[1..-2]
+      end
+
+      def parse
+        _, name = *lines.first.match("`([^ ]*)`")
+        pk_line = create_definitions.grep(primary).first
+
+        if pk_line
+          _, pk = *pk_line.match(primary)
+          table = Table.new(name, pk, @ddl)
+
+          create_definitions.each do |definition|
+            case definition
+              when index
+                table.indices[$1] = { :metadata => $2 }
+              when column
+                table.columns[$1] = { :type => $2, :metadata => $3 }
+            end
+          end
+
+          table
+        end
+      end
+
+    private
+
+      def primary
+        /^PRIMARY KEY (?:USING (?:HASH|[BR]TREE) )?\(`([^ ]*)`\),?$/
+      end
+
+      def index
+        /^(?:UNIQUE )?(?:INDEX|KEY) `([^ ]*)` (.*?),?$/
+      end
+
+      def column
+        /^`([^ ]*)` ([^ ]*) (.*?),?$/
+      end
+    end
+  end
+end
+

data/spec/bootstrap.rb

@@ -0,0 +1,16 @@
+#
+#  Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+#  Schmidt
+#
+
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'minitest/mock'
+require "pathname"
+
+$project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
+$spec = $project.join("spec")
+$fixtures = $spec.join("fixtures")
+
+$: << $project.join("lib").to_s
+

data/spec/fixtures/destination.ddl

@@ -0,0 +1,7 @@
+CREATE TABLE `destination` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `destination` int(11) DEFAULT NULL,
+  `common` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+

data/spec/fixtures/origin.ddl

@@ -0,0 +1,7 @@
+CREATE TABLE `origin` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `origin` int(11) DEFAULT NULL,
+  `common` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+

data/spec/fixtures/users.ddl

@@ -0,0 +1,11 @@
+CREATE TABLE `users` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `reference` int(11) DEFAULT NULL,
+  `username` varchar(255) DEFAULT NULL,
+  `created_at` datetime DEFAULT NULL,
+  `comment` varchar(20) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_users_on_reference` (`reference`),
+  KEY `index_users_on_username_and_created_at` (`username`,`created_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+