sbader-lhm 1.1.0

data/lib/lhm/entangler.rb

@@ -0,0 +1,98 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/sql_helper'
+
+module Lhm
+  class Entangler
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    # Creates entanglement between two tables. All creates, updates and deletes
+    # to origin will be repeated on the destination table.
+    def initialize(migration, connection = nil)
+      @common = migration.intersection
+      @origin = migration.origin
+      @destination = migration.destination
+      @connection = connection
+    end
+
+    def entangle
+      [
+        create_delete_trigger,
+        create_insert_trigger,
+        create_update_trigger
+      ]
+    end
+
+    def untangle
+      [
+        "drop trigger if exists `#{ trigger(:del) }`",
+        "drop trigger if exists `#{ trigger(:ins) }`",
+        "drop trigger if exists `#{ trigger(:upd) }`"
+      ]
+    end
+
+    def create_insert_trigger
+      strip %Q{
+        create trigger `#{ trigger(:ins) }`
+        after insert on `#{ @origin.name }` for each row
+        replace into `#{ @destination.name }` (#{ @common.combined_joined }) #{ SqlHelper.annotation }
+        values (#{ @common.combined_typed("NEW") })
+      }
+    end
+
+    def create_update_trigger
+      strip %Q{
+        create trigger `#{ trigger(:upd) }`
+        after update on `#{ @origin.name }` for each row
+        replace into `#{ @destination.name }` (#{ @common.joined }) #{ SqlHelper.annotation }
+        values (#{ @common.typed("NEW") })
+      }
+    end
+
+    def create_delete_trigger
+      strip %Q{
+        create trigger `#{ trigger(:del) }`
+        after delete on `#{ @origin.name }` for each row
+        delete ignore from `#{ @destination.name }` #{ SqlHelper.annotation }
+        where `#{ @destination.name }`.`id` = OLD.`id`
+      }
+    end
+
+    def trigger(type)
+      "lhmt_#{ type }_#{ @origin.name }"
+    end
+
+    def validate
+      unless table?(@origin.name)
+        error("#{ @origin.name } does not exist")
+      end
+
+      unless table?(@destination.name)
+        error("#{ @destination.name } does not exist")
+      end
+    end
+
+    def before
+      sql(entangle)
+    end
+
+    def after
+      sql(untangle)
+    end
+
+    def revert
+      after
+    end
+
+  private
+
+    def strip(sql)
+      sql.strip.gsub(/\n */, "\n")
+    end
+  end
+end

data/lib/lhm/intersection.rb

@@ -0,0 +1,63 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  #  Determine and format columns common to origin and destination.
+  class Intersection
+    def initialize(origin, destination, insert_trigger_additions)
+      @origin = origin
+      @destination = destination
+      @insert_trigger_additions = insert_trigger_additions
+    end
+
+    def common
+      (@origin.columns.keys & @destination.columns.keys).sort
+    end
+
+    def insert
+      @insert_trigger_additions.keys
+    end
+
+    def escaped_insert
+      insert.map { |name| tick(name)  }
+    end
+
+    def combined_joined
+      (escaped + escaped_insert).join(", ")
+    end
+
+    def combined_typed(type)
+      (common.map { |name| qualified(name, type)  } + @insert_trigger_additions.values.map { |name| parenthesize(name) }).join(", ")
+    end
+
+    def escaped
+      common.map { |name| tick(name)  }
+    end
+
+    def joined
+      escaped.join(", ")
+    end
+
+    def typed_unjoined(type)
+      common.map { |name| qualified(name, type)  }
+    end
+
+    def typed(type)
+      common.map { |name| qualified(name, type)  }.join(", ")
+    end
+
+  private
+
+    def qualified(name, type)
+      "#{ type }.`#{ name }`"
+    end
+
+    def tick(name)
+      "`#{ name }`"
+    end
+
+    def parenthesize(name)
+      "(#{ name })"
+    end
+  end
+end

data/lib/lhm/invoker.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/chunker'
+require 'lhm/entangler'
+require 'lhm/atomic_switcher'
+require 'lhm/locked_switcher'
+require 'lhm/migrator'
+
+module Lhm
+  # Copies an origin table to an altered destination table. Live activity is
+  # synchronized into the destination table using triggers.
+  #
+  # Once the origin and destination tables have converged, origin is archived
+  # and replaced by destination.
+  class Invoker
+    include SqlHelper
+
+    attr_reader :migrator, :connection
+
+    def initialize(origin, connection)
+      @connection = connection
+      @migrator = Migrator.new(origin, connection)
+    end
+
+    def run(options = {})
+      if !options.include?(:atomic_switch)
+        if supports_atomic_switch?
+          options[:atomic_switch] = true
+        else
+          raise Error.new(
+            "Using mysql #{version_string}. You must explicitly set " +
+            "options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)")
+        end
+      end
+
+      migration = @migrator.run
+
+      Entangler.new(migration, @connection).run do
+        Chunker.new(migration, @connection, options).run
+        if options[:atomic_switch]
+          AtomicSwitcher.new(migration, @connection).run
+        else
+          LockedSwitcher.new(migration, @connection).run
+        end
+      end
+    end
+  end
+end

data/lib/lhm/locked_switcher.rb

@@ -0,0 +1,71 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+  # Switches origin with destination table nonatomically using a locked write.
+  # LockedSwitcher adopts the Facebook strategy, with the following caveat:
+  #
+  #   "Since alter table causes an implicit commit in innodb, innodb locks get
+  #   released after the first alter table. So any transaction that sneaks in
+  #   after the first alter table and before the second alter table gets
+  #   a 'table not found' error. The second alter table is expected to be very
+  #   fast though because copytable is not visible to other transactions and so
+  #   there is no need to wait."
+  #
+  class LockedSwitcher
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def statements
+      uncommitted { switch }
+    end
+
+    def switch
+      [
+        "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
+        "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
+        "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
+        "commit",
+        "unlock tables"
+      ]
+    end
+
+    def uncommitted(&block)
+      [
+        "set @lhm_auto_commit = @@session.autocommit",
+        "set session autocommit = 0",
+        yield,
+        "set session autocommit = @lhm_auto_commit"
+      ].flatten
+    end
+
+    def validate
+      unless table?(@origin.name) && table?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+  private
+
+    def revert
+      sql "unlock tables"
+    end
+
+    def execute
+      sql statements
+    end
+  end
+end

data/lib/lhm/migration.rb

@@ -0,0 +1,30 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/intersection'
+
+module Lhm
+  class Migration
+    attr_reader :origin, :destination, :insert_joins
+
+    def initialize(origin, destination, time = Time.now, insert_trigger_additions, insert_joins)
+      @origin = origin
+      @destination = destination
+      @start = time
+      @insert_trigger_additions = insert_trigger_additions
+      @insert_joins = insert_joins
+    end
+
+    def archive_name
+      "lhma_#{ startstamp }_#{ @origin.name }"
+    end
+
+    def intersection
+      Intersection.new(@origin, @destination, @insert_trigger_additions)
+    end
+
+    def startstamp
+      @start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
+    end
+  end
+end

data/lib/lhm/migrator.rb

@@ -0,0 +1,219 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+require 'lhm/table'
+
+module Lhm
+  # Copies existing schema and applies changes using alter on the empty table.
+  # `run` returns a Migration which can be used for the remaining process.
+  class Migrator
+    include Command
+    include SqlHelper
+
+    attr_reader :name, :statements, :connection
+
+    def initialize(table, connection = nil)
+      @connection = connection
+      @origin = table
+      @name = table.destination_name
+      @statements = []
+      @insert_trigger_additions = {}
+      @insert_joins = []
+    end
+
+    # Alter a table with a custom statement
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.ddl("ALTER TABLE #{m.name} ADD COLUMN age INT(11)")
+    #   end
+    #
+    # @param [String] statement SQL alter statement
+    # @note
+    #
+    #   Don't write the table name directly into the statement. Use the #name
+    #   getter instead, because the alter statement will be executed against a
+    #   temporary table.
+    #
+    def ddl(statement)
+      statements << statement
+    end
+
+    # Adds joins to the chunked insert. Helpful if you would need to do an update
+    # after the change_table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_column(:comment, "VARCHAR(12) DEFAULT '0'")
+    #     m.join_on_insert(:people, :description, :comment, "people.user_id = users.id")
+    #   end
+    #
+    # @param [String] table Table to join to
+    # @param [String] origin_field Column in the origin (joined) table
+    # @param [String] destination_field Column in the destination table
+    # @param [String] statement Valid sql join statement
+    def join_on_insert(table, origin_field, destination_field, statement)
+      @insert_joins << { :table => table, :origin_field => origin_field, :destination_field => destination_field, :statement => statement }
+    end
+
+    # Adds additional columns to the trigger that is created for inserts.
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_column(:comment, "VARCHAR(12) DEFAULT '0'")
+    #     m.insert_trigger(:comment, "SELECT comment FROM people WHERE NEW.id = people.id")
+    #   end
+    #
+    # @param [String] key Column to insert value
+    # @param [String] statement Valid sql query, can use NEW to reference the row in trigger
+    def insert_trigger(key, statement)
+      @insert_trigger_additions[key] = statement
+    end
+
+    # Add a column to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_column(:comment, "VARCHAR(12) DEFAULT '0'")
+    #   end
+    #
+    # @param [String] name Name of the column to add
+    # @param [String] definition Valid SQL column definition
+    def add_column(name, definition)
+      ddl("alter table `%s` add column `%s` %s" % [@name, name, definition])
+    end
+
+    # Change an existing column to a new definition
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.change_column(:comment, "VARCHAR(12) DEFAULT '0' NOT NULL")
+    #   end
+    #
+    # @param [String] name Name of the column to change
+    # @param [String] definition Valid SQL column definition
+    def change_column(name, definition)
+      ddl("alter table `%s` modify column `%s` %s" % [@name, name, definition])
+    end
+
+    # Remove a column from a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_column(:comment)
+    #   end
+    #
+    # @param [String] name Name of the column to delete
+    def remove_column(name)
+      ddl("alter table `%s` drop `%s`" % [@name, name])
+    end
+
+    # Add an index to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_index(:comment)
+    #     m.add_index([:username, :created_at])
+    #     m.add_index("comment(10)")
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be created
+    def add_index(columns, index_name = nil)
+      ddl(index_ddl(columns, false, index_name))
+    end
+
+    # Add a unique index to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_unique_index(:comment)
+    #     m.add_unique_index([:username, :created_at])
+    #     m.add_unique_index("comment(10)")
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be created
+    def add_unique_index(columns, index_name = nil)
+      ddl(index_ddl(columns, true, index_name))
+    end
+
+    # Remove an index from a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_index(:comment)
+    #     m.remove_index([:username, :created_at])
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be removed
+    def remove_index(columns, index_name = nil)
+      index_name ||= idx_name(@origin.name, columns)
+      ddl("drop index `%s` on `%s`" % [index_name, @name])
+    end
+
+  private
+
+    def validate
+      unless table?(@origin.name)
+        error("could not find origin table #{ @origin.name }")
+      end
+
+      unless @origin.satisfies_primary_key?
+        error("origin does not satisfy primary key requirements")
+      end
+
+      dest = @origin.destination_name
+
+      if table?(dest)
+        error("#{ dest } should not exist; not cleaned up from previous run?")
+      end
+    end
+
+    def execute
+      destination_create
+      sql(@statements)
+      Migration.new(@origin, destination_read, @insert_trigger_additions, @insert_joins)
+    end
+
+    def destination_create
+      original = "CREATE TABLE `#{ @origin.name }`"
+      replacement = "CREATE TABLE `#{ @origin.destination_name }`"
+
+      sql(@origin.ddl.gsub(original, replacement))
+    end
+
+    def destination_read
+      Table.parse(@origin.destination_name, connection)
+    end
+
+    def index_ddl(cols, unique = nil, index_name = nil)
+      type = unique ? "unique index" : "index"
+      index_name ||= idx_name(@origin.name, cols)
+      parts = [type, index_name, @name, idx_spec(cols)]
+      "create %s `%s` on `%s` (%s)" % parts
+    end
+  end
+end