lhm-teak 3.6.0

data/lib/lhm/entangler.rb

@@ -0,0 +1,112 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/sql_helper'
+require 'lhm/sql_retry'
+require 'lhm/connection'
+
+module Lhm
+  class Entangler
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    LOG_PREFIX = "Entangler"
+
+    # Creates entanglement between two tables. All creates, updates and deletes
+    # to origin will be repeated on the destination table.
+    def initialize(migration, connection = nil)
+      @intersection = migration.intersection
+      @origin = migration.origin
+      @destination = migration.destination
+      @connection = connection
+    end
+
+    def entangle
+      [
+        create_delete_trigger,
+        create_insert_trigger,
+        create_update_trigger
+      ]
+    end
+
+    def untangle
+      [
+        "drop trigger if exists `#{ trigger(:del) }`",
+        "drop trigger if exists `#{ trigger(:ins) }`",
+        "drop trigger if exists `#{ trigger(:upd) }`"
+      ]
+    end
+
+    def create_insert_trigger
+      strip %Q{
+        create trigger `#{ trigger(:ins) }`
+        after insert on `#{ @origin.name }` for each row
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
+      }
+    end
+
+    def create_update_trigger
+      strip %Q{
+        create trigger `#{ trigger(:upd) }`
+        after update on `#{ @origin.name }` for each row
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
+      }
+    end
+
+    def create_delete_trigger
+      strip %Q{
+        create trigger `#{ trigger(:del) }`
+        after delete on `#{ @origin.name }` for each row
+        delete ignore from `#{ @destination.name }` #{ SqlHelper.annotation }
+        where `#{ @destination.name }`.`id` = OLD.`id`
+      }
+    end
+
+    def trigger(type)
+      "lhmt_#{ type }_#{ @origin.name }"[0...64]
+    end
+
+    def expected_triggers
+      [trigger(:ins), trigger(:upd), trigger(:del)]
+    end
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name)
+        error("#{ @origin.name } does not exist")
+      end
+
+      unless @connection.data_source_exists?(@destination.name)
+        error("#{ @destination.name } does not exist")
+      end
+    end
+
+    def before
+      entangle.each do |stmt|
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
+      end
+      Lhm.logger.info("Created triggers on #{@origin.name}")
+    end
+
+    def after
+      untangle.each do |stmt|
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
+      end
+      Lhm.logger.info("Dropped triggers on #{@origin.name}")
+    end
+
+    def revert
+      after
+    end
+
+    private
+
+    def strip(sql)
+      sql.strip.gsub(/\n */, "\n")
+    end
+  end
+end

data/lib/lhm/intersection.rb

@@ -0,0 +1,51 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  #  Determine and format columns common to origin and destination.
+  class Intersection
+    def initialize(origin, destination, renames = {})
+      @origin = origin
+      @destination = destination
+      @renames = renames
+    end
+
+    def origin
+      (common + @renames.keys).extend(Joiners)
+    end
+
+    def destination
+      (common + @renames.values).extend(Joiners)
+    end
+
+    private
+
+    def common
+      (@origin.columns.keys & @destination.columns.keys).sort
+    end
+
+    module Joiners
+      def escaped
+        map { |name| tick(name)  }
+      end
+
+      def joined
+        escaped.join(', ')
+      end
+
+      def typed(type)
+        map { |name| qualified(name, type)  }.join(', ')
+      end
+
+      private
+
+      def qualified(name, type)
+        "`#{ type }`.`#{ name }`"
+      end
+
+      def tick(name)
+        "`#{ name }`"
+      end
+    end
+  end
+end

data/lib/lhm/invoker.rb

@@ -0,0 +1,100 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/chunker'
+require 'lhm/entangler'
+require 'lhm/atomic_switcher'
+require 'lhm/locked_switcher'
+require 'lhm/migrator'
+
+module Lhm
+  # Copies an origin table to an altered destination table. Live activity is
+  # synchronized into the destination table using triggers.
+  #
+  # Once the origin and destination tables have converged, origin is archived
+  # and replaced by destination.
+  class Invoker
+    include SqlHelper
+    LOCK_WAIT_TIMEOUT_DELTA = 10
+    INNODB_LOCK_WAIT_TIMEOUT_MAX = 1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+    LOCK_WAIT_TIMEOUT_MAX = 31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+
+    attr_reader :migrator, :connection
+
+    def initialize(origin, connection)
+      @connection = connection
+      @migrator = Migrator.new(origin, connection)
+    end
+
+    def set_session_lock_wait_timeouts
+      global_innodb_lock_wait_timeout = @connection.select_one("SHOW GLOBAL VARIABLES LIKE 'innodb_lock_wait_timeout'")
+      global_lock_wait_timeout = @connection.select_one("SHOW GLOBAL VARIABLES LIKE 'lock_wait_timeout'")
+
+      if global_innodb_lock_wait_timeout
+        desired_innodb_lock_wait_timeout = global_innodb_lock_wait_timeout['Value'].to_i + LOCK_WAIT_TIMEOUT_DELTA
+        if desired_innodb_lock_wait_timeout <= INNODB_LOCK_WAIT_TIMEOUT_MAX
+          @connection.execute("SET SESSION innodb_lock_wait_timeout=#{desired_innodb_lock_wait_timeout}")
+        end
+      end
+
+      if global_lock_wait_timeout
+        desired_lock_wait_timeout = global_lock_wait_timeout['Value'].to_i + LOCK_WAIT_TIMEOUT_DELTA
+        if desired_lock_wait_timeout <= LOCK_WAIT_TIMEOUT_MAX
+          @connection.execute("SET SESSION lock_wait_timeout=#{desired_lock_wait_timeout}")
+        end
+      end
+    end
+
+    def run(options = {})
+      normalize_options(options)
+      set_session_lock_wait_timeouts
+      migration = @migrator.run
+      entangler = Entangler.new(migration, @connection)
+
+      entangler.run do
+        options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
+        options.fetch(:chunker_class, Chunker).new(migration, @connection, options).run
+        raise "Required triggers do not exist" unless triggers_still_exist?(@connection, entangler)
+        if options[:atomic_switch]
+          AtomicSwitcher.new(migration, @connection).run
+        else
+          LockedSwitcher.new(migration, @connection).run
+        end
+      end
+    end
+
+    def triggers_still_exist?(conn, entangler)
+      triggers = conn.select_values("SHOW TRIGGERS LIKE '%#{migrator.origin.name}'").select { |name| name =~ /^lhmt/ }
+      triggers.sort == entangler.expected_triggers.sort
+    end
+
+    private
+
+    def normalize_options(options)
+      Lhm.logger.info "Starting LHM run on table=#{@migrator.name}"
+
+      unless options.include?(:atomic_switch)
+        if supports_atomic_switch?
+          options[:atomic_switch] = true
+        else
+          raise Error.new(
+            "Using mysql #{version_string}. You must explicitly set " \
+            'options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)')
+        end
+      end
+
+      if options[:throttler]
+        throttler_options = options[:throttler_options] || {}
+        options[:throttler] = Throttler::Factory.create_throttler(options[:throttler], throttler_options)
+      else
+        options[:throttler] = Lhm.throttler
+      end
+
+      Lhm.connection.retry_config = options[:retriable] || {}
+
+    rescue => e
+      Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
+      raise
+    end
+  end
+end

data/lib/lhm/locked_switcher.rb

@@ -0,0 +1,76 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+  # Switches origin with destination table nonatomically using a locked write.
+  # LockedSwitcher adopts the Facebook strategy, with the following caveat:
+  #
+  #   "Since alter table causes an implicit commit in innodb, innodb locks get
+  #   released after the first alter table. So any transaction that sneaks in
+  #   after the first alter table and before the second alter table gets
+  #   a 'table not found' error. The second alter table is expected to be very
+  #   fast though because copytable is not visible to other transactions and so
+  #   there is no need to wait."
+  #
+  class LockedSwitcher
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    LOG_PREFIX = "LockedSwitcher"
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def statements
+      uncommitted { switch }
+    end
+
+    def switch
+      [
+        "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
+        "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
+        "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
+        'commit',
+        'unlock tables'
+      ]
+    end
+
+    def uncommitted
+      [
+        'set @lhm_auto_commit = @@session.autocommit',
+        'set session autocommit = 0',
+        yield,
+        'set session autocommit = @lhm_auto_commit'
+      ].flatten
+    end
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name) &&
+             @connection.data_source_exists?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+    private
+
+    def revert
+      @connection.execute(tagged('unlock tables'))
+    end
+
+    def execute
+      statements.each do |stmt|
+        @connection.execute(tagged(stmt))
+      end
+    end
+  end
+end

data/lib/lhm/migration.rb

@@ -0,0 +1,51 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/intersection'
+require 'lhm/timestamp'
+
+module Lhm
+  class Migration
+    attr_reader :origin, :destination, :renames
+
+    def initialize(origin, destination, conditions = nil, renames = {}, time = Time.now)
+      @origin = origin
+      @destination = destination
+      @conditions = conditions
+      @renames = renames
+      @table_name = TableName.new(@origin.name, time)
+    end
+
+    def conditions
+      if @conditions.kind_of?(Proc)
+        @conditions.call
+      else
+        @conditions
+      end
+    end
+
+    def archive_name
+      @archive_name ||= @table_name.archived
+    end
+
+    def intersection
+      Intersection.new(@origin, @destination, @renames)
+    end
+
+    def origin_name
+      @table_name.original
+    end
+
+    def origin_columns
+      @origin_columns ||= intersection.origin.typed(origin_name)
+    end
+
+    def destination_name
+      @destination_name ||= destination.name
+    end
+
+    def destination_columns
+      @destination_columns ||= intersection.destination.joined
+    end
+  end
+end

data/lib/lhm/migrator.rb

@@ -0,0 +1,244 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+require 'lhm/table'
+
+module Lhm
+  # Copies existing schema and applies changes using alter on the empty table.
+  # `run` returns a Migration which can be used for the remaining process.
+  class Migrator
+    include Command
+    include SqlHelper
+
+    attr_reader :name, :statements, :connection, :conditions, :renames, :origin
+
+    def initialize(table, connection = nil)
+      @connection = connection
+      @origin = table
+      @name = table.destination_name
+      @statements = []
+      @renames = {}
+    end
+
+    # Alter a table with a custom statement
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.ddl("ALTER TABLE #{m.name} ADD COLUMN age INT(11)")
+    #   end
+    #
+    # @param [String] statement SQL alter statement
+    # @note
+    #
+    #   Don't write the table name directly into the statement. Use the #name
+    #   getter instead, because the alter statement will be executed against a
+    #   temporary table.
+    #
+    def ddl(statement)
+      statements << statement
+    end
+
+    # Add a column to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_column(:comment, "VARCHAR(12) DEFAULT '0'")
+    #   end
+    #
+    # @param [String] name Name of the column to add
+    # @param [String] definition Valid SQL column definition
+    def add_column(name, definition)
+      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition])
+    end
+
+    # Change an existing column to a new definition
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.change_column(:comment, "VARCHAR(12) DEFAULT '0' NOT NULL")
+    #   end
+    #
+    # @param [String] name Name of the column to change
+    # @param [String] definition Valid SQL column definition
+    def change_column(name, definition)
+      ddl('alter table `%s` modify column `%s` %s' % [@name, name, definition])
+    end
+
+    # Rename an existing column.
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.rename_column(:login, :username)
+    #   end
+    #
+    # @param [String] old Name of the column to change
+    # @param [String] nu New name to use for the column
+    def rename_column(old, nu)
+      col = @origin.columns[old.to_s]
+
+      definition = col[:type]
+
+      definition += ' NOT NULL' unless col[:is_nullable] == "YES"
+      definition += " DEFAULT #{@connection.quote(col[:column_default])}" if col[:column_default]
+      definition += " COMMENT #{@connection.quote(col[:comment])}" if col[:comment]
+      definition += " COLLATE #{@connection.quote(col[:collate])}" if col[:collate]
+
+      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition])
+      @renames[old.to_s] = nu.to_s
+    end
+
+    # Remove a column from a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_column(:comment)
+    #   end
+    #
+    # @param [String] name Name of the column to delete
+    def remove_column(name)
+      ddl('alter table `%s` drop `%s`' % [@name, name])
+    end
+
+    # Add an index to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_index(:comment)
+    #     m.add_index([:username, :created_at])
+    #     m.add_index("comment(10)")
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be created
+    def add_index(columns, index_name = nil)
+      ddl(index_ddl(columns, false, index_name))
+    end
+
+    # Add a unique index to a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.add_unique_index(:comment)
+    #     m.add_unique_index([:username, :created_at])
+    #     m.add_unique_index("comment(10)")
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes. It's possible to pass a length limit.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be created
+    def add_unique_index(columns, index_name = nil)
+      ddl(index_ddl(columns, true, index_name))
+    end
+
+    # Remove an index from a table
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.remove_index(:comment)
+    #     m.remove_index([:username, :created_at])
+    #   end
+    #
+    # @param [String, Symbol, Array<String, Symbol>] columns
+    #   A column name given as String or Symbol. An Array of Strings or Symbols
+    #   for compound indexes.
+    # @param [String, Symbol] index_name
+    #   Optional name of the index to be removed
+    def remove_index(columns, index_name = nil)
+      columns = [columns].flatten.map(&:to_sym)
+      from_origin = @origin.indices.find { |_, cols| cols.map(&:to_sym) == columns }
+      index_name ||= from_origin[0] unless from_origin.nil?
+      index_name ||= idx_name(@origin.name, columns)
+      ddl('drop index `%s` on `%s`' % [index_name, @name])
+    end
+
+    # Filter the data that is copied into the new table by the provided SQL.
+    # This SQL will be inserted into the copy directly after the "from"
+    # statement - so be sure to use inner/outer join syntax and not cross joins.
+    #
+    # @example Add a conditions filter to the migration.
+    #   Lhm.change_table(:sounds) do |m|
+    #     m.filter("inner join users on users.`id` = sounds.`user_id` and sounds.`public` = 1")
+    #   end
+    #
+    # @example Add a dynamic conditions filter to the migration.
+    #   Lhm.change_table(:sounds) do |m|
+    #     m.filter(-> { "where sounds.created_at <= '#{Time.now.to_fs(:db)}'" })
+    #   end
+    #
+    # @param [ String, Proc ] sql The sql filter or a proc that returns a sql filter.
+    #
+    # @return [ String, Proc ] The sql filter.
+    def filter(sql)
+      @conditions = sql
+    end
+
+    private
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name)
+        error("could not find origin table #{ @origin.name }")
+      end
+
+      unless @origin.satisfies_id_column_requirement?
+        error('origin does not satisfy `id` key requirements')
+      end
+
+      dest = @origin.destination_name
+
+      if @connection.data_source_exists?(dest)
+        error("#{ dest } should not exist; not cleaned up from previous run?")
+      end
+    end
+
+    def execute
+      destination_create
+      @statements.each do |stmt|
+        @connection.execute(tagged(stmt))
+      end
+      Migration.new(@origin, destination_read, conditions, renames)
+    end
+
+    def destination_create
+      original    = %{CREATE TABLE `#{ @origin.name }`}
+      replacement = %{CREATE TABLE `#{ @origin.destination_name }`}
+      stmt = @origin.ddl.gsub(original, replacement)
+      @connection.execute(tagged(stmt))
+
+      Lhm.logger.info("Created destination table #{@origin.destination_name}")
+    end
+
+    def destination_read
+      Table.parse(@origin.destination_name, connection)
+    end
+
+    def index_ddl(cols, unique = nil, index_name = nil)
+      assert_valid_idx_name(index_name)
+      type = unique ? 'unique index' : 'index'
+      index_name ||= idx_name(@origin.name, cols)
+      parts = [type, index_name, @name, idx_spec(cols)]
+      'create %s `%s` on `%s` (%s)' % parts
+    end
+
+    def assert_valid_idx_name(index_name)
+      if index_name && !(index_name.is_a?(String) || index_name.is_a?(Symbol))
+        raise ArgumentError, 'index_name must be a string or symbol'
+      end
+    end
+  end
+end

data/lib/lhm/printer.rb

@@ -0,0 +1,63 @@
+module Lhm
+  module Printer
+    class Output
+      def write(message)
+        print message
+      end
+    end
+
+    class Base
+      def initialize
+        @output = Output.new
+      end
+    end
+
+    class Percentage
+      def initialize
+        @max_length = 0
+      end
+
+      def notify(lowest, highest)
+        return if !highest || highest == 0
+
+        # The argument lowest represents the next_to_insert row id, and highest represents the 
+        # maximum id upto which chunker has to copy the data. 
+        # If all the rows are inserted upto highest, then lowest passed here from chunker was 
+        # highest + 1, which leads to the printer printing the progress > 100%.
+        return if lowest >= highest
+        
+        message = "%.2f%% (#{lowest}/#{highest}) complete" % (lowest.to_f / highest * 100.0)
+        write(message)
+      end
+
+      def end
+        write('100% complete')
+      end
+
+      def exception(e)
+        Lhm.logger.error("failed: #{e}")
+      end
+
+      private
+
+      def write(message)
+        if (extra = @max_length - message.length) < 0
+          @max_length = message.length
+          extra = 0
+        end
+
+        Lhm.logger.info(message)
+      end
+    end
+
+    class Dot < Base
+      def notify(*)
+        @output.write '.'
+      end
+
+      def end
+        @output.write "\n"
+      end
+    end
+  end
+end

data/lib/lhm/proxysql_helper.rb

@@ -0,0 +1,10 @@
+module Lhm
+  module ProxySQLHelper
+    extend self
+    ANNOTATION = "/*maintenance:lhm*/"
+
+    def tagged(sql)
+      "#{sql} #{ANNOTATION}"
+    end
+  end
+end