lhm-shopify 3.3.5

data/lib/lhm/cleanup/current.rb

@@ -0,0 +1,74 @@
+require 'lhm/timestamp'
+require 'lhm/sql_retry'
+
+module Lhm
+  module Cleanup
+    class Current
+      def initialize(run, origin_table_name, connection, options = {})
+        @run = run
+        @table_name = TableName.new(origin_table_name)
+        @connection = connection
+        @ddls = []
+        @retry_helper = SqlRetry.new(
+          @connection,
+          {
+            log_prefix: "Cleanup::Current"
+          }.merge!(options.fetch(:retriable, {}))
+        )
+      end
+
+      attr_reader :run, :connection, :ddls
+
+      def execute
+        build_statements_for_drop_lhm_triggers_for_origin
+        build_statements_for_rename_lhmn_tables_for_origin
+        if run
+          execute_ddls
+        else
+          report_ddls
+        end
+      end
+
+      private
+
+      def build_statements_for_drop_lhm_triggers_for_origin
+        lhm_triggers_for_origin.each do |trigger|
+          @ddls << "drop trigger if exists #{trigger}"
+        end
+      end
+
+      def lhm_triggers_for_origin
+        @lhm_triggers_for_origin ||= all_triggers_for_origin.select { |name| name =~ /^lhmt/ }
+      end
+
+      def all_triggers_for_origin
+        @all_triggers_for_origin ||= connection.select_values("show triggers like '%#{@table_name.original}'").collect do |trigger|
+          trigger.respond_to?(:trigger) ? trigger.trigger : trigger
+        end
+      end
+
+      def build_statements_for_rename_lhmn_tables_for_origin
+        lhmn_tables_for_origin.each do |table|
+          @ddls << "rename table #{table} to #{@table_name.failed}"
+        end
+      end
+
+      def lhmn_tables_for_origin
+        @lhmn_tables_for_origin ||= connection.select_values("show tables like '#{@table_name.new}'")
+      end
+
+      def execute_ddls
+        ddls.each do |ddl|
+          @retry_helper.with_retries do |retriable_connection|
+            retriable_connection.execute(ddl)
+          end
+        end
+      end
+
+      def report_ddls
+        puts "The following DDLs would be executed:"
+        ddls.each { |ddl| puts ddl }
+      end
+    end
+  end
+end

data/lib/lhm/command.rb

@@ -0,0 +1,48 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  class Error < StandardError
+  end
+
+  module Command
+    def run(&block)
+      Lhm.logger.info "Starting run of class=#{self.class}"
+      validate
+
+      if block_given?
+        before
+        block.call(self)
+        after
+      else
+        execute
+      end
+    rescue => e
+      Lhm.logger.error "Error in class=#{self.class}, reverting. exception=#{e.class} message=#{e.message}"
+      revert
+      raise
+    end
+
+    private
+
+    def validate
+    end
+
+    def revert
+    end
+
+    def execute
+      raise NotImplementedError.new(self.class.name)
+    end
+
+    def before
+    end
+
+    def after
+    end
+
+    def error(msg)
+      raise Error.new(msg)
+    end
+  end
+end

data/lib/lhm/entangler.rb

@@ -0,0 +1,117 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/sql_helper'
+require 'lhm/sql_retry'
+
+module Lhm
+  class Entangler
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    # Creates entanglement between two tables. All creates, updates and deletes
+    # to origin will be repeated on the destination table.
+    def initialize(migration, connection = nil, options = {})
+      @intersection = migration.intersection
+      @origin = migration.origin
+      @destination = migration.destination
+      @connection = connection
+      @retry_helper = SqlRetry.new(
+        @connection,
+        {
+          log_prefix: "Entangler"
+        }.merge!(options.fetch(:retriable, {}))
+      )
+    end
+
+    def entangle
+      [
+        create_delete_trigger,
+        create_insert_trigger,
+        create_update_trigger
+      ]
+    end
+
+    def untangle
+      [
+        "drop trigger if exists `#{ trigger(:del) }`",
+        "drop trigger if exists `#{ trigger(:ins) }`",
+        "drop trigger if exists `#{ trigger(:upd) }`"
+      ]
+    end
+
+    def create_insert_trigger
+      strip %Q{
+        create trigger `#{ trigger(:ins) }`
+        after insert on `#{ @origin.name }` for each row
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
+      }
+    end
+
+    def create_update_trigger
+      strip %Q{
+        create trigger `#{ trigger(:upd) }`
+        after update on `#{ @origin.name }` for each row
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
+      }
+    end
+
+    def create_delete_trigger
+      strip %Q{
+        create trigger `#{ trigger(:del) }`
+        after delete on `#{ @origin.name }` for each row
+        delete ignore from `#{ @destination.name }` #{ SqlHelper.annotation }
+        where `#{ @destination.name }`.`id` = OLD.`id`
+      }
+    end
+
+    def trigger(type)
+      "lhmt_#{ type }_#{ @origin.name }"[0...64]
+    end
+
+    def expected_triggers
+      [trigger(:ins), trigger(:upd), trigger(:del)]
+    end
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name)
+        error("#{ @origin.name } does not exist")
+      end
+
+      unless @connection.data_source_exists?(@destination.name)
+        error("#{ @destination.name } does not exist")
+      end
+    end
+
+    def before
+      entangle.each do |stmt|
+        @retry_helper.with_retries do |retriable_connection|
+          retriable_connection.execute(stmt)
+        end
+      end
+    end
+
+    def after
+      untangle.each do |stmt|
+        @retry_helper.with_retries do |retriable_connection|
+          retriable_connection.execute(stmt)
+        end
+      end
+    end
+
+    def revert
+      after
+    end
+
+    private
+
+    def strip(sql)
+      sql.strip.gsub(/\n */, "\n")
+    end
+  end
+end

data/lib/lhm/intersection.rb

@@ -0,0 +1,51 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  #  Determine and format columns common to origin and destination.
+  class Intersection
+    def initialize(origin, destination, renames = {})
+      @origin = origin
+      @destination = destination
+      @renames = renames
+    end
+
+    def origin
+      (common + @renames.keys).extend(Joiners)
+    end
+
+    def destination
+      (common + @renames.values).extend(Joiners)
+    end
+
+    private
+
+    def common
+      (@origin.columns.keys & @destination.columns.keys).sort
+    end
+
+    module Joiners
+      def escaped
+        map { |name| tick(name)  }
+      end
+
+      def joined
+        escaped.join(', ')
+      end
+
+      def typed(type)
+        map { |name| qualified(name, type)  }.join(', ')
+      end
+
+      private
+
+      def qualified(name, type)
+        "`#{ type }`.`#{ name }`"
+      end
+
+      def tick(name)
+        "`#{ name }`"
+      end
+    end
+  end
+end

data/lib/lhm/invoker.rb

@@ -0,0 +1,98 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/chunker'
+require 'lhm/entangler'
+require 'lhm/atomic_switcher'
+require 'lhm/locked_switcher'
+require 'lhm/migrator'
+
+module Lhm
+  # Copies an origin table to an altered destination table. Live activity is
+  # synchronized into the destination table using triggers.
+  #
+  # Once the origin and destination tables have converged, origin is archived
+  # and replaced by destination.
+  class Invoker
+    include SqlHelper
+    LOCK_WAIT_TIMEOUT_DELTA = 10
+      INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+      LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+
+    attr_reader :migrator, :connection
+
+    def initialize(origin, connection)
+      @connection = connection
+      @migrator = Migrator.new(origin, connection)
+    end
+
+    def set_session_lock_wait_timeouts
+      global_innodb_lock_wait_timeout = @connection.select_one("SHOW GLOBAL VARIABLES LIKE 'innodb_lock_wait_timeout'")
+      global_lock_wait_timeout = @connection.select_one("SHOW GLOBAL VARIABLES LIKE 'lock_wait_timeout'")
+
+      if global_innodb_lock_wait_timeout
+        desired_innodb_lock_wait_timeout = global_innodb_lock_wait_timeout['Value'].to_i + LOCK_WAIT_TIMEOUT_DELTA
+        if desired_innodb_lock_wait_timeout <= INNODB_LOCK_WAIT_TIMEOUT_MAX
+          @connection.execute("SET SESSION innodb_lock_wait_timeout=#{desired_innodb_lock_wait_timeout}")
+        end
+      end
+
+      if global_lock_wait_timeout
+        desired_lock_wait_timeout = global_lock_wait_timeout['Value'].to_i + LOCK_WAIT_TIMEOUT_DELTA
+        if desired_lock_wait_timeout <= LOCK_WAIT_TIMEOUT_MAX
+          @connection.execute("SET SESSION lock_wait_timeout=#{desired_lock_wait_timeout}")
+        end
+      end
+    end
+
+    def run(options = {})
+      normalize_options(options)
+      set_session_lock_wait_timeouts
+      migration = @migrator.run
+      entangler = Entangler.new(migration, @connection, options)
+
+      entangler.run do
+        options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
+        Chunker.new(migration, @connection, options).run
+        raise "Required triggers do not exist" unless triggers_still_exist?(@connection, entangler)
+        if options[:atomic_switch]
+          AtomicSwitcher.new(migration, @connection, options).run
+        else
+          LockedSwitcher.new(migration, @connection).run
+        end
+      end
+    end
+
+    def triggers_still_exist?(conn, entangler)
+      triggers = conn.select_values("SHOW TRIGGERS LIKE '%#{migrator.origin.name}'").select { |name| name =~ /^lhmt/ }
+      triggers.sort == entangler.expected_triggers.sort
+    end
+
+    private
+
+    def normalize_options(options)
+      Lhm.logger.info "Starting LHM run on table=#{@migrator.name}"
+
+      unless options.include?(:atomic_switch)
+        if supports_atomic_switch?
+          options[:atomic_switch] = true
+        else
+          raise Error.new(
+            "Using mysql #{version_string}. You must explicitly set " \
+            'options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)')
+        end
+      end
+
+      if options[:throttler]
+        throttler_options = options[:throttler_options] || {}
+        options[:throttler] = Throttler::Factory.create_throttler(options[:throttler], throttler_options)
+      else
+        options[:throttler] = Lhm.throttler
+      end
+
+    rescue => e
+      Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
+      raise
+    end
+  end
+end

data/lib/lhm/locked_switcher.rb

@@ -0,0 +1,74 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_helper'
+
+module Lhm
+  # Switches origin with destination table nonatomically using a locked write.
+  # LockedSwitcher adopts the Facebook strategy, with the following caveat:
+  #
+  #   "Since alter table causes an implicit commit in innodb, innodb locks get
+  #   released after the first alter table. So any transaction that sneaks in
+  #   after the first alter table and before the second alter table gets
+  #   a 'table not found' error. The second alter table is expected to be very
+  #   fast though because copytable is not visible to other transactions and so
+  #   there is no need to wait."
+  #
+  class LockedSwitcher
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def statements
+      uncommitted { switch }
+    end
+
+    def switch
+      [
+        "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
+        "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
+        "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
+        'commit',
+        'unlock tables'
+      ]
+    end
+
+    def uncommitted
+      [
+        'set @lhm_auto_commit = @@session.autocommit',
+        'set session autocommit = 0',
+        yield,
+        'set session autocommit = @lhm_auto_commit'
+      ].flatten
+    end
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name) &&
+             @connection.data_source_exists?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+    private
+
+    def revert
+      @connection.execute(tagged('unlock tables'))
+    end
+
+    def execute
+      statements.each do |stmt|
+        @connection.execute(tagged(stmt))
+      end
+    end
+  end
+end