lhm-teak 3.6.0

data/gemfiles/activerecord_7.0.0.alpha2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "7.0.0.alpha2"
+
+gemspec path: "../"

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.5)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activerecord (7.0.0.alpha2)
+      activemodel (= 7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activesupport (7.0.0.alpha2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  arm64-darwin-21
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.0.0.alpha2)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/lhm.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$:.unshift(lib) unless $:.include?(lib)
+
+require 'lhm/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'lhm-teak'
+  s.version       = Lhm::VERSION
+  s.licenses      = ['BSD-3-Clause']
+  s.platform      = Gem::Platform::RUBY
+  s.authors       = ['SoundCloud', 'Shopify', 'Rany Keddo', 'Tobias Bielohlawek', 'Tobias Schmidt', 'Teak.io']
+  s.email         = %q{team@teak.io}
+  s.summary       = %q{online schema changer for mysql}
+  s.description   = %q{Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.}
+  s.homepage      = %q{http://github.com/GoCarrot/lhm}
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+  s.executables   = []
+  s.metadata['allowed_push_host'] = "https://rubygems.org"
+
+  s.required_ruby_version = '>= 2.3.0'
+
+  s.add_dependency 'retriable', '>= 3.0.0'
+
+  s.add_development_dependency 'activerecord'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'mocha'
+  s.add_development_dependency 'after_do'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'toxiproxy'
+  s.add_development_dependency 'appraisal'
+  s.add_development_dependency 'byebug'
+end

data/lib/lhm/atomic_switcher.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_retry'
+
+module Lhm
+  # Switches origin with destination table using an atomic rename.
+  #
+  # It should only be used if the MySQL server version is not affected by the
+  # bin log affecting bug #39675. This can be verified using
+  # Lhm::SqlHelper.supports_atomic_switch?.
+  class AtomicSwitcher
+    include Command
+
+    attr_reader :connection
+
+    LOG_PREFIX = "AtomicSwitcher"
+
+    def initialize(migration, connection = nil)
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+    end
+
+    def atomic_switch
+      "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " \
+      "`#{ @destination.name }` to `#{ @origin.name }`"
+    end
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name) &&
+        @connection.data_source_exists?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+    private
+
+    def execute
+      @connection.execute(atomic_switch, should_retry: true, log_prefix: LOG_PREFIX)
+    end
+  end
+end

data/lib/lhm/chunk_finder.rb

@@ -0,0 +1,62 @@
+module Lhm
+  class ChunkFinder
+    LOG_PREFIX = "Chunker"
+
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
+      @start = options[:start] || select_start_from_db
+      @limit = options[:limit] || select_limit_from_db
+      @throttler = options[:throttler]
+      @processed_rows = 0
+    end
+
+    def table_empty?
+      start.nil? && limit.nil?
+    end
+
+    def validate
+      if start > limit
+        raise ArgumentError, "impossible chunk options (limit (#{limit.inspect} must be greater than start (#{start.inspect})"
+      end
+    end
+
+    def each_chunk
+      next_id = @start
+      @processed_rows = 0
+      while next_id <= @limit
+        top = upper_id(next_id)
+        @processed_rows += @throttler.stride
+        yield ChunkInsert.new(@migration, @connection, next_id, top)
+        next_id = top + 1
+      end
+    end
+
+    def max_rows
+      @limit - @start + 1
+    end
+
+    def processed_rows
+      @processed_rows
+    end
+
+    private
+
+    attr_reader :start, :limit
+
+    def select_start_from_db
+      @connection.select_value("select min(id) from `#{ @migration.origin_name }`")
+    end
+
+    def select_limit_from_db
+      @connection.select_value("select max(id) from `#{ @migration.origin_name }`")
+    end
+
+    def upper_id(next_id)
+      sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ @throttler.stride - 1}"
+      top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
+
+      [top ? top.to_i : @limit, @limit].min
+    end
+  end
+end

data/lib/lhm/chunk_insert.rb

@@ -0,0 +1,61 @@
+require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
+
+module Lhm
+  class ChunkInsert
+
+    LOG_PREFIX = "ChunkInsert"
+
+    def initialize(migration, connection, lowest, highest, retry_options = {})
+      @migration = migration
+      @connection = connection
+      @lowest = lowest
+      @highest = highest
+      @retry_options = retry_options
+    end
+
+    def insert_and_return_count_of_rows_created
+      @connection.update(sql, should_retry: true, log_prefix: LOG_PREFIX)
+    end
+
+    def bottom
+      @lowest
+    end
+
+    def top
+      @highest
+    end
+
+    def expected_rows
+      top - bottom + 1
+    end
+
+    private
+
+    def sql
+      "insert ignore into `#{ @migration.destination_name }` (#{ @migration.destination_columns }) " \
+      "select #{ @migration.origin_columns } from `#{ @migration.origin_name }` " \
+      "#{ conditions } `#{ @migration.origin_name }`.`id` between #{ @lowest } and #{ @highest }"
+    end
+
+    # XXX this is extremely brittle and doesn't work when filter contains more
+    # than one SQL clause, e.g. "where ... group by foo". Before making any
+    # more changes here, please consider either:
+    #
+    # 1. Letting users only specify part of defined clauses (i.e. don't allow
+    # `filter` on Migrator to accept both WHERE and INNER JOIN
+    # 2. Changing query building so that it uses structured data rather than
+    # strings until the last possible moment.
+    def conditions
+      if @migration.conditions
+        @migration.conditions.
+          # strip ending paren
+          sub(/\)\Z/, '').
+          # put any where conditions in parens
+          sub(/where\s(\w.*)\Z/, 'where (\\1)') + ' and'
+      else
+        'where'
+      end
+    end
+  end
+end

data/lib/lhm/chunker.rb

@@ -0,0 +1,95 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+require 'lhm/command'
+require 'lhm/sql_helper'
+require 'lhm/printer'
+require 'lhm/chunk_insert'
+require 'lhm/chunk_finder'
+
+module Lhm
+  class Chunker
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    LOG_PREFIX = "Chunker"
+
+    # Copy from origin to destination in chunks of size `stride`.
+    # Use the `throttler` class to sleep between each stride.
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
+      @chunk_finder = options.fetch(:chuck_finder, ChunkFinder).new(migration, connection, options)
+      @options = options
+      @raise_on_warnings = options.fetch(:raise_on_warnings, false)
+      @verifier = options[:verifier]
+      if @throttler = options[:throttler]
+        @throttler.connection = @connection if @throttler.respond_to?(:connection=)
+      end
+      @printer = options[:printer] || Printer::Percentage.new
+      @retry_options = options[:retriable] || {}
+      @retry_helper = SqlRetry.new(
+        @connection,
+        retry_options: @retry_options
+      )
+    end
+
+    def execute
+      @start_time = Time.now
+
+      return if @chunk_finder.table_empty?
+      @chunk_finder.each_chunk do |chunk|
+        verify_can_run
+
+        affected_rows = chunk.insert_and_return_count_of_rows_created
+
+        # Only log the chunker progress every 5 minutes instead of every iteration
+        current_time = Time.now
+        if current_time - @start_time > (5 * 60)
+          Lhm.logger.info("Inserted #{affected_rows} rows into the destination table from #{chunk.bottom} to #{chunk.top}")
+          @start_time = current_time
+        end
+
+        if affected_rows < chunk.expected_rows
+          raise_on_non_pk_duplicate_warning
+        end
+
+        if @throttler && affected_rows > 0
+          @throttler.run
+        end
+
+        @printer.notify(@chunk_finder.processed_rows, @chunk_finder.max_rows)
+      end
+      @printer.end
+    rescue => e
+      @printer.exception(e) if @printer.respond_to?(:exception)
+      raise
+    end
+
+    private
+
+    def raise_on_non_pk_duplicate_warning
+      @connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
+        unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
+          m = "Unexpected warning found for inserted row: #{message}"
+          Lhm.logger.warn(m)
+          raise Error.new(m) if @raise_on_warnings
+        end
+      end
+    end
+
+    def verify_can_run
+      return unless @verifier
+      @retry_helper.with_retries(log_prefix: LOG_PREFIX) do |retriable_connection|
+        raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
+      end
+    end
+
+    def validate
+      return if @chunk_finder.table_empty?
+      @chunk_finder.validate
+    end
+
+  end
+end

data/lib/lhm/cleanup/current.rb

@@ -0,0 +1,71 @@
+require 'lhm/timestamp'
+require 'lhm/sql_retry'
+
+module Lhm
+  module Cleanup
+    class Current
+
+      LOG_PREFIX = "Current"
+
+      def initialize(run, origin_table_name, connection, options={})
+        @run = run
+        @table_name = TableName.new(origin_table_name)
+        @connection = connection
+        @ddls = []
+        @retry_config = options[:retriable] || {}
+      end
+
+      attr_reader :run, :connection, :ddls
+
+      def execute
+        build_statements_for_drop_lhm_triggers_for_origin
+        build_statements_for_rename_lhmn_tables_for_origin
+        if run
+          execute_ddls
+        else
+          report_ddls
+        end
+      end
+
+      private
+
+      def build_statements_for_drop_lhm_triggers_for_origin
+        lhm_triggers_for_origin.each do |trigger|
+          @ddls << "drop trigger if exists #{trigger}"
+        end
+      end
+
+      def lhm_triggers_for_origin
+        @lhm_triggers_for_origin ||= all_triggers_for_origin.select { |name| name =~ /^lhmt/ }
+      end
+
+      def all_triggers_for_origin
+        @all_triggers_for_origin ||= connection.select_values("show triggers like '%#{@table_name.original}'").collect do |trigger|
+          trigger.respond_to?(:trigger) ? trigger.trigger : trigger
+        end
+      end
+
+      def build_statements_for_rename_lhmn_tables_for_origin
+        lhmn_tables_for_origin.each do |table|
+          @ddls << "rename table #{table} to #{@table_name.failed}"
+        end
+      end
+
+      def lhmn_tables_for_origin
+        @lhmn_tables_for_origin ||= connection.select_values("show tables like '#{@table_name.new}'")
+      end
+
+      def execute_ddls
+        ddls.each do |ddl|
+            @connection.execute(ddl, should_retry: true, log_prefix: LOG_PREFIX)
+        end
+        Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
+        Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
+      end
+
+      def report_ddls
+        Lhm.logger.info("The following DDLs would be executed: #{ddls}")
+      end
+    end
+  end
+end

data/lib/lhm/command.rb

@@ -0,0 +1,48 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  class Error < StandardError
+  end
+
+  module Command
+    def run(&block)
+      Lhm.logger.info "Starting run of class=#{self.class}"
+      validate
+
+      if block_given?
+        before
+        block.call(self)
+        after
+      else
+        execute
+      end
+    rescue => e
+      Lhm.logger.error "Error in class=#{self.class}, reverting. exception=#{e.class} message=#{e.message}"
+      revert
+      raise
+    end
+
+    private
+
+    def validate
+    end
+
+    def revert
+    end
+
+    def execute
+      raise NotImplementedError.new(self.class.name)
+    end
+
+    def before
+    end
+
+    def after
+    end
+
+    def error(msg)
+      raise Error.new(msg)
+    end
+  end
+end

data/lib/lhm/connection.rb

@@ -0,0 +1,108 @@
+require 'delegate'
+require 'forwardable'
+require 'lhm/sql_retry'
+
+module Lhm
+    # Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
+    # connection.
+  class Connection < SimpleDelegator
+    extend Forwardable
+
+    # Will delegate the following function to @sql_retry object, while leaving them accessible from the Lhm::Connection
+    # object
+    def_delegators :@sql_retry, :reconnect_with_consistent_host, :reconnect_with_consistent_host=, :retry_config=
+
+    alias ar_connection __getobj__
+
+    def initialize(connection:, options: {})
+      @sql_retry = Lhm::SqlRetry.new(
+        connection,
+        retry_options: options[:retriable] || {},
+        reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
+      )
+
+      # Creates delegation for the ActiveRecord Connection
+      super(connection)
+    end
+
+    def ar_connection=(connection)
+      raise Lhm::Error.new("Lhm::Connection requires an active record connection to operate") if connection.nil?
+
+      @sql_retry.connection = connection
+      # Sets connection as the delegated object
+      __setobj__(connection)
+    end
+
+    # ActiveRecord::Base overridden methods to incorporate custom retry logic
+    # All other methods will be delegated
+    def execute(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:execute, query, log_prefix)
+      else
+        exec(:execute, query)
+      end
+    end
+
+    def update(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:update, query, log_prefix)
+      else
+        exec(:update, query)
+      end
+    end
+
+    def select_value(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_value, query, log_prefix)
+      else
+        exec(:select_value, query)
+      end
+    end
+
+    def select_values(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_values, query, log_prefix)
+      else
+        exec(:select_values, query)
+      end
+    end
+
+    def select_one(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_one, query, log_prefix)
+      else
+        exec(:select_one, query)
+      end
+    end
+
+    private
+
+    def exec(method, sql)
+      ar_connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+    end
+
+    def exec_with_retries(method, sql, log_prefix=nil)
+      effective_log_prefix = log_prefix || file
+      @sql_retry.with_retries(log_prefix: effective_log_prefix) do |conn|
+        conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+      end
+    end
+
+    # Returns camelized file name of caller (e.g. chunk_insert.rb -> ChunkInsert)
+    def file
+      # Find calling file and extract name
+      /[\/]*(\w+).rb:\d+:in/.match(relevant_caller)
+      name = $1&.camelize || "Connection"
+      "#{name}"
+    end
+
+    def relevant_caller
+      lhm_stack = caller.select { |x| x.include?("/lhm") }
+      first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
+
+      # Find the file that called the `#execute` (fallbacks to current file)
+      return lhm_stack.first unless first_candidate_index
+      lhm_stack.at(first_candidate_index)
+    end
+  end
+end