lhm-shopify 3.3.5

data/gemfiles/ar-2.3_mysql.gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gem "mysql", "~> 2.8.1"
+gem "activerecord", "~> 2.3.18"
+gem "iconv", "~> 1.0.4"
+gemspec :path=>"../"

data/gemfiles/ar-3.2_mysql.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql", "~> 2.8.1"
+gem "activerecord", "~> 3.2.2"
+gemspec :path=>"../"

data/gemfiles/ar-3.2_mysql2.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.3.11"
+gem "activerecord", "~> 3.2.2"
+gemspec :path=>"../"

data/gemfiles/ar-4.0_mysql2.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.3.17"
+gem "activerecord", "~> 4.0.13"
+gemspec :path=>"../"

data/gemfiles/ar-4.1_mysql2.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.3.17"
+gem "activerecord", "~> 4.1.9"
+gemspec :path=>"../"

data/gemfiles/ar-4.2_mysql2.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.3.17"
+gem "activerecord", "~> 4.2.0"
+gemspec :path=>"../"

data/gemfiles/ar-5.0_mysql2.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.4.5"
+gem "activerecord", "~> 5.0.2"
+gemspec :path=>"../"

data/lhm.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib', __FILE__)
+$:.unshift(lib) unless $:.include?(lib)
+
+require 'lhm/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'lhm-shopify'
+  s.version       = Lhm::VERSION
+  s.licenses      = ['BSD-3-Clause']
+  s.platform      = Gem::Platform::RUBY
+  s.authors       = ['SoundCloud', 'Shopify', 'Rany Keddo', 'Tobias Bielohlawek', 'Tobias Schmidt']
+  s.email         = %q{database-engineering@shopify.com}
+  s.summary       = %q{online schema changer for mysql}
+  s.description   = %q{Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.}
+  s.homepage      = %q{http://github.com/shopify/lhm}
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+  s.executables   = []
+  s.metadata['allowed_push_host'] = "https://rubygems.org"
+
+  s.required_ruby_version = '>= 2.3.0'
+
+  s.add_dependency 'retriable', '>= 3.0.0'
+
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'mocha'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'activerecord'
+  s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'simplecov'
+end

data/lib/lhm.rb

@@ -0,0 +1,131 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/table_name'
+require 'lhm/table'
+require 'lhm/invoker'
+require 'lhm/throttler'
+require 'lhm/version'
+require 'lhm/cleanup/current'
+require 'lhm/sql_retry'
+require 'lhm/test_support'
+require 'lhm/railtie' if defined?(Rails::Railtie)
+require 'logger'
+
+# Large hadron migrator - online schema change tool
+#
+# @example
+#
+#   Lhm.change_table(:users) do |m|
+#     m.add_column(:arbitrary, "INT(12)")
+#     m.add_index([:arbitrary, :created_at])
+#     m.ddl("alter table %s add column flag tinyint(1)" % m.name)
+#   end
+#
+module Lhm
+  extend Throttler
+  extend self
+
+  DEFAULT_LOGGER_OPTIONS =  { level: Logger::INFO, file: STDOUT }
+
+  # Alters a table with the changes described in the block
+  #
+  # @param [String, Symbol] table_name Name of the table
+  # @param [Hash] options Optional options to alter the chunk / switch behavior
+  # @option options [Integer] :stride
+  #   Size of a chunk (defaults to: 2,000)
+  # @option options [Integer] :throttle
+  #   Time to wait between chunks in milliseconds (defaults to: 100)
+  # @option options [Integer] :start
+  #   Primary Key position at which to start copying chunks
+  # @option options [Integer] :limit
+  #   Primary Key position at which to stop copying chunks
+  # @option options [Boolean] :atomic_switch
+  #   Use atomic switch to rename tables (defaults to: true)
+  #   If using a version of mysql affected by atomic switch bug, LHM forces user
+  #   to set this option (see SqlHelper#supports_atomic_switch?)
+  # @yield [Migrator] Yielded Migrator object records the changes
+  # @return [Boolean] Returns true if the migration finishes
+  # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
+  def change_table(table_name, options = {}, &block)
+    origin = Table.parse(table_name, connection)
+    invoker = Invoker.new(origin, connection)
+    block.call(invoker.migrator)
+    invoker.run(options)
+    true
+  end
+
+  # Cleanup tables and triggers
+  #
+  # @param [Boolean] run execute now or just display information
+  # @param [Hash] options Optional options to alter cleanup behaviour
+  # @option options [Time] :until
+  #   Filter to only remove tables up to specified time (defaults to: nil)
+  def cleanup(run = false, options = {})
+    lhm_tables = connection.select_values('show tables').select { |name| name =~ /^lhm(a|n)_/ }
+    if options[:until]
+      lhm_tables.select! do |table|
+        table_date_time = Time.strptime(table, 'lhma_%Y_%m_%d_%H_%M_%S')
+        table_date_time <= options[:until]
+      end
+    end
+
+    lhm_triggers = connection.select_values('show triggers').collect do |trigger|
+      trigger.respond_to?(:trigger) ? trigger.trigger : trigger
+    end.select { |name| name =~ /^lhmt/ }
+
+    drop_tables_and_triggers(run, lhm_triggers, lhm_tables)
+  end
+
+  def cleanup_current_run(run, table_name, options = {})
+    Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
+  end
+
+  def setup(connection)
+    @@connection = connection
+  end
+
+  def connection
+    @@connection ||=
+      begin
+        raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+        ActiveRecord::Base.connection
+      end
+  end
+
+  def self.logger=(new_logger)
+    @@logger = new_logger
+  end
+
+  def self.logger
+    @@logger ||=
+      begin
+        logger = Logger.new(DEFAULT_LOGGER_OPTIONS[:file])
+        logger.level = DEFAULT_LOGGER_OPTIONS[:level]
+        logger.formatter = nil
+        logger
+      end
+  end
+
+  private
+
+  def drop_tables_and_triggers(run = false, triggers, tables)
+    if run
+      triggers.each do |trigger|
+        connection.execute("drop trigger if exists #{trigger}")
+      end
+      tables.each do |table|
+        connection.execute("drop table if exists #{table}")
+      end
+      true
+    elsif tables.empty? && triggers.empty?
+      puts 'Everything is clean. Nothing to do.'
+      true
+    else
+      puts "Would drop LHM backup tables: #{tables.join(', ')}."
+      puts "Would drop LHM triggers: #{triggers.join(', ')}."
+      puts 'Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.'
+      false
+    end
+  end
+end

data/lib/lhm/atomic_switcher.rb

@@ -0,0 +1,52 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/command'
+require 'lhm/migration'
+require 'lhm/sql_retry'
+
+module Lhm
+  # Switches origin with destination table using an atomic rename.
+  #
+  # It should only be used if the MySQL server version is not affected by the
+  # bin log affecting bug #39675. This can be verified using
+  # Lhm::SqlHelper.supports_atomic_switch?.
+  class AtomicSwitcher
+    include Command
+
+    attr_reader :connection
+
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
+      @origin = migration.origin
+      @destination = migration.destination
+      @retry_helper = SqlRetry.new(
+        @connection,
+        {
+          log_prefix: "AtomicSwitcher"
+        }.merge!(options.fetch(:retriable, {}))
+      )
+    end
+
+    def atomic_switch
+      "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " \
+      "`#{ @destination.name }` to `#{ @origin.name }`"
+    end
+
+    def validate
+      unless @connection.data_source_exists?(@origin.name) &&
+             @connection.data_source_exists?(@destination.name)
+        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+      end
+    end
+
+    private
+
+    def execute
+      @retry_helper.with_retries do |retriable_connection|
+        retriable_connection.execute atomic_switch
+      end
+    end
+  end
+end

data/lib/lhm/chunk_finder.rb

@@ -0,0 +1,32 @@
+module Lhm
+  class ChunkFinder
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
+      @start = options[:start] || select_start_from_db
+      @limit = options[:limit] || select_limit_from_db
+    end
+
+    attr_accessor :start, :limit
+
+    def table_empty?
+      start.nil? && limit.nil?
+    end
+
+    def validate
+      if start > limit
+        raise ArgumentError, "impossible chunk options (limit (#{limit.inspect} must be greater than start (#{start.inspect})"
+      end
+    end
+
+    private
+
+    def select_start_from_db
+      @connection.select_value("select min(id) from `#{ @migration.origin_name }`")
+    end
+
+    def select_limit_from_db
+      @connection.select_value("select max(id) from `#{ @migration.origin_name }`")
+    end
+  end
+end

data/lib/lhm/chunk_insert.rb

@@ -0,0 +1,51 @@
+require 'lhm/sql_retry'
+
+module Lhm
+  class ChunkInsert
+    def initialize(migration, connection, lowest, highest, options = {})
+      @migration = migration
+      @connection = connection
+      @lowest = lowest
+      @highest = highest
+      @retry_helper = SqlRetry.new(
+        @connection,
+        {
+          log_prefix: "Chunker Insert"
+        }.merge!(options.fetch(:retriable, {}))
+      )
+    end
+
+    def insert_and_return_count_of_rows_created
+      @retry_helper.with_retries do |retriable_connection|
+        retriable_connection.update sql
+      end
+    end
+
+    def sql
+      "insert ignore into `#{ @migration.destination_name }` (#{ @migration.destination_columns }) " \
+      "select #{ @migration.origin_columns } from `#{ @migration.origin_name }` " \
+      "#{ conditions } `#{ @migration.origin_name }`.`id` between #{ @lowest } and #{ @highest }"
+    end
+
+    private
+    # XXX this is extremely brittle and doesn't work when filter contains more
+    # than one SQL clause, e.g. "where ... group by foo". Before making any
+    # more changes here, please consider either:
+    #
+    # 1. Letting users only specify part of defined clauses (i.e. don't allow
+    # `filter` on Migrator to accept both WHERE and INNER JOIN
+    # 2. Changing query building so that it uses structured data rather than
+    # strings until the last possible moment.
+    def conditions
+      if @migration.conditions
+        @migration.conditions.
+          # strip ending paren
+          sub(/\)\Z/, '').
+          # put any where conditions in parens
+          sub(/where\s(\w.*)\Z/, 'where (\\1)') + ' and'
+      else
+        'where'
+      end
+    end
+  end
+end

data/lib/lhm/chunker.rb

@@ -0,0 +1,87 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+require 'lhm/command'
+require 'lhm/sql_helper'
+require 'lhm/printer'
+require 'lhm/chunk_insert'
+require 'lhm/chunk_finder'
+
+module Lhm
+  class Chunker
+    include Command
+    include SqlHelper
+
+    attr_reader :connection
+
+    # Copy from origin to destination in chunks of size `stride`.
+    # Use the `throttler` class to sleep between each stride.
+    def initialize(migration, connection = nil, options = {})
+      @migration = migration
+      @connection = connection
+      @chunk_finder = ChunkFinder.new(migration, connection, options)
+      @options = options
+      @verifier = options[:verifier]
+      if @throttler = options[:throttler]
+        @throttler.connection = @connection if @throttler.respond_to?(:connection=)
+      end
+      @start = @chunk_finder.start
+      @limit = @chunk_finder.limit
+      @printer = options[:printer] || Printer::Percentage.new
+      @retry_helper = SqlRetry.new(
+        @connection,
+        {
+          log_prefix: "Chunker"
+        }.merge!(options.fetch(:retriable, {}))
+      )
+    end
+
+    def execute
+      return if @chunk_finder.table_empty?
+      @next_to_insert = @start
+      while @next_to_insert <= @limit || (@start == @limit)
+        stride = @throttler.stride
+        top = upper_id(@next_to_insert, stride)
+        verify_can_run
+
+        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
+        if @throttler && affected_rows > 0
+          @throttler.run
+        end
+        @printer.notify(bottom, @limit)
+        @next_to_insert = top + 1
+        break if @start == @limit
+      end
+      @printer.end
+    rescue => e
+      @printer.exception(e) if @printer.respond_to?(:exception)
+      raise
+    end
+
+    private
+
+    def bottom
+      @next_to_insert
+    end
+
+    def verify_can_run
+      return unless @verifier
+      @retry_helper.with_retries do |retriable_connection|
+        raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
+      end
+    end
+
+    def upper_id(next_id, stride)
+      sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
+      top = @retry_helper.with_retries do |retriable_connection|
+        retriable_connection.select_value(sql)
+      end
+
+      [top ? top.to_i : @limit, @limit].min
+    end
+
+    def validate
+      return if @chunk_finder.table_empty?
+      @chunk_finder.validate
+    end
+  end
+end