lhm-teak 3.6.0

data/lib/lhm/throttler.rb

@@ -0,0 +1,36 @@
+require 'lhm/throttler/time'
+require 'lhm/throttler/slave_lag'
+require 'lhm/throttler/threads_running'
+
+module Lhm
+  module Throttler
+    CLASSES = { :time_throttler => Throttler::Time,
+                :slave_lag_throttler => Throttler::SlaveLag,
+                :threads_running_throttler => Throttler::ThreadsRunning }
+
+    def throttler
+      @throttler ||= Throttler::Time.new
+    end
+
+    def setup_throttler(type, options = {})
+      @throttler = Factory.create_throttler(type, options)
+    end
+
+    class Factory
+      def self.create_throttler(type, options = {})
+        case type
+        when Lhm::Command
+          type
+        when Symbol
+          CLASSES[type].new(options)
+        when String
+          CLASSES[type.to_sym].new(options)
+        when Class
+          type.new(options)
+        else
+          raise ArgumentError, 'type argument must be a Symbol, String or Class'
+        end
+      end
+    end
+  end
+end

data/lib/lhm/timestamp.rb

@@ -0,0 +1,11 @@
+module Lhm
+  class Timestamp
+    def initialize(time)
+      @time = time
+    end
+
+    def to_s
+      @time.strftime "%Y_%m_%d_%H_%M_%S_#{ '%03d' % (@time.usec / 1000) }"
+    end
+  end
+end

data/lib/lhm/version.rb

@@ -0,0 +1,6 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+module Lhm
+  VERSION = '3.6.0'
+end

data/lib/lhm-shopify.rb

@@ -0,0 +1 @@
+require "lhm"

data/lib/lhm.rb

@@ -0,0 +1,156 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/table_name'
+require 'lhm/table'
+require 'lhm/invoker'
+require 'lhm/throttler'
+require 'lhm/version'
+require 'lhm/cleanup/current'
+require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
+require 'lhm/connection'
+require 'lhm/test_support'
+require 'lhm/railtie' if defined?(Rails::Railtie)
+require 'logger'
+
+# Large hadron migrator - online schema change tool
+#
+# @example
+#
+#   Lhm.change_table(:users) do |m|
+#     m.add_column(:arbitrary, "INT(12)")
+#     m.add_index([:arbitrary, :created_at])
+#     m.ddl("alter table %s add column flag tinyint(1)" % m.name)
+#   end
+#
+module Lhm
+  extend Throttler
+  extend self
+
+  DEFAULT_LOGGER_OPTIONS = { level: Logger::INFO, file: STDOUT }
+
+  # Alters a table with the changes described in the block
+  #
+  # @param [String, Symbol] table_name Name of the table
+  # @param [Hash] options Optional options to alter the chunk / switch behavior
+  # @option options [Integer] :stride
+  #   Size of a chunk (defaults to: 2,000)
+  # @option options [Integer] :throttle
+  #   Time to wait between chunks in milliseconds (defaults to: 100)
+  # @option options [Integer] :start
+  #   Primary Key position at which to start copying chunks
+  # @option options [Integer] :limit
+  #   Primary Key position at which to stop copying chunks
+  # @option options [Boolean] :atomic_switch
+  #   Use atomic switch to rename tables (defaults to: true)
+  #   If using a version of mysql affected by atomic switch bug, LHM forces user
+  #   to set this option (see SqlHelper#supports_atomic_switch?)
+  # @option options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
+  # @yield [Migrator] Yielded Migrator object records the changes
+  # @return [Boolean] Returns true if the migration finishes
+  # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
+  def change_table(table_name, options = {}, &block)
+    with_flags(options) do
+      origin = Table.parse(table_name, connection)
+      invoker = Invoker.new(origin, connection)
+      block.call(invoker.migrator)
+      invoker.run(options)
+      true
+    end
+  end
+
+  # Cleanup tables and triggers
+  #
+  # @param [Boolean] run execute now or just display information
+  # @param [Hash] options Optional options to alter cleanup behaviour
+  # @option options [Time] :until
+  #   Filter to only remove tables up to specified time (defaults to: nil)
+  def cleanup(run = false, options = {})
+    lhm_tables = connection.select_values('show tables').select { |name| name =~ /^lhm(a|n)_/ }
+    if options[:until]
+      lhm_tables.select! do |table|
+        table_date_time = Time.strptime(table, 'lhma_%Y_%m_%d_%H_%M_%S')
+        table_date_time <= options[:until]
+      end
+    end
+
+    lhm_triggers = connection.select_values('show triggers').collect do |trigger|
+      trigger.respond_to?(:trigger) ? trigger.trigger : trigger
+    end.select { |name| name =~ /^lhmt/ }
+
+    drop_tables_and_triggers(run, lhm_triggers, lhm_tables)
+  end
+
+  def cleanup_current_run(run, table_name, options = {})
+    Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
+  end
+
+  # Setups DB connection
+  #
+  # @param [ActiveRecord::Base] connection ActiveRecord Connection
+  def setup(connection)
+    @@connection = Connection.new(connection: connection)
+  end
+
+  # Returns DB connection (or initializes it if not created yet)
+  def connection
+    @@connection ||= begin
+                       raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+                       @@connection = Connection.new(connection: ActiveRecord::Base.connection)
+                     end
+  end
+
+  def self.logger=(new_logger)
+    @@logger = new_logger
+  end
+
+  def self.logger
+    @@logger ||=
+      begin
+        logger = Logger.new(DEFAULT_LOGGER_OPTIONS[:file])
+        logger.level = DEFAULT_LOGGER_OPTIONS[:level]
+        logger.formatter = nil
+        logger
+      end
+  end
+
+  private
+
+  def drop_tables_and_triggers(run = false, triggers, tables)
+    if run
+      triggers.each do |trigger|
+        connection.execute("drop trigger if exists #{trigger}")
+      end
+      logger.info("Dropped triggers #{triggers.join(', ')}")
+
+      tables.each do |table|
+        connection.execute("drop table if exists #{table}")
+      end
+      logger.info("Dropped tables #{tables.join(', ')}")
+
+      true
+    elsif tables.empty? && triggers.empty?
+      logger.info('Everything is clean. Nothing to do.')
+      true
+    else
+      logger.info("Would drop LHM backup tables: #{tables.join(', ')}.")
+      logger.info("Would drop LHM triggers: #{triggers.join(', ')}.")
+      logger.info('Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.')
+      false
+    end
+  end
+
+  def with_flags(options)
+    old_flags = {
+      reconnect_with_consistent_host: Lhm.connection.reconnect_with_consistent_host,
+    }
+
+    Lhm.connection.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
+
+    yield
+  ensure
+    Lhm.connection.reconnect_with_consistent_host = old_flags[:reconnect_with_consistent_host]
+  end
+end

data/scripts/helpers/wait-for-dbs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Wait for writer
+echo "Waiting for MySQL-1: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33006 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for reader
+echo "Waiting for MySQL-2: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33007 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for proxysql
+echo "Waiting for ProxySQL:"
+while ! (mysqladmin ping --host="127.0.0.1" --port=33005 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+
+echo "All DBs are ready"

data/scripts/mysql/reader/create_replication.sql

@@ -0,0 +1,10 @@
+STOP SLAVE;
+CHANGE MASTER TO
+MASTER_HOST='mysql-1',
+MASTER_AUTO_POSITION=1,
+MASTER_USER='replication',
+MASTER_PASSWORD='password',
+MASTER_CONNECT_RETRY=1,
+MASTER_RETRY_COUNT=300; -- 5 minutes
+
+start slave;

data/scripts/mysql/writer/create_test_db.sql

@@ -0,0 +1 @@
+CREATE DATABASE test;

data/scripts/mysql/writer/create_users.sql

@@ -0,0 +1,6 @@
+# Creates replication user in Writer
+CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
+CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
+
+CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
+GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';

data/scripts/proxysql/proxysql.cnf

@@ -0,0 +1,117 @@
+#file proxysql.cfg
+
+datadir="/var/lib/proxysql"
+restart_on_missing_heartbeats=999999
+query_parser_token_delimiters=","
+query_parser_key_value_delimiters=":"
+unit_of_work_identifiers="consistent_read_id"
+
+admin_variables=
+{
+  mysql_ifaces="0.0.0.0:6032"
+  admin_credentials="admin:password;remote-admin:password"
+}
+
+mysql_servers =
+(
+  {
+    address="mysql-1"
+    port=3306
+    hostgroup=0
+    max_connections=200
+  },
+  {
+    address="mysql-2"
+    port=3306
+    hostgroup=1
+    max_connections=200
+  }
+)
+
+mysql_variables=
+{
+  session_idle_ms=1
+    auto_increment_delay_multiplex=0
+
+    threads=8
+    max_connections=100000
+    interfaces="0.0.0.0:3306"
+    server_version="5.7.18-proxysql"
+    connect_timeout_server=10000
+    connect_timeout_server_max=10000
+    connect_retries_on_failure=0
+    default_charset="utf8mb4"
+    free_connections_pct=100
+    connection_warming=true
+    max_allowed_packet=16777216
+    monitor_enabled=false
+    query_retries_on_failure=0
+    shun_on_failures=999999
+    shun_recovery_time_sec=0
+    kill_backend_connection_when_disconnect=false
+    stats_time_backend_query=false
+    stats_time_query_processor=false
+    max_stmts_per_connection=5
+    default_max_latency_ms=999999
+    wait_timeout=1800000
+    eventslog_format=3
+    log_multiplexing_disabled=true
+    log_unhealthy_connections=false
+}
+
+# defines all the MySQL users
+mysql_users:
+(
+  {
+    username = "root"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=1000
+    active = 1
+  },
+  {
+    username = "writer"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  },
+  {
+    username = "reader"
+    password = "password"
+    default_hostgroup = 1
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  }
+)
+
+#defines MySQL Query Rules
+mysql_query_rules:
+(
+  {
+      rule_id = 1
+      active = 1
+      match_digest = "@@SESSION"
+      multiplex = 2
+    },
+    {
+      rule_id = 2
+      active = 1
+      match_digest = "@@global\.server_id"
+      multiplex = 2
+    },
+    {
+      rule_id = 3
+      active = 1
+      match_digest = "@@global\.hostname"
+      multiplex = 2
+    },
+    {
+      rule_id = 4
+      active = 1
+      match_pattern = "maintenance:lhm"
+      destination_hostgroup = 0
+    }
+)

data/spec/.lhm.example

@@ -0,0 +1,4 @@
+mysqldir=/usr/local/mysql
+basedir=~/lhm-cluster
+master_port=3306
+slave_port=3307

data/spec/README.md

@@ -0,0 +1,58 @@
+# Preparing for master slave integration tests
+
+## Configuration
+
+create ~/.lhm:
+
+    mysqldir=/usr/local/mysql
+    basedir=~/lhm-cluster
+    master_port=3306
+    slave_port=3307
+
+mysqldir specifies the location of your mysql install. basedir is the
+directory master and slave databases will get installed into.
+
+## Automatic setup
+
+### Run
+
+    bin/lhm-spec-clobber.sh
+
+You can set the integration specs up to run against a master slave setup by
+running the included that. This deletes the configured lhm master slave setup and reinstalls and configures a master slave setup.
+
+Follow the manual instructions if you want more control over this process.
+
+## Manual setup
+
+### set up instances
+
+    bin/lhm-spec-setup-cluster.sh
+
+### start instances
+
+    basedir=/opt/lhm-luster
+    mysqld --defaults-file="$basedir/master/my.cnf"
+    mysqld --defaults-file="$basedir/slave/my.cnf"
+
+### run the grants
+
+    bin/lhm-spec-grants.sh
+
+## run specs
+
+Setup the dependency gems
+
+    export BUNDLE_GEMFILE=gemfiles/ar-4.2_mysql2.gemfile
+    bundle install
+
+To run specs in slave mode, set the MASTER_SLAVE=1 when running tests:
+
+    MASTER_SLAVE=1 bundle exec rake specs
+
+# connecting
+
+you can connect by running (with the respective ports):
+
+    mysql --protocol=TCP -p3307
+

data/spec/fixtures/bigint_table.ddl

@@ -0,0 +1,4 @@
+CREATE TABLE `bigint_table` (
+  `id` BIGINT(20) auto_increment,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/composite_primary_key.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `composite_primary_key` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint(20) NOT NULL,
+  CONSTRAINT `pk_composite` PRIMARY KEY (`shop_id`,`id`),
+  INDEX `index_key_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/composite_primary_key_dest.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `composite_primary_key_dest` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint(20) NOT NULL,
+  CONSTRAINT `pk_composite` PRIMARY KEY (`shop_id`,`id`),
+  INDEX `index_key_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/custom_primary_key.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `custom_primary_key` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `pk` varchar(255),
+  PRIMARY KEY (`pk`),
+  UNIQUE KEY `index_custom_primary_key_on_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/custom_primary_key_dest.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `custom_primary_key_dest` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `pk` varchar(255),
+  PRIMARY KEY (`pk`),
+  UNIQUE KEY `index_custom_primary_key_on_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/destination.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `destination` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `destination` int(11) DEFAULT NULL,
+  `common` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/lines.ddl

@@ -0,0 +1,7 @@
+CREATE TABLE `lines` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `between` varchar(10),
+  `lines` int(11),
+  `key` varchar(10),
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/origin.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `origin` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `origin` int(11) DEFAULT NULL,
+  `common` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/permissions.ddl

@@ -0,0 +1,5 @@
+CREATE TABLE `permissions` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `track_id` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/small_table.ddl

@@ -0,0 +1,4 @@
+CREATE TABLE `small_table` (
+  `id` INT(11) auto_increment,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/tracks.ddl

@@ -0,0 +1,5 @@
+CREATE TABLE `tracks` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `public` int(4) DEFAULT 0,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/users.ddl

@@ -0,0 +1,14 @@
+CREATE TABLE `users` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `reference` int(11) DEFAULT NULL,
+  `username` varchar(255) DEFAULT NULL,
+  `group` varchar(255) DEFAULT 'Superfriends',
+  `created_at` datetime DEFAULT NULL,
+  `comment` varchar(20) DEFAULT NULL,
+  `description` text,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_users_on_reference` (`reference`),
+  KEY `index_users_on_username_and_created_at` (`username`,`created_at`),
+  KEY `index_with_a_custom_name` (`username`,`group`)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/wo_id_int_column.ddl

@@ -0,0 +1,6 @@
+-- Without id int column
+CREATE TABLE `wo_id_int_column` (
+  `id` varchar(15) NOT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+

data/spec/integration/atomic_switcher_spec.rb

@@ -0,0 +1,129 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/atomic_switcher'
+require 'lhm/connection'
+
+describe Lhm::AtomicSwitcher do
+  include IntegrationHelper
+
+  before(:each) { connect_master! }
+
+  describe 'switching' do
+    before(:each) do
+      Thread.abort_on_exception = true
+      @origin = table_create('origin')
+      @destination = table_create('destination')
+      @migration = Lhm::Migration.new(@origin, @destination)
+      @logs = StringIO.new
+      Lhm.logger = Logger.new(@logs)
+      @connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
+      @connection.execute('SET GLOBAL lock_wait_timeout=3')
+    end
+
+    after(:each) do
+      Thread.abort_on_exception = false
+    end
+
+    it 'should retry and log on lock wait timeouts' do
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # Matches initial host -> triggers retry
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 3,
+          base_interval: 0
+        }
+      })
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+
+      assert switcher.run
+
+      log_messages = @logs.string.split("\n")
+      assert_equal(2, log_messages.length)
+      assert log_messages[0].include? "Starting run of class=Lhm::AtomicSwitcher"
+      # On failure of this assertion, check for Lhm::Connection#file
+      assert log_messages[1].include? "[AtomicSwitcher] ActiveRecord::StatementInvalid: 'Lock wait timeout exceeded; try restarting transaction.' - 1 tries"
+    end
+
+    it 'should give up on lock wait timeouts after a configured number of tries' do
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:active?).returns(true)
+      ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 1
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+                   .then
+                   .returns([["dummy"]]) # triggers retry 2
+                   .then
+                   .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
+
+      connection = Lhm::Connection.new(connection: ar_connection, options: {
+        reconnect_with_consistent_host: true,
+        retriable: {
+          tries: 2,
+          base_interval: 0
+        }
+      })
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+
+      assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
+    end
+
+    it 'should raise on non lock wait timeout exceptions' do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.send :define_singleton_method, :atomic_switch do
+        'SELECT * FROM nonexistent'
+      end
+      value(-> { switcher.run }).must_raise(ActiveRecord::StatementInvalid)
+    end
+
+    it "should raise when destination doesn't exist" do
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(false)
+
+      connection = Lhm::Connection.new(connection: ar_connection)
+
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+
+      assert_raises(Lhm::Error) { switcher.run }
+    end
+
+    it 'rename origin to archive' do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        value(data_source_exists?(@origin)).must_equal true
+        value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
+      end
+    end
+
+    it 'rename destination to origin' do
+      switcher = Lhm::AtomicSwitcher.new(@migration, connection)
+      switcher.run
+
+      slave do
+        value(data_source_exists?(@destination)).must_equal false
+        value(table_read(@origin.name).columns.keys).must_include 'destination'
+      end
+    end
+  end
+end