lhm-shopify 3.3.5

data/lib/lhm/table.rb

@@ -0,0 +1,121 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require 'lhm/sql_helper'
+
+module Lhm
+  class Table
+    attr_reader :name, :columns, :indices, :pk, :ddl
+
+    def initialize(name, pk = 'id', ddl = nil)
+      @name = name
+      @table_name = TableName.new(name)
+      @columns = {}
+      @indices = {}
+      @pk = pk
+      @ddl = ddl
+    end
+
+    def satisfies_id_column_requirement?
+      !!((id = columns['id']) &&
+        id[:type] =~ /(bigint|int)(\(\d+\))?/)
+    end
+
+    def destination_name
+      @destination_name ||= @table_name.new
+    end
+
+    def self.parse(table_name, connection)
+      Parser.new(table_name, connection).parse
+    end
+
+    class Parser
+      include SqlHelper
+
+      def initialize(table_name, connection)
+        @table_name = table_name.to_s
+        @schema_name = connection.current_database
+        @connection = connection
+      end
+
+      def ddl
+        sql = "show create table `#{ @table_name }`"
+        specification = nil
+        @connection.execute(sql).each { |row| specification = row.last }
+        specification
+      end
+
+      def parse
+        schema = read_information_schema
+
+        Table.new(@table_name, extract_primary_key(schema), ddl).tap do |table|
+          schema.each do |defn|
+            column_name    = struct_key(defn, 'COLUMN_NAME')
+            column_type    = struct_key(defn, 'COLUMN_TYPE')
+            is_nullable    = struct_key(defn, 'IS_NULLABLE')
+            column_default = struct_key(defn, 'COLUMN_DEFAULT')
+            comment = struct_key(defn, 'COLUMN_COMMENT')
+            collate = struct_key(defn, 'COLLATION_NAME')
+
+            table.columns[defn[column_name]] = {
+              :type => defn[column_type],
+              :is_nullable => defn[is_nullable],
+              :column_default => defn[column_default],
+              :comment => defn[comment],
+              :collate => defn[collate],
+            }
+          end
+
+          extract_indices(read_indices).each do |idx, columns|
+            table.indices[idx] = columns
+          end
+        end
+      end
+
+      private
+
+      def read_information_schema
+        @connection.select_all %Q{
+          select *
+            from information_schema.columns
+           where table_name = '#{ @table_name }'
+             and table_schema = '#{ @schema_name }'
+        }
+      end
+
+      def read_indices
+        @connection.select_all %Q{
+          show indexes from `#{ @schema_name }`.`#{ @table_name }`
+         where key_name != 'PRIMARY'
+        }
+      end
+
+      def extract_indices(indices)
+        indices.
+          map do |row|
+            key_name = struct_key(row, 'Key_name')
+            column_name = struct_key(row, 'COLUMN_NAME')
+            [row[key_name], row[column_name]]
+          end.
+          inject(Hash.new { |h, k| h[k] = [] }) do |memo, (idx, column)|
+            memo[idx] << column
+            memo
+          end
+      end
+
+      def extract_primary_key(schema)
+        cols = schema.select do |defn|
+          column_key = struct_key(defn, 'COLUMN_KEY')
+          defn[column_key] == 'PRI'
+        end
+
+        keys = cols.map do |defn|
+          column_name = struct_key(defn, 'COLUMN_NAME')
+          defn[column_name]
+        end
+
+        keys.length == 1 ? keys.first : keys
+      end
+    end
+  end
+end

data/lib/lhm/table_name.rb

@@ -0,0 +1,23 @@
+module Lhm
+  class TableName
+    def initialize(original, time = Time.now)
+      @original = original
+      @time = time
+      @timestamp = Timestamp.new(time)
+    end
+
+    attr_reader :original
+
+    def archived
+      "lhma_#{@timestamp}_#{@original}"[0...64]
+    end
+
+    def failed
+      archived[0...57] + "_failed"
+    end
+
+    def new
+      "lhmn_#{@original}"[0...64]
+    end
+  end
+end

data/lib/lhm/test_support.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module Lhm
+  module TestMigrator
+    def initialize(*)
+      super
+      @name = @origin.name
+    end
+
+    def execute
+      @statements.each do |stmt|
+        @connection.execute(tagged(stmt))
+      end
+    end
+  end
+
+  module TestInvoker
+    def run(options = {})
+      normalize_options(options)
+      set_session_lock_wait_timeouts
+      @migrator.run
+    rescue => e
+      Lhm.logger.error("LHM run failed with exception=#{e.class} message=#{e.message}")
+      raise
+    end
+  end
+
+  # Patch LHM to execute ALTER TABLE directly on original tables,
+  # without the online migration dance.
+  # This mode is designed for local/CI environments where we can speed
+  # things up by not invoking "real" LHM logic.
+  def self.execute_inline!
+    Lhm::Migrator.prepend(TestMigrator)
+    Lhm::Invoker.prepend(TestInvoker)
+  end
+end

data/lib/lhm/throttler.rb

@@ -0,0 +1,36 @@
+require 'lhm/throttler/time'
+require 'lhm/throttler/slave_lag'
+require 'lhm/throttler/threads_running'
+
+module Lhm
+  module Throttler
+    CLASSES = { :time_throttler => Throttler::Time,
+                :slave_lag_throttler => Throttler::SlaveLag,
+                :threads_running_throttler => Throttler::ThreadsRunning }
+
+    def throttler
+      @throttler ||= Throttler::Time.new
+    end
+
+    def setup_throttler(type, options = {})
+      @throttler = Factory.create_throttler(type, options)
+    end
+
+    class Factory
+      def self.create_throttler(type, options = {})
+        case type
+        when Lhm::Command
+          type
+        when Symbol
+          CLASSES[type].new(options)
+        when String
+          CLASSES[type.to_sym].new(options)
+        when Class
+          type.new(options)
+        else
+          raise ArgumentError, 'type argument must be a Symbol, String or Class'
+        end
+      end
+    end
+  end
+end

data/lib/lhm/throttler/slave_lag.rb

@@ -0,0 +1,145 @@
+module Lhm
+  module Throttler
+
+    def self.format_hosts(hosts)
+      formatted_hosts = []
+      hosts.each do |host|
+        if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
+          formatted_hosts << host.partition(':')[0]
+        end
+      end
+      formatted_hosts
+    end
+
+    class SlaveLag
+      include Command
+
+      INITIAL_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 2_000
+      DEFAULT_MAX_ALLOWED_LAG = 10
+
+      MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
+
+      attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
+
+      def initialize(options = {})
+        @timeout_seconds = INITIAL_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+        @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
+        @slaves = {}
+        @get_config = options[:current_config]
+        @check_only = options[:check_only]
+      end
+
+      def execute
+        sleep(throttle_seconds)
+      end
+
+      private
+
+      def throttle_seconds
+        lag = max_current_slave_lag
+
+        if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
+          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of slave lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds * 2
+        elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
+          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of slave lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
+          @timeout_seconds = @timeout_seconds / 2
+        else
+          @timeout_seconds
+        end
+      end
+
+      def slaves
+        @slaves[@connection] ||= get_slaves
+      end
+
+      def get_slaves
+        slaves = []
+        if @check_only.nil? or !@check_only.respond_to?(:call)
+          slave_hosts = master_slave_hosts
+          while slave_hosts.any? do
+            host = slave_hosts.pop
+            slave = Slave.new(host, @get_config)
+            if !slaves.map(&:host).include?(host) && slave.connection
+              slaves << slave
+              slave_hosts.concat(slave.slave_hosts)
+            end
+          end
+        else
+          slave_config = @check_only.call
+          slaves << Slave.new(slave_config['host'], @get_config)
+        end
+        slaves
+      end
+
+      def master_slave_hosts
+        Throttler.format_hosts(@connection.select_values(Slave::SQL_SELECT_SLAVE_HOSTS))
+      end
+
+      def max_current_slave_lag
+        max = slaves.map { |slave| slave.lag }.push(0).max
+        Lhm.logger.info "Max current slave lag: #{max}"
+        max
+      end
+    end
+
+    class Slave
+      SQL_SELECT_SLAVE_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
+      SQL_SELECT_MAX_SLAVE_LAG = 'SHOW SLAVE STATUS'
+
+      attr_reader :host, :connection
+
+      def initialize(host, connection_config = nil)
+        @host = host
+        @connection_config = prepare_connection_config(connection_config)
+        @connection = client(@connection_config)
+      end
+
+      def slave_hosts
+        Throttler.format_hosts(query_connection(SQL_SELECT_SLAVE_HOSTS, 'host'))
+      end
+
+      def lag
+        query_connection(SQL_SELECT_MAX_SLAVE_LAG, 'Seconds_Behind_Master').first.to_i
+      end
+
+      private
+
+      def client(config)
+        begin
+          Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
+          Mysql2::Client.new(config)
+        rescue Mysql2::Error => e
+          Lhm.logger.info "Error connecting to #{@host}: #{e}"
+          nil
+        end
+      end
+
+      def prepare_connection_config(config_proc)
+        config = if config_proc
+          if config_proc.respond_to?(:call) # if we get a proc
+            config_proc.call
+          else
+            raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
+          end
+        else # otherwise default to ActiveRecord provided config
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+        config.deep_symbolize_keys!
+        config[:host] = @host
+        config
+      end
+
+      def query_connection(query, result)
+        begin
+          @connection.query(query).map { |row| row[result] }
+        rescue Mysql2::Error => e
+          Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
+          [nil]
+        end
+      end
+    end
+  end
+end

data/lib/lhm/throttler/threads_running.rb

@@ -0,0 +1,53 @@
+module Lhm
+  module Throttler
+    class ThreadsRunning
+      include Command
+
+      DEFAULT_INITIAL_TIMEOUT = 0.1
+      DEFAULT_HEALTHY_RANGE = (0..50)
+
+      attr_accessor :timeout_seconds, :healthy_range, :connection
+      attr_reader :max_timeout_seconds, :initial_timeout_seconds
+
+      def initialize(options = {})
+        @initial_timeout_seconds = options[:initial_timeout] || DEFAULT_INITIAL_TIMEOUT
+        @max_timeout_seconds = options[:max_timeout] || (@initial_timeout_seconds * 1024)
+        @timeout_seconds = @initial_timeout_seconds
+        @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
+        @connection = options[:connection]
+      end
+
+      def threads_running
+        query = <<~SQL.squish
+              SELECT COUNT(*) as Threads_running
+              FROM (
+                SELECT 1 FROM performance_schema.threads
+                WHERE NAME='thread/sql/one_connection'
+                  AND PROCESSLIST_STATE IS NOT NULL
+                LIMIT #{@healthy_range.max + 1}
+              ) AS LIM
+        SQL
+
+        @connection.select_value(query)
+      end
+
+      def throttle_seconds
+        current_threads_running = threads_running
+
+        if !healthy_range.cover?(current_threads_running) && @timeout_seconds < @max_timeout_seconds
+          Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because threads running is greater than the maximum of #{@healthy_range.max} allowed.")
+          @timeout_seconds = @timeout_seconds * 2
+        elsif healthy_range.cover?(current_threads_running) && @timeout_seconds > @initial_timeout_seconds
+          Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because threads running is less than the maximum of #{@healthy_range.max} allowed.")
+          @timeout_seconds = @timeout_seconds / 2
+        else
+          @timeout_seconds
+        end
+      end
+
+      def execute
+        sleep throttle_seconds
+      end
+    end
+  end
+end

data/lib/lhm/throttler/time.rb

@@ -0,0 +1,29 @@
+module Lhm
+  module Throttler
+    class Time
+      include Command
+
+      DEFAULT_TIMEOUT = 0.1
+      DEFAULT_STRIDE = 2_000
+
+      attr_accessor :timeout_seconds
+      attr_accessor :stride
+
+      def initialize(options = {})
+        @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
+        @stride = options[:stride] || DEFAULT_STRIDE
+      end
+
+      def execute
+        sleep timeout_seconds
+      end
+    end
+
+    class LegacyTime < Time
+      def initialize(timeout, stride)
+        @timeout_seconds = timeout / 1000.0
+        @stride = stride
+      end
+    end
+  end
+end