lhm 2.1.0 → 2.2.0

data/bin/lhm-spec-clobber.sh

@@ -3,7 +3,7 @@
 set -e
 set -u
 
-source ~/.lhm
+source `dirname $0`/lhm-config.sh
 
 lhmkill() {
   echo killing lhm-cluster

data/bin/lhm-spec-grants.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source ~/.lhm
+source `dirname $0`/lhm-config.sh
 
 master() { "$mysqldir"/bin/mysql --protocol=TCP -P $master_port -uroot; }
 slave()  { "$mysqldir"/bin/mysql --protocol=TCP -P $slave_port -uroot; }
@@ -12,7 +12,7 @@ echo "grant replication slave on *.* to 'slave'@'localhost'" | master
 
 # set up slave
 
-echo "change master to master_user = 'slave', master_password = 'slave', master_port = 3306, master_host = 'localhost'" | slave
+echo "change master to master_user = 'slave', master_password = 'slave', master_port = $master_port, master_host = 'localhost'" | slave
 echo "start slave" | slave
 echo "show slave status \G" | slave
 

data/bin/lhm-spec-setup-cluster.sh

@@ -7,7 +7,7 @@
 set -e
 set -u
 
-source ~/.lhm
+source `dirname $0`/lhm-config.sh
 
 #
 # Main

data/gemfiles/ar-2.3_mysql.gemfile

@@ -2,4 +2,5 @@ source "https://rubygems.org"
 
 gem "mysql", "~> 2.8.1"
 gem "activerecord", "~> 2.3.14"
+gem "iconv", "~> 1.0.4"
 gemspec :path=>"../"

data/lhm.gemspec

@@ -6,19 +6,19 @@ $:.unshift(lib) unless $:.include?(lib)
 require 'lhm/version'
 
 Gem::Specification.new do |s|
-  s.name          = "lhm"
+  s.name          = 'lhm'
   s.version       = Lhm::VERSION
   s.platform      = Gem::Platform::RUBY
-  s.authors       = ["SoundCloud", "Rany Keddo", "Tobias Bielohlawek", "Tobias Schmidt"]
+  s.authors       = ['SoundCloud', 'Rany Keddo', 'Tobias Bielohlawek', 'Tobias Schmidt']
   s.email         = %q{rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com}
   s.summary       = %q{online schema changer for mysql}
   s.description   = %q{Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.}
   s.homepage      = %q{http://github.com/soundcloud/lhm}
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.require_paths = ["lib"]
-  s.executables   = ["lhm-kill-queue"]
+  s.require_paths = ['lib']
+  s.executables   = ['lhm-kill-queue']
 
-  s.add_development_dependency "minitest", "~> 5.0.8"
-  s.add_development_dependency "rake"
+  s.add_development_dependency 'minitest', '~> 5.0.8'
+  s.add_development_dependency 'rake'
 end

data/lib/lhm.rb

@@ -51,9 +51,22 @@ module Lhm
     true
   end
 
-  def cleanup(run = false)
-    lhm_tables = connection.select_values("show tables").select { |name| name =~ /^lhm(a|n)_/ }
-    lhm_triggers = connection.select_values("show triggers").collect do |trigger|
+  # Cleanup tables and triggers
+  #
+  # @param [Boolean] run execute now or just display information
+  # @param [Hash] options Optional options to alter cleanup behaviour
+  # @option options [Time] :until
+  #   Filter to only remove tables up to specified time (defaults to: nil)
+  def cleanup(run = false, options = {})
+    lhm_tables = connection.select_values('show tables').select { |name| name =~ /^lhm(a|n)_/ }
+    if options[:until]
+      lhm_tables.select! { |table|
+        table_date_time = Time.strptime(table, 'lhma_%Y_%m_%d_%H_%M_%S')
+        table_date_time <= options[:until]
+      }
+    end
+
+    lhm_triggers = connection.select_values('show triggers').collect do |trigger|
       trigger.respond_to?(:trigger) ? trigger.trigger : trigger
     end.select { |name| name =~ /^lhmt/ }
 
@@ -66,12 +79,12 @@ module Lhm
       end
       true
     elsif lhm_tables.empty? && lhm_triggers.empty?
-      puts "Everything is clean. Nothing to do."
+      puts 'Everything is clean. Nothing to do.'
       true
     else
-      puts "Existing LHM backup tables: #{lhm_tables.join(", ")}."
-      puts "Existing LHM triggers: #{lhm_triggers.join(", ")}."
-      puts "Run Lhm.cleanup(true) to drop them all."
+      puts "Existing LHM backup tables: #{lhm_tables.join(', ')}."
+      puts "Existing LHM triggers: #{lhm_triggers.join(', ')}."
+      puts 'Run Lhm.cleanup(true) to drop them all.'
       false
     end
   end
@@ -107,5 +120,4 @@ module Lhm
   def connection
     Connection.new(adapter)
   end
-
 end

data/lib/lhm/atomic_switcher.rb

@@ -29,7 +29,7 @@ module Lhm
 
     def atomic_switch
       [
-        "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+        "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " \
         "`#{ @destination.name }` to `#{ @origin.name }`"
       ]
     end
@@ -42,6 +42,7 @@ module Lhm
     end
 
   private
+
     def execute
       @connection.sql(statements)
     end

data/lib/lhm/chunker.rb

@@ -25,7 +25,7 @@ module Lhm
     def execute
       return unless @start && @limit
       @next_to_insert = @start
-      until @next_to_insert >= @limit
+      while @next_to_insert < @limit || (@next_to_insert == 1 && @start == 1)
         stride = @throttler.stride
         affected_rows = @connection.update(copy(bottom, top(stride)))
 
@@ -50,8 +50,8 @@ module Lhm
     end
 
     def copy(lowest, highest)
-      "insert ignore into `#{ destination_name }` (#{ columns }) " +
-      "select #{ select_columns } from `#{ origin_name }` " +
+      "insert ignore into `#{ destination_name }` (#{ destination_columns }) " \
+      "select #{ origin_columns } from `#{ origin_name }` " \
       "#{ conditions } `#{ origin_name }`.`id` between #{ lowest } and #{ highest }"
     end
 
@@ -65,22 +65,22 @@ module Lhm
       limit ? limit.to_i : nil
     end
 
-    #XXX this is extremely brittle and doesn't work when filter contains more
-    #than one SQL clause, e.g. "where ... group by foo". Before making any
-    #more changes here, please consider either:
+    # XXX this is extremely brittle and doesn't work when filter contains more
+    # than one SQL clause, e.g. "where ... group by foo". Before making any
+    # more changes here, please consider either:
     #
-    #1. Letting users only specify part of defined clauses (i.e. don't allow
-    #`filter` on Migrator to accept both WHERE and INNER JOIN
-    #2. Changing query building so that it uses structured data rather than
-    #strings until the last possible moment.
+    # 1. Letting users only specify part of defined clauses (i.e. don't allow
+    # `filter` on Migrator to accept both WHERE and INNER JOIN
+    # 2. Changing query building so that it uses structured data rather than
+    # strings until the last possible moment.
     def conditions
       if @migration.conditions
         @migration.conditions.
-          sub(/\)\Z/, "").
-          #put any where conditions in parens
-          sub(/where\s(\w.*)\Z/, "where (\\1)") + " and"
+          sub(/\)\Z/, '').
+          # put any where conditions in parens
+          sub(/where\s(\w.*)\Z/, 'where (\\1)') + ' and'
       else
-        "where"
+        'where'
       end
     end
 
@@ -92,17 +92,17 @@ module Lhm
       @migration.origin.name
     end
 
-    def columns
-      @columns ||= @migration.intersection.joined
+    def origin_columns
+      @origin_columns ||= @migration.intersection.origin.typed(origin_name)
     end
 
-    def select_columns
-      @select_columns ||= @migration.intersection.typed("`#{origin_name}`")
+    def destination_columns
+      @destination_columns ||= @migration.intersection.destination.joined
     end
 
     def validate
       if @start && @limit && @start > @limit
-        error("impossible chunk options (limit must be greater than start)")
+        error('impossible chunk options (limit must be greater than start)')
       end
     end
   end

data/lib/lhm/command.rb

@@ -10,7 +10,7 @@ module Lhm
       Lhm.logger.info "Starting run of class=#{self.class}"
       validate
 
-      if(block_given?)
+      if block_given?
         before
         block.call(self)
         after

data/lib/lhm/connection.rb

@@ -77,6 +77,14 @@ module Lhm
              and table_name = '#{ table_name }'
         })
       end
+
+      def quote_value(value)
+        quoter.quote_value(value)
+      end
+
+      def quoter
+        @quoter ||= Object.new.tap { |o| o.extend(DataObjects::Quoting) }
+      end
     end
 
     class ActiveRecordConnection
@@ -138,6 +146,10 @@ module Lhm
       def table_exists?(table_name)
         @adapter.table_exists?(table_name)
       end
+
+      def quote_value(value)
+        @adapter.quote(value)
+      end
     end
   end
 end

data/lib/lhm/entangler.rb

@@ -14,7 +14,7 @@ module Lhm
     # Creates entanglement between two tables. All creates, updates and deletes
     # to origin will be repeated on the destination table.
     def initialize(migration, connection = nil)
-      @common = migration.intersection
+      @intersection = migration.intersection
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
@@ -40,8 +40,8 @@ module Lhm
       strip %Q{
         create trigger `#{ trigger(:ins) }`
         after insert on `#{ @origin.name }` for each row
-        replace into `#{ @destination.name }` (#{ @common.joined }) #{ SqlHelper.annotation }
-        values (#{ @common.typed("NEW") })
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
       }
     end
 
@@ -49,8 +49,8 @@ module Lhm
       strip %Q{
         create trigger `#{ trigger(:upd) }`
         after update on `#{ @origin.name }` for each row
-        replace into `#{ @destination.name }` (#{ @common.joined }) #{ SqlHelper.annotation }
-        values (#{ @common.typed("NEW") })
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
       }
     end
 

data/lib/lhm/intersection.rb

@@ -4,35 +4,48 @@
 module Lhm
   #  Determine and format columns common to origin and destination.
   class Intersection
-    def initialize(origin, destination)
+    def initialize(origin, destination, renames = {})
       @origin = origin
       @destination = destination
+      @renames = renames
     end
 
-    def common
-      (@origin.columns.keys & @destination.columns.keys).sort
+    def origin
+      (common + @renames.keys).extend(Joiners)
     end
 
-    def escaped
-      common.map { |name| tick(name)  }
+    def destination
+      (common + @renames.values).extend(Joiners)
     end
 
-    def joined
-      escaped.join(", ")
-    end
+    private
 
-    def typed(type)
-      common.map { |name| qualified(name, type)  }.join(", ")
+    def common
+      (@origin.columns.keys & @destination.columns.keys).sort
     end
 
-  private
+    module Joiners
+      def escaped
+        self.map { |name| tick(name)  }
+      end
 
-    def qualified(name, type)
-      "#{ type }.`#{ name }`"
-    end
+      def joined
+        escaped.join(', ')
+      end
+
+      def typed(type)
+        self.map { |name| qualified(name, type)  }.join(', ')
+      end
+
+    private
+
+      def qualified(name, type)
+        "`#{ type }`.`#{ name }`"
+      end
 
-    def tick(name)
-      "`#{ name }`"
+      def tick(name)
+        "`#{ name }`"
+      end
     end
   end
 end

data/lib/lhm/invoker.rb

@@ -47,8 +47,8 @@ module Lhm
           options[:atomic_switch] = true
         else
           raise Error.new(
-            "Using mysql #{version_string}. You must explicitly set " +
-            "options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)")
+            "Using mysql #{version_string}. You must explicitly set " \
+            'options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)')
         end
       end
 
@@ -56,7 +56,7 @@ module Lhm
         options[:throttler] = Throttler::Factory.create_throttler(*options[:throttler])
       elsif options[:throttle] || options[:stride]
         # we still support the throttle and stride as a Fixnum input
-        warn "throttle option will no longer accept a Fixnum in the next versions."
+        warn 'throttle option will no longer accept a Fixnum in the next versions.'
         options[:throttler] = Throttler::LegacyTime.new(options[:throttle], options[:stride])
       else
         options[:throttler] = Lhm.throttler

data/lib/lhm/locked_switcher.rb

@@ -38,17 +38,17 @@ module Lhm
         "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
         "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
         "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
-        "commit",
-        "unlock tables"
+        'commit',
+        'unlock tables'
       ]
     end
 
     def uncommitted(&block)
       [
-        "set @lhm_auto_commit = @@session.autocommit",
-        "set session autocommit = 0",
+        'set @lhm_auto_commit = @@session.autocommit',
+        'set session autocommit = 0',
         yield,
-        "set session autocommit = @lhm_auto_commit"
+        'set session autocommit = @lhm_auto_commit'
       ].flatten
     end
 
@@ -62,7 +62,7 @@ module Lhm
   private
 
     def revert
-      @connection.sql("unlock tables")
+      @connection.sql('unlock tables')
     end
 
     def execute

data/lib/lhm/migration.rb

@@ -5,13 +5,14 @@ require 'lhm/intersection'
 
 module Lhm
   class Migration
-    attr_reader :origin, :destination, :conditions
+    attr_reader :origin, :destination, :conditions, :renames
 
-    def initialize(origin, destination, conditions = nil, time = Time.now)
+    def initialize(origin, destination, conditions = nil, renames = {}, time = Time.now)
       @origin = origin
       @destination = destination
       @conditions = conditions
       @start = time
+      @renames = renames
     end
 
     def archive_name
@@ -19,11 +20,11 @@ module Lhm
     end
 
     def intersection
-      Intersection.new(@origin, @destination)
+      Intersection.new(@origin, @destination, @renames)
     end
 
     def startstamp
-      @start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
+      @start.strftime "%Y_%m_%d_%H_%M_%S_#{ '%03d' % (@start.usec / 1000) }"
     end
   end
 end

data/lib/lhm/migrator.rb

@@ -13,13 +13,14 @@ module Lhm
     include Command
     include SqlHelper
 
-    attr_reader :name, :statements, :connection, :conditions
+    attr_reader :name, :statements, :connection, :conditions, :renames
 
     def initialize(table, connection = nil)
       @connection = connection
       @origin = table
       @name = table.destination_name
       @statements = []
+      @renames = {}
     end
 
     # Alter a table with a custom statement
@@ -52,7 +53,7 @@ module Lhm
     # @param [String] name Name of the column to add
     # @param [String] definition Valid SQL column definition
     def add_column(name, definition)
-      ddl("alter table `%s` add column `%s` %s" % [@name, name, definition])
+      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition])
     end
 
     # Change an existing column to a new definition
@@ -66,7 +67,28 @@ module Lhm
     # @param [String] name Name of the column to change
     # @param [String] definition Valid SQL column definition
     def change_column(name, definition)
-      ddl("alter table `%s` modify column `%s` %s" % [@name, name, definition])
+      ddl('alter table `%s` modify column `%s` %s' % [@name, name, definition])
+    end
+
+    # Rename an existing column.
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.rename_column(:login, :username)
+    #   end
+    #
+    # @param [String] old Name of the column to change
+    # @param [String] nu New name to use for the column
+    def rename_column(old, nu)
+      col = @origin.columns[old.to_s]
+
+      definition = col[:type]
+      definition += ' NOT NULL' unless col[:is_nullable]
+      definition += " DEFAULT #{@connection.quote_value(col[:column_default])}" if col[:column_default]
+
+      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition])
+      @renames[old.to_s] = nu.to_s
     end
 
     # Remove a column from a table
@@ -79,7 +101,7 @@ module Lhm
     #
     # @param [String] name Name of the column to delete
     def remove_column(name)
-      ddl("alter table `%s` drop `%s`" % [@name, name])
+      ddl('alter table `%s` drop `%s`' % [@name, name])
     end
 
     # Add an index to a table
@@ -136,10 +158,10 @@ module Lhm
     #   Optional name of the index to be removed
     def remove_index(columns, index_name = nil)
       columns = [columns].flatten.map(&:to_sym)
-      from_origin = @origin.indices.find {|name, cols| cols.map(&:to_sym) == columns}
+      from_origin = @origin.indices.find { |name, cols| cols.map(&:to_sym) == columns }
       index_name ||= from_origin[0] unless from_origin.nil?
       index_name ||= idx_name(@origin.name, columns)
-      ddl("drop index `%s` on `%s`" % [index_name, @name])
+      ddl('drop index `%s` on `%s`' % [index_name, @name])
     end
 
     # Filter the data that is copied into the new table by the provided SQL.
@@ -166,7 +188,7 @@ module Lhm
       end
 
       unless @origin.satisfies_primary_key?
-        error("origin does not satisfy primary key requirements")
+        error('origin does not satisfy primary key requirements')
       end
 
       dest = @origin.destination_name
@@ -179,7 +201,7 @@ module Lhm
     def execute
       destination_create
       @connection.sql(@statements)
-      Migration.new(@origin, destination_read, conditions)
+      Migration.new(@origin, destination_read, conditions, renames)
     end
 
     def destination_create
@@ -191,10 +213,17 @@ module Lhm
     end
 
     def index_ddl(cols, unique = nil, index_name = nil)
-      type = unique ? "unique index" : "index"
+      assert_valid_idx_name(index_name)
+      type = unique ? 'unique index' : 'index'
       index_name ||= idx_name(@origin.name, cols)
       parts = [type, index_name, @name, idx_spec(cols)]
-      "create %s `%s` on `%s` (%s)" % parts
+      'create %s `%s` on `%s` (%s)' % parts
+    end
+
+    def assert_valid_idx_name(index_name)
+      if index_name && !(index_name.is_a?(String) || index_name.is_a?(Symbol))
+        raise ArgumentError, 'index_name must be a string or symbol'
+      end
     end
   end
 end