lhm 1.2.0 → 2.2.0

data/lib/lhm/atomic_switcher.rb

@@ -29,7 +29,7 @@ module Lhm
 
     def atomic_switch
       [
-        "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " +
+        "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " \
         "`#{ @destination.name }` to `#{ @origin.name }`"
       ]
     end
@@ -42,6 +42,7 @@ module Lhm
     end
 
   private
+
     def execute
       @connection.sql(statements)
     end

data/lib/lhm/chunker.rb

@@ -1,8 +1,8 @@
 # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
 # Schmidt
-
 require 'lhm/command'
 require 'lhm/sql_helper'
+require 'lhm/printer'
 
 module Lhm
   class Chunker
@@ -16,53 +16,74 @@ module Lhm
     def initialize(migration, connection = nil, options = {})
       @migration = migration
       @connection = connection
-      @stride = options[:stride] || 40_000
-      @throttle = options[:throttle] || 100
+      @throttler = options[:throttler]
       @start = options[:start] || select_start
       @limit = options[:limit] || select_limit
+      @printer = options[:printer] || Printer::Percentage.new
     end
 
-    # Copies chunks of size `stride`, starting from `start` up to id `limit`.
-    def up_to(&block)
-      1.upto(traversable_chunks_size) do |n|
-        yield(bottom(n), top(n))
+    def execute
+      return unless @start && @limit
+      @next_to_insert = @start
+      while @next_to_insert < @limit || (@next_to_insert == 1 && @start == 1)
+        stride = @throttler.stride
+        affected_rows = @connection.update(copy(bottom, top(stride)))
+
+        if @throttler && affected_rows > 0
+          @throttler.run
+        end
+
+        @printer.notify(bottom, @limit)
+        @next_to_insert = top(stride) + 1
       end
+      @printer.end
     end
 
-    def traversable_chunks_size
-      @limit && @start ? ((@limit - @start + 1) / @stride.to_f).ceil : 0
-    end
+  private
 
-    def bottom(chunk)
-      (chunk - 1) * @stride + @start
+    def bottom
+      @next_to_insert
     end
 
-    def top(chunk)
-      [chunk * @stride + @start - 1, @limit].min
+    def top(stride)
+      [(@next_to_insert + stride - 1), @limit].min
     end
 
     def copy(lowest, highest)
-      "insert ignore into `#{ destination_name }` (#{ columns }) " +
-      "select #{ columns } from `#{ origin_name }` " +
-      "where `id` between #{ lowest } and #{ highest }"
+      "insert ignore into `#{ destination_name }` (#{ destination_columns }) " \
+      "select #{ origin_columns } from `#{ origin_name }` " \
+      "#{ conditions } `#{ origin_name }`.`id` between #{ lowest } and #{ highest }"
     end
 
     def select_start
-      start = connection.select_value("select min(id) from #{ origin_name }")
+      start = connection.select_value("select min(id) from `#{ origin_name }`")
       start ? start.to_i : nil
     end
 
     def select_limit
-      limit = connection.select_value("select max(id) from #{ origin_name }")
+      limit = connection.select_value("select max(id) from `#{ origin_name }`")
       limit ? limit.to_i : nil
     end
 
-    def throttle_seconds
-      @throttle / 1000.0
+    # XXX this is extremely brittle and doesn't work when filter contains more
+    # than one SQL clause, e.g. "where ... group by foo". Before making any
+    # more changes here, please consider either:
+    #
+    # 1. Letting users only specify part of defined clauses (i.e. don't allow
+    # `filter` on Migrator to accept both WHERE and INNER JOIN
+    # 2. Changing query building so that it uses structured data rather than
+    # strings until the last possible moment.
+    def conditions
+      if @migration.conditions
+        @migration.conditions.
+          sub(/\)\Z/, '').
+          # put any where conditions in parens
+          sub(/where\s(\w.*)\Z/, 'where (\\1)') + ' and'
+      else
+        'where'
+      end
     end
 
-  private
-
     def destination_name
       @migration.destination.name
     end
@@ -71,27 +92,18 @@ module Lhm
       @migration.origin.name
     end
 
-    def columns
-      @columns ||= @migration.intersection.joined
+    def origin_columns
+      @origin_columns ||= @migration.intersection.origin.typed(origin_name)
     end
 
-    def validate
-      if @start && @limit && @start > @limit
-        error("impossible chunk options (limit must be greater than start)")
-      end
+    def destination_columns
+      @destination_columns ||= @migration.intersection.destination.joined
     end
 
-    def execute
-      up_to do |lowest, highest|
-        affected_rows = @connection.update(copy(lowest, highest))
-
-        if affected_rows > 0
-          sleep(throttle_seconds)
-        end
-
-        print "."
+    def validate
+      if @start && @limit && @start > @limit
+        error('impossible chunk options (limit must be greater than start)')
       end
-      print "\n"
     end
   end
 end

data/lib/lhm/command.rb

@@ -7,16 +7,18 @@ module Lhm
 
   module Command
     def run(&block)
+      Lhm.logger.info "Starting run of class=#{self.class}"
       validate
 
-      if(block_given?)
+      if block_given?
         before
         block.call(self)
         after
       else
         execute
       end
-    rescue
+    rescue => e
+      Lhm.logger.error "Error in class=#{self.class}, reverting. exception=#{e.class} message=#{e.message}"
       revert
       raise
     end

data/lib/lhm/connection.rb

@@ -28,7 +28,7 @@ module Lhm
 
       def show_create(table_name)
         sql = "show create table `#{ table_name }`"
-        select_values(sql).last
+        select_one(sql).values.last
       end
 
       def current_database
@@ -51,7 +51,7 @@ module Lhm
       end
 
       def select_values(sql)
-        select_one(sql).values
+        select_all(sql)
       end
 
       def select_value(sql)
@@ -77,6 +77,14 @@ module Lhm
              and table_name = '#{ table_name }'
         })
       end
+
+      def quote_value(value)
+        quoter.quote_value(value)
+      end
+
+      def quoter
+        @quoter ||= Object.new.tap { |o| o.extend(DataObjects::Quoting) }
+      end
     end
 
     class ActiveRecordConnection
@@ -138,6 +146,10 @@ module Lhm
       def table_exists?(table_name)
         @adapter.table_exists?(table_name)
       end
+
+      def quote_value(value)
+        @adapter.quote(value)
+      end
     end
   end
 end

data/lib/lhm/entangler.rb

@@ -14,7 +14,7 @@ module Lhm
     # Creates entanglement between two tables. All creates, updates and deletes
     # to origin will be repeated on the destination table.
     def initialize(migration, connection = nil)
-      @common = migration.intersection
+      @intersection = migration.intersection
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
@@ -40,8 +40,8 @@ module Lhm
       strip %Q{
         create trigger `#{ trigger(:ins) }`
         after insert on `#{ @origin.name }` for each row
-        replace into `#{ @destination.name }` (#{ @common.joined }) #{ SqlHelper.annotation }
-        values (#{ @common.typed("NEW") })
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
       }
     end
 
@@ -49,8 +49,8 @@ module Lhm
       strip %Q{
         create trigger `#{ trigger(:upd) }`
         after update on `#{ @origin.name }` for each row
-        replace into `#{ @destination.name }` (#{ @common.joined }) #{ SqlHelper.annotation }
-        values (#{ @common.typed("NEW") })
+        replace into `#{ @destination.name }` (#{ @intersection.destination.joined }) #{ SqlHelper.annotation }
+        values (#{ @intersection.origin.typed('NEW') })
       }
     end
 

data/lib/lhm/intersection.rb

@@ -4,35 +4,48 @@
 module Lhm
   #  Determine and format columns common to origin and destination.
   class Intersection
-    def initialize(origin, destination)
+    def initialize(origin, destination, renames = {})
       @origin = origin
       @destination = destination
+      @renames = renames
     end
 
-    def common
-      (@origin.columns.keys & @destination.columns.keys).sort
+    def origin
+      (common + @renames.keys).extend(Joiners)
     end
 
-    def escaped
-      common.map { |name| tick(name)  }
+    def destination
+      (common + @renames.values).extend(Joiners)
     end
 
-    def joined
-      escaped.join(", ")
-    end
+    private
 
-    def typed(type)
-      common.map { |name| qualified(name, type)  }.join(", ")
+    def common
+      (@origin.columns.keys & @destination.columns.keys).sort
     end
 
-  private
+    module Joiners
+      def escaped
+        self.map { |name| tick(name)  }
+      end
 
-    def qualified(name, type)
-      "#{ type }.`#{ name }`"
-    end
+      def joined
+        escaped.join(', ')
+      end
+
+      def typed(type)
+        self.map { |name| qualified(name, type)  }.join(', ')
+      end
+
+    private
+
+      def qualified(name, type)
+        "`#{ type }`.`#{ name }`"
+      end
 
-    def tick(name)
-      "`#{ name }`"
+      def tick(name)
+        "`#{ name }`"
+      end
     end
   end
 end

data/lib/lhm/invoker.rb

@@ -24,16 +24,7 @@ module Lhm
     end
 
     def run(options = {})
-      if !options.include?(:atomic_switch)
-        if supports_atomic_switch?
-          options[:atomic_switch] = true
-        else
-          raise Error.new(
-            "Using mysql #{version_string}. You must explicitly set " +
-            "options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)")
-        end
-      end
-
+      normalize_options(options)
       migration = @migrator.run
 
       Entangler.new(migration, @connection).run do
@@ -45,5 +36,35 @@ module Lhm
         end
       end
     end
+
+    private
+
+    def normalize_options(options)
+      Lhm.logger.info "Starting LHM run on table=#{@migrator.name}"
+
+      if !options.include?(:atomic_switch)
+        if supports_atomic_switch?
+          options[:atomic_switch] = true
+        else
+          raise Error.new(
+            "Using mysql #{version_string}. You must explicitly set " \
+            'options[:atomic_switch] (re SqlHelper#supports_atomic_switch?)')
+        end
+      end
+
+      if options[:throttler]
+        options[:throttler] = Throttler::Factory.create_throttler(*options[:throttler])
+      elsif options[:throttle] || options[:stride]
+        # we still support the throttle and stride as a Fixnum input
+        warn 'throttle option will no longer accept a Fixnum in the next versions.'
+        options[:throttler] = Throttler::LegacyTime.new(options[:throttle], options[:stride])
+      else
+        options[:throttler] = Lhm.throttler
+      end
+
+    rescue => e
+      Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
+      raise
+    end
   end
 end

data/lib/lhm/locked_switcher.rb

@@ -38,17 +38,17 @@ module Lhm
         "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
         "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
         "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
-        "commit",
-        "unlock tables"
+        'commit',
+        'unlock tables'
       ]
     end
 
     def uncommitted(&block)
       [
-        "set @lhm_auto_commit = @@session.autocommit",
-        "set session autocommit = 0",
+        'set @lhm_auto_commit = @@session.autocommit',
+        'set session autocommit = 0',
         yield,
-        "set session autocommit = @lhm_auto_commit"
+        'set session autocommit = @lhm_auto_commit'
       ].flatten
     end
 
@@ -62,7 +62,7 @@ module Lhm
   private
 
     def revert
-      @connection.sql("unlock tables")
+      @connection.sql('unlock tables')
     end
 
     def execute

data/lib/lhm/migration.rb

@@ -5,24 +5,26 @@ require 'lhm/intersection'
 
 module Lhm
   class Migration
-    attr_reader :origin, :destination
+    attr_reader :origin, :destination, :conditions, :renames
 
-    def initialize(origin, destination, time = Time.now)
+    def initialize(origin, destination, conditions = nil, renames = {}, time = Time.now)
       @origin = origin
       @destination = destination
+      @conditions = conditions
       @start = time
+      @renames = renames
     end
 
     def archive_name
-      "lhma_#{ startstamp }_#{ @origin.name }"
+      "lhma_#{ startstamp }_#{ @origin.name }"[0...64]
     end
 
     def intersection
-      Intersection.new(@origin, @destination)
+      Intersection.new(@origin, @destination, @renames)
     end
 
     def startstamp
-      @start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
+      @start.strftime "%Y_%m_%d_%H_%M_%S_#{ '%03d' % (@start.usec / 1000) }"
     end
   end
 end

data/lib/lhm/migrator.rb

@@ -13,13 +13,14 @@ module Lhm
     include Command
     include SqlHelper
 
-    attr_reader :name, :statements, :connection
+    attr_reader :name, :statements, :connection, :conditions, :renames
 
     def initialize(table, connection = nil)
       @connection = connection
       @origin = table
       @name = table.destination_name
       @statements = []
+      @renames = {}
     end
 
     # Alter a table with a custom statement
@@ -52,7 +53,7 @@ module Lhm
     # @param [String] name Name of the column to add
     # @param [String] definition Valid SQL column definition
     def add_column(name, definition)
-      ddl("alter table `%s` add column `%s` %s" % [@name, name, definition])
+      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition])
     end
 
     # Change an existing column to a new definition
@@ -66,7 +67,28 @@ module Lhm
     # @param [String] name Name of the column to change
     # @param [String] definition Valid SQL column definition
     def change_column(name, definition)
-      ddl("alter table `%s` modify column `%s` %s" % [@name, name, definition])
+      ddl('alter table `%s` modify column `%s` %s' % [@name, name, definition])
+    end
+
+    # Rename an existing column.
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.rename_column(:login, :username)
+    #   end
+    #
+    # @param [String] old Name of the column to change
+    # @param [String] nu New name to use for the column
+    def rename_column(old, nu)
+      col = @origin.columns[old.to_s]
+
+      definition = col[:type]
+      definition += ' NOT NULL' unless col[:is_nullable]
+      definition += " DEFAULT #{@connection.quote_value(col[:column_default])}" if col[:column_default]
+
+      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition])
+      @renames[old.to_s] = nu.to_s
     end
 
     # Remove a column from a table
@@ -79,7 +101,7 @@ module Lhm
     #
     # @param [String] name Name of the column to delete
     def remove_column(name)
-      ddl("alter table `%s` drop `%s`" % [@name, name])
+      ddl('alter table `%s` drop `%s`' % [@name, name])
     end
 
     # Add an index to a table
@@ -135,8 +157,27 @@ module Lhm
     # @param [String, Symbol] index_name
     #   Optional name of the index to be removed
     def remove_index(columns, index_name = nil)
+      columns = [columns].flatten.map(&:to_sym)
+      from_origin = @origin.indices.find { |name, cols| cols.map(&:to_sym) == columns }
+      index_name ||= from_origin[0] unless from_origin.nil?
       index_name ||= idx_name(@origin.name, columns)
-      ddl("drop index `%s` on `%s`" % [index_name, @name])
+      ddl('drop index `%s` on `%s`' % [index_name, @name])
+    end
+
+    # Filter the data that is copied into the new table by the provided SQL.
+    # This SQL will be inserted into the copy directly after the "from"
+    # statement - so be sure to use inner/outer join syntax and not cross joins.
+    #
+    # @example Add a conditions filter to the migration.
+    #   Lhm.change_table(:sounds) do |m|
+    #     m.filter("inner join users on users.`id` = sounds.`user_id` and sounds.`public` = 1")
+    #   end
+    #
+    # @param [ String ] sql The sql filter.
+    #
+    # @return [ String ] The sql filter.
+    def filter(sql)
+      @conditions = sql
     end
 
   private
@@ -147,7 +188,7 @@ module Lhm
       end
 
       unless @origin.satisfies_primary_key?
-        error("origin does not satisfy primary key requirements")
+        error('origin does not satisfy primary key requirements')
       end
 
       dest = @origin.destination_name
@@ -160,7 +201,7 @@ module Lhm
     def execute
       destination_create
       @connection.sql(@statements)
-      Migration.new(@origin, destination_read)
+      Migration.new(@origin, destination_read, conditions, renames)
     end
 
     def destination_create
@@ -172,10 +213,17 @@ module Lhm
     end
 
     def index_ddl(cols, unique = nil, index_name = nil)
-      type = unique ? "unique index" : "index"
+      assert_valid_idx_name(index_name)
+      type = unique ? 'unique index' : 'index'
       index_name ||= idx_name(@origin.name, cols)
       parts = [type, index_name, @name, idx_spec(cols)]
-      "create %s `%s` on `%s` (%s)" % parts
+      'create %s `%s` on `%s` (%s)' % parts
+    end
+
+    def assert_valid_idx_name(index_name)
+      if index_name && !(index_name.is_a?(String) || index_name.is_a?(Symbol))
+        raise ArgumentError, 'index_name must be a string or symbol'
+      end
     end
   end
 end

data/lib/lhm/printer.rb

@@ -0,0 +1,54 @@
+module Lhm
+  module Printer
+    class Output
+      def write(message)
+        print message
+      end
+    end
+
+    class Base
+      def initialize
+        @output = Output.new
+      end
+    end
+
+    class Percentage < Base
+      def initialize
+        super
+        @max_length = 0
+      end
+
+      def notify(lowest, highest)
+        return if !highest || highest == 0
+        message = "%.2f%% (#{lowest}/#{highest}) complete" % (lowest.to_f / highest * 100.0)
+        write(message)
+      end
+
+      def end
+        write('100% complete')
+        @output.write "\n"
+      end
+
+      private
+
+      def write(message)
+        if (extra = @max_length - message.length) < 0
+          @max_length = message.length
+          extra = 0
+        end
+
+        @output.write "\r#{message}" + (' ' * extra)
+      end
+    end
+
+    class Dot < Base
+      def notify(lowest = nil, highest = nil)
+        @output.write '.'
+      end
+
+      def end
+        @output.write "\n"
+      end
+    end
+  end
+end

data/lib/lhm/sql_helper.rb

@@ -6,12 +6,12 @@ module Lhm
     extend self
 
     def annotation
-      "/* large hadron migration */"
+      '/* large hadron migration */'
     end
 
     def idx_name(table_name, cols)
       column_names = column_definition(cols).map(&:first)
-      "index_#{ table_name }_on_#{ column_names.join("_and_") }"
+      "index_#{ table_name }_on_#{ column_names.join('_and_') }"
     end
 
     def idx_spec(cols)
@@ -22,7 +22,7 @@ module Lhm
 
     def version_string
       row = connection.select_one("show variables like 'version'")
-      value = struct_key(row, "Value")
+      value = struct_key(row, 'Value')
       row[value]
     end
 
@@ -71,7 +71,7 @@ module Lhm
                struct.members
              end
 
-      keys.find {|k| k.to_s.downcase == key.to_s.downcase }
+      keys.find { |k| k.to_s.downcase == key.to_s.downcase }
     end
   end
 end