lhm 2.0.0 → 2.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZWY5MTRiZmFmYTBkNzg0NDhhNjM1MjBlOGM5OWQ1YTljMTkzOTYxOQ==
+  data.tar.gz: !binary |-
+    MjE3ODI5NDk3MjUxNmRlNzFkN2ZmYTVmZjZmNWM1OTk3MzA1N2NiNA==
+SHA512:
+  metadata.gz: !binary |-
+    MjBlNGMyNjllMDU5Y2ZmY2E0MjQwOWY0YzIzODk5YjQ0ZDAxOWVlMjU2OTA0
+    ZWUxYmJmMzFiYzg1MTMzNmMzYzI5OWU3Y2M2MjBmOTQ3MzllZjA5MmViMGNh
+    MzQ5NmM0YzVmZGMxZWQ3NzdiNmIxMzI1MGE2ZDM0MTU4NGU1NjQ=
+  data.tar.gz: !binary |-
+    ZmFlNTRkMDkyOWJlOTkzZjA2ZTdiNjA1MGNjZWVjODNiMGY3OWE1MWY2YmVk
+    ZWFmZGFhMGJhODNjNGVjZGE1NmQyZWRjMTg5NzBkZDY5NWMxNDJmNWE2NTU3
+    ODc4MmYxMWRmODM5NDdiNTRkNmVlMGQwZTEzNmE5MzM4YzlhMzY=

data/.gitignore

@@ -6,3 +6,8 @@ pkg/*
 .rvmrc
 .ruby-version
 .ruby-gemset
+bin/rake
+spec/integration/database.yml
+Gemfile
+gemfiles/vendor
+omg.ponies

data/.travis.yml

@@ -4,9 +4,11 @@ before_script:
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.1
 matrix:
   allow_failures:
-    - rvm: 2.0.0
+    - gemfile: gemfiles/ar-2.3_mysql.gemfile
+  fast_finish: true
 gemfile:
   - gemfiles/ar-2.3_mysql.gemfile
   - gemfiles/ar-3.2_mysql.gemfile

data/README.md

@@ -182,23 +182,31 @@ during the run will happen on the new table as well.
 ## Cleaning up after an interrupted Lhm run
 
 If an Lhm migration is interrupted, it may leave behind the temporary tables
-used in the migration. If the migration is re-started, the unexpected presence
-of these tables will cause an error. In this case, `Lhm.cleanup` can be used
-to drop any orphaned Lhm temporary tables.
+and/or triggers used in the migration. If the migration is re-started, the
+unexpected presence of these tables will cause an error.
 
-To see what Lhm tables are found:
+In this case, `Lhm.cleanup` can be used to drop any orphaned Lhm temporary tables or triggers.
+
+To see what Lhm tables/triggers are found:
 
 ```ruby
 Lhm.cleanup
 ```
 
-To remove any Lhm tables found:
+To remove any Lhm tables/triggers found:
 ```ruby
 Lhm.cleanup(true)
 ```
 
 ## Contributing
 
+First, get set up for local development:
+
+    git clone git://github.com/soundcloud/lhm.git
+    cd lhm
+
+To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md).
+
 We'll check out your contribution if you:
 
   * Provide a comprehensive suite of tests for your fork.
@@ -223,4 +231,4 @@ The license is included as LICENSE in this directory.
 [2]: https://github.com/freels/table_migrator
 [3]: http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html
 [4]: https://travis-ci.org/soundcloud/lhm
-[5]: https://travis-ci.org/soundcloud/lhm.png?branch=master
+[5]: https://travis-ci.org/soundcloud/lhm.svg?branch=master

data/Rakefile

@@ -4,13 +4,15 @@ require 'bundler'
 Bundler::GemHelper.install_tasks
 
 Rake::TestTask.new("unit") do |t|
-  t.libs.push "lib"
+  t.libs << 'lib'
+  t.libs << 'spec'
   t.test_files = FileList['spec/unit/*_spec.rb']
   t.verbose = true
 end
 
 Rake::TestTask.new("integration") do |t|
-  t.libs.push "lib"
+  t.libs << 'lib'
+  t.libs << 'spec'
   t.test_files = FileList['spec/integration/*_spec.rb']
   t.verbose = true
 end

data/bin/lhm-spec-clobber.sh

@@ -8,12 +8,13 @@ source ~/.lhm
 lhmkill() {
   echo killing lhm-cluster
   ps -ef | sed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
+  echo running homebrew mysql instance
+  launchctl load -w ~/Library/LaunchAgents/homebrew.mxcl.mysql.plist
   sleep 2
 }
 
-echo stopping other running mysql instance
-launchctl remove com.mysql.mysqld || { echo launchctl did not remove mysqld; }
-"$mysqldir"/bin/mysqladmin shutdown || { echo mysqladmin did not shut down anything; }
+echo stopping homebrew running mysql instance
+ls -lrt -d -1 ~/Library/LaunchAgents/* |  grep 'mysql.plist' | xargs launchctl unload -w
 
 lhmkill
 

data/bin/lhm-spec-setup-cluster.sh

@@ -13,7 +13,6 @@ source ~/.lhm
 # Main
 #
 
-install_bin="$(echo ./*/mysql_install_db)"
 
 mkdir -p "$basedir/master/data" "$basedir/slave/data"
 
@@ -61,7 +60,7 @@ CNF
 
 (
   cd "$mysqldir"
+  install_bin="$(echo ./*/mysql_install_db | tr " " "\\n" | head -1)"
   $install_bin --datadir="$basedir/master/data"
   $install_bin --datadir="$basedir/slave/data"
-
 )

data/lhm.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.executables   = ["lhm-kill-queue"]
 
-  s.add_development_dependency "minitest", "= 2.10.0"
+  s.add_development_dependency "minitest", "~> 5.0.8"
   s.add_development_dependency "rake"
 end
-

data/lib/lhm.rb

@@ -4,7 +4,9 @@
 require 'lhm/table'
 require 'lhm/invoker'
 require 'lhm/connection'
+require 'lhm/throttler'
 require 'lhm/version'
+require 'logger'
 
 # Large hadron migrator - online schema change tool
 #
@@ -17,6 +19,10 @@ require 'lhm/version'
 #   end
 #
 module Lhm
+  extend Throttler
+  extend self
+
+  DEFAULT_LOGGER_OPTIONS =  { level: Logger::INFO, file: STDOUT }
 
   # Alters a table with the changes described in the block
   #
@@ -37,7 +43,7 @@ module Lhm
   # @yield [Migrator] Yielded Migrator object records the changes
   # @return [Boolean] Returns true if the migration finishes
   # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
-  def self.change_table(table_name, options = {}, &block)
+  def change_table(table_name, options = {}, &block)
     origin = Table.parse(table_name, connection)
     invoker = Invoker.new(origin, connection)
     block.call(invoker.migrator)
@@ -45,28 +51,36 @@ module Lhm
     true
   end
 
-  def self.cleanup(run = false)
-    lhm_tables = connection.select_values("show tables").select do |name|
-      name =~ /^lhm(a|n)_/
-    end
-    return true if lhm_tables.empty?
+  def cleanup(run = false)
+    lhm_tables = connection.select_values("show tables").select { |name| name =~ /^lhm(a|n)_/ }
+    lhm_triggers = connection.select_values("show triggers").collect do |trigger|
+      trigger.respond_to?(:trigger) ? trigger.trigger : trigger
+    end.select { |name| name =~ /^lhmt/ }
+
     if run
+      lhm_triggers.each do |trigger|
+        connection.execute("drop trigger if exists #{trigger}")
+      end
       lhm_tables.each do |table|
-        connection.execute("drop table #{table}")
+        connection.execute("drop table if exists #{table}")
       end
       true
+    elsif lhm_tables.empty? && lhm_triggers.empty?
+      puts "Everything is clean. Nothing to do."
+      true
     else
       puts "Existing LHM backup tables: #{lhm_tables.join(", ")}."
+      puts "Existing LHM triggers: #{lhm_triggers.join(", ")}."
       puts "Run Lhm.cleanup(true) to drop them all."
       false
     end
   end
 
-  def self.setup(adapter)
+  def setup(adapter)
     @@adapter = adapter
   end
 
-  def self.adapter
+  def adapter
     @@adapter ||=
       begin
         raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
@@ -74,9 +88,24 @@ module Lhm
       end
   end
 
+  def self.logger=(new_logger)
+    @@logger = new_logger
+  end
+
+  def self.logger
+    @@logger ||=
+      begin
+        logger = Logger.new(DEFAULT_LOGGER_OPTIONS[:file])
+        logger.level = DEFAULT_LOGGER_OPTIONS[:level]
+        logger.formatter = nil
+        logger
+      end
+  end
+
   protected
 
-  def self.connection
+  def connection
     Connection.new(adapter)
   end
+
 end

data/lib/lhm/chunker.rb

@@ -1,8 +1,8 @@
 # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
 # Schmidt
-
 require 'lhm/command'
 require 'lhm/sql_helper'
+require 'lhm/printer'
 
 module Lhm
   class Chunker
@@ -16,55 +16,72 @@ module Lhm
     def initialize(migration, connection = nil, options = {})
       @migration = migration
       @connection = connection
-      @stride = options[:stride] || 40_000
-      @throttle = options[:throttle] || 100
+      @throttler = options[:throttler]
       @start = options[:start] || select_start
       @limit = options[:limit] || select_limit
+      @printer = options[:printer] || Printer::Percentage.new
     end
 
-    # Copies chunks of size `stride`, starting from `start` up to id `limit`.
-    def up_to(&block)
-      1.upto(traversable_chunks_size) do |n|
-        yield(bottom(n), top(n))
+    def execute
+      return unless @start && @limit
+      @next_to_insert = @start
+      until @next_to_insert >= @limit
+        stride = @throttler.stride
+        affected_rows = @connection.update(copy(bottom, top(stride)))
+
+        if @throttler && affected_rows > 0
+          @throttler.run
+        end
+
+        @printer.notify(bottom, @limit)
+        @next_to_insert = top(stride) + 1
       end
+      @printer.end
     end
 
-    def traversable_chunks_size
-      @limit && @start ? ((@limit - @start + 1) / @stride.to_f).ceil : 0
-    end
+  private
 
-    def bottom(chunk)
-      (chunk - 1) * @stride + @start
+    def bottom
+      @next_to_insert
     end
 
-    def top(chunk)
-      [chunk * @stride + @start - 1, @limit].min
+    def top(stride)
+      [(@next_to_insert + stride - 1), @limit].min
     end
 
     def copy(lowest, highest)
       "insert ignore into `#{ destination_name }` (#{ columns }) " +
       "select #{ select_columns } from `#{ origin_name }` " +
-      "#{ conditions } #{ origin_name }.`id` between #{ lowest } and #{ highest }"
+      "#{ conditions } `#{ origin_name }`.`id` between #{ lowest } and #{ highest }"
     end
 
     def select_start
-      start = connection.select_value("select min(id) from #{ origin_name }")
+      start = connection.select_value("select min(id) from `#{ origin_name }`")
       start ? start.to_i : nil
     end
 
     def select_limit
-      limit = connection.select_value("select max(id) from #{ origin_name }")
+      limit = connection.select_value("select max(id) from `#{ origin_name }`")
       limit ? limit.to_i : nil
     end
 
-    def throttle_seconds
-      @throttle / 1000.0
-    end
-
-  private
-
+    #XXX this is extremely brittle and doesn't work when filter contains more
+    #than one SQL clause, e.g. "where ... group by foo". Before making any
+    #more changes here, please consider either:
+    #
+    #1. Letting users only specify part of defined clauses (i.e. don't allow
+    #`filter` on Migrator to accept both WHERE and INNER JOIN
+    #2. Changing query building so that it uses structured data rather than
+    #strings until the last possible moment.
     def conditions
-      @migration.conditions ? "#{@migration.conditions} and" : "where"
+      if @migration.conditions
+        @migration.conditions.
+          sub(/\)\Z/, "").
+          #put any where conditions in parens
+          sub(/where\s(\w.*)\Z/, "where (\\1)") + " and"
+      else
+        "where"
+      end
     end
 
     def destination_name
@@ -80,7 +97,7 @@ module Lhm
     end
 
     def select_columns
-      @select_columns ||= @migration.intersection.typed(origin_name)
+      @select_columns ||= @migration.intersection.typed("`#{origin_name}`")
     end
 
     def validate
@@ -88,18 +105,5 @@ module Lhm
         error("impossible chunk options (limit must be greater than start)")
       end
     end
-
-    def execute
-      up_to do |lowest, highest|
-        affected_rows = @connection.update(copy(lowest, highest))
-
-        if affected_rows > 0
-          sleep(throttle_seconds)
-        end
-
-        print "."
-      end
-      print "\n"
-    end
   end
 end

data/lib/lhm/command.rb

@@ -7,6 +7,7 @@ module Lhm
 
   module Command
     def run(&block)
+      Lhm.logger.info "Starting run of class=#{self.class}"
       validate
 
       if(block_given?)
@@ -16,7 +17,8 @@ module Lhm
       else
         execute
       end
-    rescue
+    rescue => e
+      Lhm.logger.error "Error in class=#{self.class}, reverting. exception=#{e.class} message=#{e.message}"
       revert
       raise
     end

data/lib/lhm/invoker.rb

@@ -24,6 +24,24 @@ module Lhm
     end
 
     def run(options = {})
+      normalize_options(options)
+      migration = @migrator.run
+
+      Entangler.new(migration, @connection).run do
+        Chunker.new(migration, @connection, options).run
+        if options[:atomic_switch]
+          AtomicSwitcher.new(migration, @connection).run
+        else
+          LockedSwitcher.new(migration, @connection).run
+        end
+      end
+    end
+
+    private
+
+    def normalize_options(options)
+      Lhm.logger.info "Starting LHM run on table=#{@migrator.name}"
+
       if !options.include?(:atomic_switch)
         if supports_atomic_switch?
           options[:atomic_switch] = true
@@ -34,16 +52,19 @@ module Lhm
         end
       end
 
-      migration = @migrator.run
-
-      Entangler.new(migration, @connection).run do
-        Chunker.new(migration, @connection, options).run
-        if options[:atomic_switch]
-          AtomicSwitcher.new(migration, @connection).run
-        else
-          LockedSwitcher.new(migration, @connection).run
-        end
+      if options[:throttler]
+        options[:throttler] = Throttler::Factory.create_throttler(*options[:throttler])
+      elsif options[:throttle] || options[:stride]
+        # we still support the throttle and stride as a Fixnum input
+        warn "throttle option will no longer accept a Fixnum in the next versions."
+        options[:throttler] = Throttler::LegacyTime.new(options[:throttle], options[:stride])
+      else
+        options[:throttler] = Lhm.throttler
       end
+
+    rescue => e
+      Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
+      raise
     end
   end
 end