lhm 1.2.0 → 2.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dea357ff2aae21b2cee84cc6d546aa2328fd6164
+  data.tar.gz: b6b3e4b06ed6f7ea1b14ac2be2444cc8b6357966
+SHA512:
+  metadata.gz: a4a2e08f68b46ba6141bd37093f3c5341488c60d4a1bf6069b1339481f11ff1117ebe1198a383e09f1ae635d68966e98256d31a981ff89171eb582fd7b075170
+  data.tar.gz: 49174830b37493a1652394a4ee717f52939675d39289bc183f38969ae840cc5c9f00b5668d585888751916e395f471a3f24501192c8bf52cd3321d1c9d50dc32

data/.gitignore

@@ -4,3 +4,10 @@ Gemfile.lock
 gemfiles/*.lock
 pkg/*
 .rvmrc
+.ruby-version
+.ruby-gemset
+bin/rake
+spec/integration/database.yml
+Gemfile
+gemfiles/vendor
+omg.ponies

data/.rubocop.yml

@@ -0,0 +1,256 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2015-01-12 14:49:41 +0100 using RuboCop version 0.26.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 2
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+
+# Offense count: 2
+# Configuration parameters: AllowSafeAssignment.
+Lint/AssignmentInCondition:
+  Enabled: false
+
+# Offense count: 1
+Lint/ConditionPosition:
+  Enabled: false
+
+# Offense count: 1
+Lint/HandleExceptions:
+  Enabled: false
+
+# Offense count: 6
+# Cop supports --auto-correct.
+Lint/UnusedBlockArgument:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+# Offense count: 4
+Lint/UselessAssignment:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 131
+
+# Offense count: 3
+Metrics/CyclomaticComplexity:
+  Max: 19
+
+# Offense count: 77
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 197
+
+# Offense count: 11
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 27
+
+# Offense count: 3
+Metrics/PerceivedComplexity:
+  Max: 15
+
+# Offense count: 10
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/AccessModifierIndentation:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/AlignParameters:
+  Enabled: false
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/AndOr:
+  Enabled: false
+
+# Offense count: 13
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BarePercentLiterals:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/Blocks:
+  Enabled: false
+
+# Offense count: 11
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BracesAroundHashParameters:
+  Enabled: false
+
+# Offense count: 5
+Style/ClassVars:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredMethods.
+Style/CollectionMethods:
+  Enabled: false
+
+# Offense count: 24
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 28
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/DotPosition:
+  Enabled: false
+
+# Offense count: 4
+Style/DoubleNegation:
+  Enabled: false
+
+# Offense count: 1
+Style/EachWithObject:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: Exclude.
+Style/FileName:
+  Enabled: false
+
+# Offense count: 11
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/FormatString:
+  Enabled: false
+
+# Offense count: 10
+# Configuration parameters: AllowedVariables.
+Style/GlobalVars:
+  Enabled: false
+
+# Offense count: 9
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Enabled: false
+
+# Offense count: 88
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/HashSyntax:
+  Enabled: false
+
+# Offense count: 3
+# Configuration parameters: MaxLineLength.
+Style/IfUnlessModifier:
+  Enabled: false
+
+# Offense count: 14
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/IndentHash:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/IndentationConsistency:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/IndentationWidth:
+  Enabled: false
+
+# Offense count: 2
+Style/ModuleFunction:
+  Enabled: false
+
+# Offense count: 2
+Style/MultilineBlockChain:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/NegatedIf:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Style/NumericLiterals:
+  MinDigits: 8
+
+# Offense count: 14
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredDelimiters.
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+# Offense count: 3
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/RaiseArgs:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: AllowMultipleReturnValues.
+Style/RedundantReturn:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/RedundantSelf:
+  Enabled: false
+
+# Offense count: 1
+Style/RescueModifier:
+  Enabled: false
+
+# Offense count: 1
+Style/SelfAssignment:
+  Enabled: false
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SignalException:
+  Enabled: false
+
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles.
+Style/SpaceInsideHashLiteralBraces:
+  SupportedStyles: space
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/SpecialGlobalVars:
+  Enabled: false
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: ExactNameMatch, AllowPredicates, AllowDSLWriters, Whitelist.
+Style/TrivialAccessors:
+  Enabled: false
+
+# Offense count: 15
+# Cop supports --auto-correct.
+Style/UnneededPercentQ:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+Style/WordArray:
+  MinSize: 2
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: AllowSafeAssignment.
+Style/ParenthesesAroundCondition:
+  Enabled: false

data/.travis.yml

@@ -2,8 +2,12 @@ language: ruby
 before_script:
   - "mysql -e 'create database lhm;'"
 rvm:
-  - 1.8.7
   - 1.9.3
+  - 2.0.0
+  - 2.1
+sudo: false
+matrix:
+  fast_finish: true
 gemfile:
   - gemfiles/ar-2.3_mysql.gemfile
   - gemfiles/ar-3.2_mysql.gemfile

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+# 2.2.0 (not released)
+
+* #84 - Require index names to be strings or symbols (Thibaut)
+* #39 - Adding the ability to rename columns (erikogan)
+* #67 - Allow for optional time filter on .cleanup (joelr)
+
+# 2.1.0
+
+* #48 - Add percentage output for migrations (@arthurnn)
+* #60 - Quote table names (@spickermann)
+* #59 - Escape table name in select_limit and select_start methods (@stevehodgkiss)
+* #57 - Ensure chunking 'where' clause handled separately (@rentalcustard)
+* #54 - Chunker handle stride changes (@rentalcustard)
+* #52 - Implement ability to control timeout and stride from Throttler (@edmundsalvacion)
+* #51 - Ensure Lhm.cleanup removes temporary triggers (@edmundsalvacion)
+* #46 - Allow custom throttler (@arthurnn)
+
+# 2.0.0
+
+* #44 - Conditional migrations (@durran)
+
+# 1.3.0 (May 28, 2013)
+
+* Add Lhm.cleanup method for removing copy tables, thanks @bogdan
+* Limit copy table names to 64 characters, thanks @charliesome
+
 # 1.2.0 (February 22, 2013)
 
 * Added DataMapper support, no API changes for current users. Refer to the

data/README.md

@@ -1,4 +1,4 @@
-# Large Hadron Migrator [![Build Status](https://secure.travis-ci.org/soundcloud/large-hadron-migrator.png?branch=master)][4]
+# Large Hadron Migrator [![Build Status][5]][4]
 
 Rails style database migrations are a useful way to evolve your data schema in
 an agile manner. Most Rails projects start like this, and at first, making
@@ -45,13 +45,19 @@ the [twitter solution][2], it does not require the presence of an indexed
 Lhm currently only works with MySQL databases and requires an established
 ActiveRecord or DataMapper connection.
 
-It is compatible and [continuously tested][4] with Ruby 1.8.7 and Ruby 1.9.x,
+It is compatible and [continuously tested][4] with MRI 1.9.x, 2.0.x, 2.1.x,
 ActiveRecord 2.3.x and 3.x (mysql and mysql2 adapters), as well as DataMapper
 1.2 (dm-mysql-adapter).
 
 Lhm also works with dm-master-slave-adapter, it'll bind to the master before
 running the migrations.
 
+
+## Limitations
+
+Lhm requires a monotonically increasing numeric Primary Key on the table, due to how
+the Chunker works.
+
 ## Installation
 
 Install it via `gem install lhm` or add `gem "lhm"` to your Gemfile.
@@ -133,18 +139,35 @@ end
 **Note:** Lhm won't delete the old, leftover table. This is on purpose, in order
 to prevent accidental data loss.
 
+## Throttler
+
+Lhm is using a throttle mecanism to read data in your original table.
+
+By default, 40000 rows are read each 0.1 second.
+
+If you want to change that behiavour, you can pass an instance of a throttler with the `throttler` option.
+
+In this example, 1000 rows will be read with a 10 seconds delay between each processing:
+```ruby
+my_throttler = Lhm::Throttler::Time.new(stride: 1000, delay: 10)
+
+Lhm.change_table :users, throttler: my_throttler  do |m|
+  #
+end
+```
+
 ## Table rename strategies
 
 There are two different table rename strategies available: LockedSwitcher and
 AtomicSwitcher.
 
-The LockedSwitcher strategy locks the table being migrated and issues two ALTER TABLE statements. 
+The LockedSwitcher strategy locks the table being migrated and issues two ALTER TABLE statements.
 The AtomicSwitcher uses a single atomic RENAME TABLE query and is the favored solution.
 
-Lhm chooses AtomicSwitcher if no strategy is specified, **unless** your version of MySQL is 
-affected by [binlog bug #39675](http://bugs.mysql.com/bug.php?id=39675). If your version is 
-affected, Lhm will raise an error if you don't specify a strategy. You're recommended 
-to use the LockedSwitcher in these cases to avoid replication issues. 
+Lhm chooses AtomicSwitcher if no strategy is specified, **unless** your version of MySQL is
+affected by [binlog bug #39675](http://bugs.mysql.com/bug.php?id=39675). If your version is
+affected, Lhm will raise an error if you don't specify a strategy. You're recommended
+to use the LockedSwitcher in these cases to avoid replication issues.
 
 To specify the strategy in your migration:
 
@@ -154,8 +177,63 @@ Lhm.change_table :users, :atomic_switch => true do |m|
 end
 ```
 
+## Limiting the data that is migrated
+
+For instances where you want to limit the data that is migrated to the new
+table by some conditions, you may tell the migration to filter by a set of
+conditions:
+
+```ruby
+Lhm.change_table(:sounds) do |m|
+  m.filter("inner join users on users.`id` = sounds.`user_id` and sounds.`public` = 1")
+end
+```
+
+Note that this SQL will be inserted into the copy directly after the "from"
+statement - so be sure to use inner/outer join syntax and not cross joins. These
+conditions will not affect the triggers, so any modifications to the table
+during the run will happen on the new table as well.
+
+## Cleaning up after an interrupted Lhm run
+
+If an Lhm migration is interrupted, it may leave behind the temporary tables
+and/or triggers used in the migration. If the migration is re-started, the
+unexpected presence of these tables will cause an error.
+
+In this case, `Lhm.cleanup` can be used to drop any orphaned Lhm temporary tables or triggers.
+
+To see what Lhm tables/triggers are found:
+
+```ruby
+Lhm.cleanup
+```
+
+To remove any Lhm tables/triggers found:
+```ruby
+Lhm.cleanup(true)
+```
+
+Optionally only remove tables up to a specific Time, if you want to retain previous migrations.
+
+Rails:
+```ruby
+Lhm.cleanup(true, until: 1.day.ago)
+```
+
+Ruby:
+```ruby
+Lhm.cleanup(true, until: Time.now - 86400)
+```
+
 ## Contributing
 
+First, get set up for local development:
+
+    git clone git://github.com/soundcloud/lhm.git
+    cd lhm
+
+To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md).
+
 We'll check out your contribution if you:
 
   * Provide a comprehensive suite of tests for your fork.
@@ -179,4 +257,5 @@ The license is included as LICENSE in this directory.
 [1]: http://www.facebook.com/note.php?note\_id=430801045932
 [2]: https://github.com/freels/table_migrator
 [3]: http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html
-[4]: http://travis-ci.org/soundcloud/large-hadron-migrator
+[4]: https://travis-ci.org/soundcloud/lhm
+[5]: https://travis-ci.org/soundcloud/lhm.svg?branch=master

data/Rakefile

@@ -3,14 +3,16 @@ require 'bundler'
 
 Bundler::GemHelper.install_tasks
 
-Rake::TestTask.new("unit") do |t|
-  t.libs.push "lib"
+Rake::TestTask.new('unit') do |t|
+  t.libs << 'lib'
+  t.libs << 'spec'
   t.test_files = FileList['spec/unit/*_spec.rb']
   t.verbose = true
 end
 
-Rake::TestTask.new("integration") do |t|
-  t.libs.push "lib"
+Rake::TestTask.new('integration') do |t|
+  t.libs << 'lib'
+  t.libs << 'spec'
   t.test_files = FileList['spec/integration/*_spec.rb']
   t.verbose = true
 end

data/bin/lhm-config.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ -z ${LHM_TEST_CONFIG+x} ] ; then
+  source $HOME/.lhm
+else
+  source $LHM_TEST_CONFIG
+fi

data/bin/lhm-kill-queue

@@ -6,7 +6,6 @@ require 'optparse'
 
 module Lhm
   class KillQueue
-
     def initialize
       @port = 3306
       @grace = 10
@@ -14,20 +13,20 @@ module Lhm
       @marker = "%#{ SqlHelper.annotation }%"
 
       OptionParser.new do |opts|
-        opts.on("-h", "--hostname HOSTNAME") { |v| @hostname = v }
-        opts.on("-u", "--username USERNAME") { |v| @username = v }
-        opts.on("-p", "--password PASSWORD") { |v| @password = v }
-        opts.on("-d", "--database DATABASE") { |v| @database = v }
-        opts.on("-m", "--mode MODE") { |v| @mode = v.to_sym }
-        opts.on("-y", "--confirm") { |v| @confirm = true }
+        opts.on('-h', '--hostname HOSTNAME') { |v| @hostname = v }
+        opts.on('-u', '--username USERNAME') { |v| @username = v }
+        opts.on('-p', '--password PASSWORD') { |v| @password = v }
+        opts.on('-d', '--database DATABASE') { |v| @database = v }
+        opts.on('-m', '--mode MODE') { |v| @mode = v.to_sym }
+        opts.on('-y', '--confirm') { |v| @confirm = true }
       end.parse!
 
-      unless(@hostname && @username && @password && @database)
+      unless (@hostname && @username && @password && @database)
         abort usage
       end
 
-      unless([:kill, :master, :slave].include?(@mode))
-        abort "specify -m kill OR -m master OR -m slave"
+      unless ([:kill, :master, :slave].include?(@mode))
+        abort 'specify -m kill OR -m master OR -m slave'
       end
 
       connect
@@ -73,7 +72,7 @@ module Lhm
       sleep(@grace + @tiny)
 
       [list_non_lhm].flatten.each do |process|
-        if(select?(process))
+        if (select?(process))
           kill_process(process)
           sleep(@tiny)
         end
@@ -115,11 +114,11 @@ module Lhm
     def kill_process(process_id)
       puts "killing #{ select_statement(process_id) }"
 
-      if(@confirm)
+      if (@confirm)
         print "confirm ('y' to confirm): "
 
-        if(gets.strip != 'y')
-          puts "skipped."
+        if (gets.strip != 'y')
+          puts 'skipped.'
           return
         end
       end
@@ -169,4 +168,3 @@ end
 
 killer = Lhm::KillQueue.new
 killer.run
-

data/bin/lhm-spec-clobber.sh

@@ -3,17 +3,18 @@
 set -e
 set -u
 
-source ~/.lhm
+source `dirname $0`/lhm-config.sh
 
 lhmkill() {
   echo killing lhm-cluster
   ps -ef | sed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
+  echo running homebrew mysql instance
+  launchctl load -w ~/Library/LaunchAgents/homebrew.mxcl.mysql.plist
   sleep 2
 }
 
-echo stopping other running mysql instance
-launchctl remove com.mysql.mysqld || { echo launchctl did not remove mysqld; }
-"$mysqldir"/bin/mysqladmin shutdown || { echo mysqladmin did not shut down anything; }
+echo stopping homebrew running mysql instance
+ls -lrt -d -1 ~/Library/LaunchAgents/* |  grep 'mysql.plist' | xargs launchctl unload -w
 
 lhmkill
 

data/bin/lhm-spec-grants.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source ~/.lhm
+source `dirname $0`/lhm-config.sh
 
 master() { "$mysqldir"/bin/mysql --protocol=TCP -P $master_port -uroot; }
 slave()  { "$mysqldir"/bin/mysql --protocol=TCP -P $slave_port -uroot; }
@@ -12,7 +12,7 @@ echo "grant replication slave on *.* to 'slave'@'localhost'" | master
 
 # set up slave
 
-echo "change master to master_user = 'slave', master_password = 'slave', master_port = 3306, master_host = 'localhost'" | slave
+echo "change master to master_user = 'slave', master_password = 'slave', master_port = $master_port, master_host = 'localhost'" | slave
 echo "start slave" | slave
 echo "show slave status \G" | slave
 

data/bin/lhm-spec-setup-cluster.sh

@@ -7,13 +7,12 @@
 set -e
 set -u
 
-source ~/.lhm
+source `dirname $0`/lhm-config.sh
 
 #
 # Main
 #
 
-install_bin="$(echo ./*/mysql_install_db)"
 
 mkdir -p "$basedir/master/data" "$basedir/slave/data"
 
@@ -61,7 +60,7 @@ CNF
 
 (
   cd "$mysqldir"
+  install_bin="$(echo ./*/mysql_install_db | tr " " "\\n" | head -1)"
   $install_bin --datadir="$basedir/master/data"
   $install_bin --datadir="$basedir/slave/data"
-
 )

data/gemfiles/ar-2.3_mysql.gemfile

@@ -1,5 +1,6 @@
-source :rubygems
+source "https://rubygems.org"
 
 gem "mysql", "~> 2.8.1"
 gem "activerecord", "~> 2.3.14"
+gem "iconv", "~> 1.0.4"
 gemspec :path=>"../"

data/gemfiles/ar-3.2_mysql.gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source "https://rubygems.org"
 
 gem "mysql", "~> 2.8.1"
 gem "activerecord", "~> 3.2.2"

data/gemfiles/ar-3.2_mysql2.gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.11"
 gem "activerecord", "~> 3.2.2"

data/gemfiles/dm_mysql.gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source "https://rubygems.org"
 
 gem 'dm-core'
 gem 'dm-mysql-adapter'

data/lhm.gemspec

@@ -6,20 +6,19 @@ $:.unshift(lib) unless $:.include?(lib)
 require 'lhm/version'
 
 Gem::Specification.new do |s|
-  s.name          = "lhm"
+  s.name          = 'lhm'
   s.version       = Lhm::VERSION
   s.platform      = Gem::Platform::RUBY
-  s.authors       = ["SoundCloud", "Rany Keddo", "Tobias Bielohlawek", "Tobias Schmidt"]
+  s.authors       = ['SoundCloud', 'Rany Keddo', 'Tobias Bielohlawek', 'Tobias Schmidt']
   s.email         = %q{rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com}
   s.summary       = %q{online schema changer for mysql}
   s.description   = %q{Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.}
-  s.homepage      = %q{http://github.com/soundcloud/large-hadron-migrator}
+  s.homepage      = %q{http://github.com/soundcloud/lhm}
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.require_paths = ["lib"]
-  s.executables   = ["lhm-kill-queue"]
+  s.require_paths = ['lib']
+  s.executables   = ['lhm-kill-queue']
 
-  s.add_development_dependency "minitest", "= 2.10.0"
-  s.add_development_dependency "rake"
+  s.add_development_dependency 'minitest', '~> 5.0.8'
+  s.add_development_dependency 'rake'
 end
-