lhm 1.0.0.rc.1 → 1.0.0.rc2

data/.travis.yml

@@ -3,7 +3,7 @@ rvm:
   - 1.9.2
   - 1.9.3
 before_script:
-  - "mysql -e 'create database large_hadron_migrator;'"
+  - "mysql -e 'create database lhm;'"
 branches:
   only:
     - master

data/CHANGELOG.md

@@ -1,8 +1,18 @@
-# 1.0.0-RC1
+# 1.0.0.rc2 (January 18, 2012)
 
-* rewrite.
+* Speedup migrations for tables with large ids
+* Fix conversion of milliseconds to seconds
+* Fix handling of sql errors
+* Add helper to create unique index
+* Allow index creation on prefix of column
+* Quote column names on index creation
+* Remove ambiguous method signature
+* Documentation fix
+* 1.8.7 compatibility
+
+# 1.0.0.rc1 (January 15, 2012)
 
-# 0.9.1
+* rewrite.
 
 # 0.2.1 (November 26, 2011)
 

data/README.md

@@ -42,11 +42,11 @@ twitter solution [1], it does not require the presence of an indexed
 
 ## Usage
 
-After including Lhm, `hadron_change_table` becomes available
+After extending Lhm, `hadron_change_table` becomes available
 with the following methods:
 
     class MigrateArbitrary < ActiveRecord::Migration
-      include Lhm
+      extend Lhm
 
       def self.up
         hadron_change_table(:users) do |t|

data/Rakefile

@@ -1,4 +1,7 @@
 require 'rake/testtask'
+require 'bundler'
+
+Bundler::GemHelper.install_tasks
 
 Rake::TestTask.new("unit") do |t|
   t.libs.push "lib"

data/lhm.gemspec

@@ -3,8 +3,6 @@
 lib = File.expand_path('../lib', __FILE__)
 $:.unshift(lib) unless $:.include?(lib)
 
-puts $:.inspect
-
 require 'lhm'
 
 Gem::Specification.new do |s|
@@ -23,7 +21,9 @@ Gem::Specification.new do |s|
 
   # this should be a real dependency, but we're using a different gem in our code
   s.add_development_dependency "mysql", "~> 2.8.1"
-  s.add_development_dependency "rspec", "=1.3.1"
+  s.add_development_dependency "minitest", "= 2.10.0"
   s.add_development_dependency "rake"
+
+  s.add_dependency "activerecord"
 end
 

data/lib/lhm.rb

@@ -8,7 +8,7 @@ require 'lhm/invoker'
 require 'lhm/migration'
 
 module Lhm
-  VERSION = "1.0.0.rc.1"
+  VERSION = "1.0.0.rc2"
 
   def hadron_change_table(table_name, chunk_options = {}, &block)
     origin = Table.parse(table_name, connection)

data/lib/lhm/chunker.rb

@@ -29,7 +29,7 @@ module Lhm
 
     def up_to(limit)
       traversable_chunks_up_to(limit).times do |n|
-        yield(bottom(n + 1), top(n + 1, limit)) && sleep(throttle_seconds)
+        yield(bottom(n + 1), top(n + 1, limit))
       end
     end
 
@@ -59,14 +59,18 @@ module Lhm
 
     def execute
       up_to(@limit) do |lowest, highest|
-        print "."
+        affected_rows = update(copy(lowest, highest))
+
+        if affected_rows > 0
+          sleep(throttle_seconds)
+        end
 
-        sql copy(lowest, highest)
+        print "."
       end
     end
 
     def throttle_seconds
-      @throttle / 100.0
+      @throttle / 1000.0
     end
   end
 end

data/lib/lhm/command.rb

@@ -17,9 +17,7 @@ module Lhm
 
     def validate; end
 
-    def revert
-      raise NotImplementedError.new(self.class.name)
-    end
+    def revert; end
 
     def run(&block)
       validate
@@ -56,14 +54,19 @@ module Lhm
     end
 
     def sql(statements)
-      [*statements].each do |statement|
-        begin
-          @connection.execute(statement)
-        rescue Mysql::Error => e
-          revert
-          error "#{ statement } failed: #{ e.inspect }"
-        end
+      [statements].flatten.each { |statement| @connection.execute(statement) }
+    rescue ActiveRecord::StatementInvalid, Mysql::Error => e
+      revert
+      error e.message
+    end
+
+    def update(statements)
+      [statements].flatten.inject(0) do |memo, statement|
+        memo += @connection.update(statement)
       end
+    rescue ActiveRecord::StatementInvalid, Mysql::Error => e
+      revert
+      error e.message
     end
   end
 end

data/lib/lhm/intersection.rb

@@ -13,7 +13,7 @@ module Lhm
     end
 
     def common
-      @origin.columns.keys & @destination.columns.keys
+      (@origin.columns.keys & @destination.columns.keys).sort
     end
 
     def escaped

data/lib/lhm/locked_switcher.rb

@@ -49,9 +49,10 @@ module Lhm
       [
         "set @lhm_auto_commit = @@session.autocommit",
         "set session autocommit = 0",
-        *yield,
+        yield,
         "set session autocommit = @lhm_auto_commit"
-      ]
+
+      ].flatten
     end
 
     #

data/lib/lhm/migrator.rb

@@ -61,7 +61,20 @@ module Lhm
     #
 
     def add_index(cols)
-      ddl = "create index `%s` on %s" % [@origin.idx_name(cols), idx_spec(cols)]
+      ddl = "create index `%s` on %s" % idx_parts(cols)
+      statements << ddl.strip
+    end
+
+    #
+    # Add a unique index to a table:
+    #
+    #  hadron_change_table("users") do |t|
+    #    t.add_unique_index([:comment, :created_at])
+    #  end
+    #
+
+    def add_unique_index(cols)
+      ddl = "create unique index `%s` on %s" % idx_parts(cols)
       statements << ddl.strip
     end
 
@@ -73,7 +86,7 @@ module Lhm
     #   end
     #
 
-    def remove_index(*cols)
+    def remove_index(cols)
       ddl = "drop index `%s` on `%s`" % [@origin.idx_name(cols), @name]
       statements << ddl.strip
     end
@@ -118,7 +131,11 @@ module Lhm
     end
 
     def idx_spec(cols)
-      "#{ @name }(#{ [*cols].map(&:to_s).join(', ') })"
+      "`#{ @name }` (#{ Array(cols).map(&:to_s).join(', ') })"
+    end
+
+    def idx_parts(cols)
+      [@origin.idx_name(cols), idx_spec(cols)]
     end
   end
 end

data/lib/lhm/table.rb

@@ -24,7 +24,8 @@ module Lhm
     end
 
     def idx_name(cols)
-      "index_#{ @name }_on_" + [*cols].join("_and_")
+      column_part = Array(cols).map { |c| c.to_s.sub(/\(.*/, "") }.join("_and_")
+      "index_#{ @name }_on_#{ column_part }"
     end
 
     def self.parse(table_name, connection)

data/spec/integration/integration_helper.rb

@@ -9,7 +9,6 @@ require 'active_record'
 require 'lhm/table'
 
 module IntegrationHelper
-  delegate :select_one, :select_value, :execute, :to => :connection
 
   #
   # Connectivity
@@ -30,6 +29,18 @@ module IntegrationHelper
     ActiveRecord::Base.connection
   end
 
+  def select_one(*args)
+    connection.select_one(*args)
+  end
+
+  def select_value(*args)
+    connection.select_value(*args)
+  end
+
+  def execute(*args)
+    connection.execute(*args)
+  end
+
   #
   # Test Data
   #
@@ -66,9 +77,9 @@ module IntegrationHelper
     select_value(query).to_i
   end
 
-  def key?(table, cols)
-    query = "show indexes in #{ table.name } where key_name = '#{ table.idx_name(cols) }'"
+  def key?(table, cols, type = :non_unique)
+    non_unique = type == :non_unique ? 1 : 0
+    query = "show indexes in #{ table.name } where key_name = '#{ table.idx_name(cols) }' and non_unique = #{ non_unique }"
     !!select_value(query)
   end
 end
-

data/spec/integration/lhm_spec.rb

@@ -55,9 +55,17 @@ describe Lhm do
       key?(table_read("users"), ["comment", "created_at"]).must_equal(true)
     end
 
+    it "should add a unqiue index" do
+      hadron_change_table("users") do |t|
+        t.add_unique_index(:comment)
+      end
+
+      key?(table_read(:users), :comment, :unique).must_equal(true)
+    end
+
     it "should remove an index" do
       hadron_change_table("users") do |t|
-        t.remove_index(:username, :created_at)
+        t.remove_index([:username, :created_at])
       end
 
       key?(table_read("users"), ["username", "created_at"]).must_equal(false)

data/spec/unit/migrator_spec.rb

@@ -21,7 +21,23 @@ describe Lhm::Migrator do
       @creator.add_index(["a", "b"])
 
       @creator.statements.must_equal([
-        "create index `index_alt_on_a_and_b` on lhmn_alt(a, b)"
+        "create index `index_alt_on_a_and_b` on `lhmn_alt` (a, b)"
+      ])
+    end
+
+    it "should add an index with prefixed columns" do
+      @creator.add_index(["a(10)", "b"])
+
+      @creator.statements.must_equal([
+        "create index `index_alt_on_a_and_b` on `lhmn_alt` (a(10), b)"
+      ])
+    end
+
+    it "should add an unique index" do
+      @creator.add_unique_index(["a(10)", :b])
+
+      @creator.statements.must_equal([
+        "create unique index `index_alt_on_a_and_b` on `lhmn_alt` (a(10), b)"
       ])
     end
 

data/spec/unit/table_spec.rb

@@ -25,11 +25,23 @@ describe Lhm::Table do
         must_equal("index_users_on_name")
     end
 
-    it "should name index with a multiple columns" do
+    it "should name index with multiple columns" do
       @table.
         idx_name(["name", "firstname"]).
         must_equal("index_users_on_name_and_firstname")
     end
+
+    it "should name index with prefixed column" do
+      @table.
+        idx_name(["name(10)", "firstname"]).
+        must_equal("index_users_on_name_and_firstname")
+    end
+
+    it "should name index with column names given as symbol" do
+      @table.
+        idx_name([:name, :firstname]).
+        must_equal("index_users_on_name_and_firstname")
+    end
   end
 
   describe "constraints" do

metadata

@@ -1,14 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: lhm
 version: !ruby/object:Gem::Version 
-  prerelease: true
-  segments: 
-  - 1
-  - 0
-  - 0
-  - rc
-  - 1
-  version: 1.0.0.rc.1
+  prerelease: 6
+  version: 1.0.0.rc2
 platform: ruby
 authors: 
 - SoundCloud
@@ -19,7 +13,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-16 00:00:00 +01:00
+date: 2012-01-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -30,26 +24,18 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 8
-        - 1
         version: 2.8.1
   type: :development
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: minitest
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 3
-        - 1
-        version: 1.3.1
+        version: 2.10.0
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -60,11 +46,20 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.
 email: rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com
 executables: []
@@ -82,7 +77,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- TODO
 - lhm.gemspec
 - lib/lhm.rb
 - lib/lhm/chunker.rb
@@ -125,23 +119,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">"
     - !ruby/object:Gem::Version 
-      segments: 
-      - 1
-      - 3
-      - 1
       version: 1.3.1
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.5.0
 signing_key: 
 specification_version: 3
 summary: online schema changer for mysql

data/TODO

@@ -1,11 +0,0 @@
-todo
-
-. locked_switcher
-. schema_creator
-. chunker
-
-checklist
-
-. consider entanglement epoch
-  - should not lose changes due to low epoch
-