lhm 1.0.0.rc7 → 1.0.2

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+# 1.0.2 (Febuary 17, 2012)
+
+* closes https://github.com/soundcloud/large-hadron-migrator/issues/11
+  this critical bug could cause data loss. table parser was replaced with
+  an implementation that reads directly from information_schema.
+
+# 1.0.1 (Febuary 09, 2012)
+
+* released to rubygems
+
+# 1.0.0 (Febuary 09, 2012)
+
+* added change_column
+* final 1.0 release
+
+# 1.0.0.rc8 (Febuary 09, 2012)
+
+* removed spec binaries from gem bins
+
 # 1.0.0.rc7 (January 31, 2012)
 
 * added SqlHelper.annotation into the middle of trigger statements. this

data/README.md

@@ -1,5 +1,7 @@
 # Large Hadron Migrator [![Build Status](https://secure.travis-ci.org/soundcloud/large-hadron-migrator.png)](http://travis-ci.org/soundcloud/large-hadron-migrator)
 
+Update: There is currently [An issue](https://github.com/soundcloud/large-hadron-migrator/issues/11) with the migration. Fix coming up.
+
 Rails style database migrations are a useful way to evolve your data schema in
 an agile manner. Most Rails projects start like this, and at first, making
 changes is fast and easy.

data/bin/lhm-spec-clobber.sh

@@ -21,7 +21,7 @@ echo removing $basedir
 rm -rf "$basedir"
 
 echo setting up cluster
-lhm-spec-setup-cluster
+bin/lhm-spec-setup-cluster.sh
 
 echo staring instances
 "$mysqldir"/bin/mysqld --defaults-file="$basedir/master/my.cnf" 2>&1 >$basedir/master/lhm.log &
@@ -29,7 +29,7 @@ echo staring instances
 sleep 5
 
 echo running grants
-lhm-spec-grants
+bin/lhm-spec-grants.sh
 
 trap lhmkill SIGTERM SIGINT
 

data/lhm.gemspec

@@ -17,12 +17,7 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
-  s.executables   = [
-    "lhm-spec-clobber",
-    "lhm-spec-grants",
-    "lhm-spec-setup-cluster",
-    "lhm-kill-queue"
-  ]
+  s.executables   = ["lhm-kill-queue"]
 
   # this should be a real dependency, but we're using a different gem in our code
   s.add_development_dependency "mysql", "~> 2.8.1"

data/lib/lhm/migrator.rb

@@ -55,6 +55,21 @@ module Lhm
       ddl("alter table `%s` add column `%s` %s" % [@name, name, definition])
     end
 
+    # Change an existing column to a new definition
+    #
+    # @example
+    #
+    #   Lhm.change_table(:users) do |m|
+    #     m.change_column(:comment, "VARCHAR(12) DEFAULT '0' NOT NULL")
+    #   end
+    #
+    # @param [String] name Name of the column to change
+    # @param [String] definition Valid SQL column definition
+    def change_column(name, definition)
+      remove_column(name)
+      add_column(name, definition)
+    end
+
     # Remove a column from a table
     #
     # @example

data/lib/lhm/table.rb

@@ -1,6 +1,8 @@
 # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
 # Schmidt
 
+require 'lhm/sql_helper'
+
 module Lhm
   class Table
     attr_reader :name, :columns, :indices, :pk, :ddl
@@ -22,58 +24,71 @@ module Lhm
     end
 
     def self.parse(table_name, connection)
-      sql = "show create table `#{ table_name }`"
-      ddl = connection.execute(sql).fetch_row.last
-
-      Parser.new(ddl).parse
+      Parser.new(table_name, connection).parse
     end
 
     class Parser
-      def initialize(ddl)
-        @ddl = ddl
-      end
+      include SqlHelper
 
-      def lines
-        @ddl.lines.to_a.map(&:strip).reject(&:empty?)
+      def initialize(table_name, connection)
+        @table_name = table_name.to_s
+        @schema_name = connection.current_database
+        @connection = connection
       end
 
-      def create_definitions
-        lines[1..-2]
+      def ddl
+        sql = "show create table `#{ @table_name }`"
+        @connection.execute(sql).fetch_row.last
       end
 
       def parse
-        _, name = *lines.first.match("`([^ ]*)`")
-        pk_line = create_definitions.grep(primary).first
-
-        if pk_line
-          _, pk = *pk_line.match(primary)
-          table = Table.new(name, pk, @ddl)
-
-          create_definitions.each do |definition|
-            case definition
-              when index
-                table.indices[$1] = { :metadata => $2 }
-              when column
-                table.columns[$1] = { :type => $2, :metadata => $3 }
-            end
+        schema = read_information_schema
+
+        Table.new(@table_name, extract_primary_key(schema), ddl).tap do |table|
+          schema.each do |defn|
+            table.columns[defn["COLUMN_NAME"]] = {
+              :type => defn["COLUMN_TYPE"],
+              :is_nullable => defn["IS_NULLABLE"],
+              :column_default => defn["COLUMN_DEFAULT"]
+            }
           end
 
-          table
+          extract_indices(read_indices).each do |idx, columns|
+            table.indices[idx] = columns
+          end
         end
       end
 
     private
 
-      def primary
-        /^PRIMARY KEY (?:USING (?:HASH|[BR]TREE) )?\(`([^ ]*)`\),?$/
+      def read_information_schema
+        @connection.select_all %Q{
+          select *
+            from information_schema.columns
+           where table_name = "#{ @table_name }"
+             and table_schema = "#{ @schema_name }"
+        }
       end
 
-      def index
-        /^(?:UNIQUE )?(?:INDEX|KEY) `([^ ]*)` (.*?),?$/
+      def read_indices
+        @connection.select_all %Q{
+          show indexes from `#{ @schema_name }`.`#{ @table_name }`
+         where key_name != "PRIMARY"
+        }
+      end
+
+      def extract_indices(indices)
+        indices.map { |row| [row["Key_name"], row["Column_name"]] }.
+          inject(Hash.new { |h, k| h[k] = []}) do |memo, (idx, column)|
+            memo[idx] << column
+            memo
+          end
       end
 
-      def column
-        /^`([^ ]*)` ([^ ]*) (.*?),?$/
+      def extract_primary_key(schema)
+        cols = schema.select { |defn| defn["COLUMN_KEY"] == "PRI" }
+        keys = cols.map { |defn| defn["COLUMN_NAME"] }
+        keys.length == 1 ? keys.first : keys
       end
     end
   end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = "1.0.0.rc7"
+  VERSION = "1.0.2"
 end

data/spec/README.md

@@ -16,7 +16,7 @@ directory master and slave databases will get installed into.
 # setup
 
 You can set the integration specs up to run against a master slave setup by
-running the included `lhm-spec-clobber` script. this deletes the configured
+running the included `bin/lhm-spec-clobber.sh` script. this deletes the configured
 lhm master slave setup and reinstalls and configures a master slave setup.
 
 Follow the manual instructions if you want more control over this process.
@@ -25,7 +25,7 @@ Follow the manual instructions if you want more control over this process.
 
 ## set up instances
 
-    lhm-spec-setup-cluster
+    bin/lhm-spec-setup-cluster.sh
 
 ## start instances
 
@@ -35,7 +35,7 @@ Follow the manual instructions if you want more control over this process.
 
 ## run the grants
 
-    lhm-spec-grants
+    bin/lhm-spec-grants.sh
 
 ## run specs
 

data/spec/fixtures/users.ddl

@@ -5,6 +5,7 @@ CREATE TABLE `users` (
   `group` varchar(255) DEFAULT NULL,
   `created_at` datetime DEFAULT NULL,
   `comment` varchar(20) DEFAULT NULL,
+  `description` text,
   PRIMARY KEY (`id`),
   UNIQUE KEY `index_users_on_reference` (`reference`),
   KEY `index_users_on_username_and_created_at` (`username`,`created_at`)

data/spec/integration/integration_helper.rb

@@ -102,8 +102,12 @@ module IntegrationHelper
   def key?(table_name, cols, type = :non_unique)
     non_unique = type == :non_unique ? 1 : 0
     key_name = Lhm::SqlHelper.idx_name(table_name, cols)
-    query = "show indexes in #{ table_name } where key_name = '#{ key_name }' and non_unique = #{ non_unique }"
-    !!select_value(query)
+
+    !!select_value(%Q<
+      show indexes in #{ table_name }
+     where key_name = '#{ key_name }'
+       and non_unique = #{ non_unique }
+    >)
   end
 
   #

data/spec/integration/lhm_spec.rb

@@ -23,7 +23,8 @@ describe Lhm do
       slave do
         table_read(:users).columns["logins"].must_equal({
           :type => "int(12)",
-          :metadata => "DEFAULT '0'"
+          :is_nullable => "YES",
+          :column_default => '0'
         })
       end
     end
@@ -98,7 +99,22 @@ describe Lhm do
       slave do
         table_read(:users).columns["flag"].must_equal({
           :type => "tinyint(1)",
-          :metadata => "DEFAULT NULL"
+          :is_nullable => "YES",
+          :column_default => nil
+        })
+      end
+    end
+
+    it "should change a column" do
+      Lhm.change_table(:users) do |t|
+        t.change_column(:comment, "varchar(20) DEFAULT 'none' NOT NULL")
+      end
+
+      slave do
+        table_read(:users).columns["comment"].must_equal({
+          :type => "varchar(20)",
+          :is_nullable => "NO",
+          :column_default => "none"
         })
       end
     end

data/spec/integration/table_spec.rb

@@ -0,0 +1,48 @@
+# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+
+require 'lhm'
+require 'lhm/table'
+
+describe Lhm::Table do
+  include IntegrationHelper
+
+  describe Lhm::Table::Parser do
+    describe "create table parsing" do
+      before(:each) do
+        connect_master!
+        @table = table_create(:users)
+      end
+
+      it "should parse table name in show create table" do
+        @table.name.must_equal("users")
+      end
+
+      it "should parse primary key" do
+        @table.pk.must_equal("id")
+      end
+
+      it "should parse column type in show create table" do
+        @table.columns["username"][:type].must_equal("varchar(255)")
+      end
+
+      it "should parse column metadata" do
+        @table.columns["username"][:column_default].must_equal nil
+      end
+
+      it "should parse indices" do
+        @table.
+          indices["index_users_on_username_and_created_at"].
+          must_equal(["username", "created_at"])
+      end
+
+      it "should parse index" do
+        @table.
+          indices["index_users_on_reference"].
+          must_equal(["reference"])
+      end
+    end
+  end
+end

data/spec/unit/migrator_spec.rb

@@ -64,6 +64,15 @@ describe Lhm::Migrator do
         "alter table `lhmn_alt` drop `logins`"
       ])
     end
+
+    it "should change a column" do
+      @creator.change_column("logins", "INT(255)")
+
+      @creator.statements.must_equal([
+        "alter table `lhmn_alt` drop `logins`",
+        "alter table `lhmn_alt` add column `logins` INT(255)"
+      ])
+    end
   end
 
   describe "direct changes" do

data/spec/unit/table_spec.rb

@@ -9,11 +9,8 @@ describe Lhm::Table do
   include UnitHelper
 
   describe "names" do
-    before(:each) do
-      @table = Lhm::Table::Parser.new(fixture("users.ddl")).parse
-    end
-
     it "should name destination" do
+      @table = Lhm::Table.new("users")
       @table.destination_name.must_equal "lhmn_users"
     end
   end
@@ -34,40 +31,4 @@ describe Lhm::Table do
       @table.satisfies_primary_key?.must_equal false
     end
   end
-
-  describe Lhm::Table::Parser do
-    describe "create table parsing" do
-      before(:each) do
-        @table = Lhm::Table::Parser.new(fixture("users.ddl")).parse
-      end
-
-      it "should parse table name in show create table" do
-        @table.name.must_equal("users")
-      end
-
-      it "should parse primary key" do
-        @table.pk.must_equal("id")
-      end
-
-      it "should parse column type in show create table" do
-        @table.columns["username"][:type].must_equal("varchar(255)")
-      end
-
-      it "should parse column metadata" do
-        @table.columns["username"][:metadata].must_equal("DEFAULT NULL")
-      end
-
-      it "should parse indices in show create table" do
-        @table.
-          indices["index_users_on_username_and_created_at"][:metadata].
-          must_equal("(`username`,`created_at`)")
-      end
-
-      it "should parse indices in show create table" do
-        @table.
-          indices["index_users_on_reference"][:metadata].
-          must_equal("(`reference`)")
-      end
-    end
-  end
 end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: lhm
 version: !ruby/object:Gem::Version 
-  prerelease: true
-  segments: 
-  - 1
-  - 0
-  - 0
-  - rc7
-  version: 1.0.0.rc7
+  prerelease: 
+  version: 1.0.2
 platform: ruby
 authors: 
 - SoundCloud
@@ -18,8 +13,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-31 00:00:00 +01:00
-default_executable: 
+date: 2012-02-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mysql
@@ -29,10 +23,6 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 8
-        - 1
         version: 2.8.1
   type: :development
   version_requirements: *id001
@@ -44,10 +34,6 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 10
-        - 0
         version: 2.10.0
   type: :development
   version_requirements: *id002
@@ -59,8 +45,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id003
@@ -72,17 +56,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id004
 description: Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.
 email: rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com
 executables: 
-- lhm-spec-clobber
-- lhm-spec-grants
-- lhm-spec-setup-cluster
 - lhm-kill-queue
 extensions: []
 
@@ -96,11 +75,8 @@ files:
 - README.md
 - Rakefile
 - bin/lhm-kill-queue
-- bin/lhm-spec-clobber
 - bin/lhm-spec-clobber.sh
-- bin/lhm-spec-grants
 - bin/lhm-spec-grants.sh
-- bin/lhm-spec-setup-cluster
 - bin/lhm-spec-setup-cluster.sh
 - gemfiles/ar-2.3.gemfile
 - gemfiles/ar-3.1.gemfile
@@ -128,6 +104,7 @@ files:
 - spec/integration/integration_helper.rb
 - spec/integration/lhm_spec.rb
 - spec/integration/locked_switcher_spec.rb
+- spec/integration/table_spec.rb
 - spec/unit/chunker_spec.rb
 - spec/unit/entangler_spec.rb
 - spec/unit/intersection_spec.rb
@@ -137,7 +114,6 @@ files:
 - spec/unit/sql_helper_spec.rb
 - spec/unit/table_spec.rb
 - spec/unit/unit_helper.rb
-has_rdoc: true
 homepage: http://github.com/soundcloud/large-hadron-migrator
 licenses: []
 
@@ -151,23 +127,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 1
-      - 3
-      - 1
-      version: 1.3.1
+      version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: online schema changer for mysql
@@ -182,6 +152,7 @@ test_files:
 - spec/integration/integration_helper.rb
 - spec/integration/lhm_spec.rb
 - spec/integration/locked_switcher_spec.rb
+- spec/integration/table_spec.rb
 - spec/unit/chunker_spec.rb
 - spec/unit/entangler_spec.rb
 - spec/unit/intersection_spec.rb

data/bin/lhm-spec-clobber

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'open3'
-
-bindir = File.dirname(File.expand_path(__FILE__))
-script = File.join(bindir, File.basename($0) + ".sh 2>&1")
-
-Open3.popen3(script) do |input, output|
-  output.each { |line| print(line) }
-end

data/bin/lhm-spec-grants

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'open3'
-
-bindir = File.dirname(File.expand_path(__FILE__))
-script = File.join(bindir, File.basename($0) + ".sh 2>&1")
-
-Open3.popen3(script) do |input, output|
-  output.each { |line| print(line) }
-end

data/bin/lhm-spec-setup-cluster

@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'open3'
-
-bindir = File.dirname(File.expand_path(__FILE__))
-script = File.join(bindir, File.basename($0) + ".sh 2>&1")
-
-Open3.popen3(script) do |input, output|
-  output.each { |line| print(line) }
-end