lhm 1.0.0.rc4 → 1.0.0.rc5

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 1.0.0.rc5 (January 30, 2012)
+
+* moved scripts into bin, renamed, added to gem binaries
 
 # 1.0.0.rc4 (January 29, 2012)
 

data/bin/lhm-kill-queue

@@ -0,0 +1,153 @@
+#!/usr/bin/env ruby
+
+require 'active_record'
+require 'optparse'
+
+module Lhm
+  class KillQueue
+
+    def initialize
+      @port = 3306
+
+      OptionParser.new do |opts|
+        opts.on("-h", "--hostname HOSTNAME") { |v| @hostname = v }
+        opts.on("-u", "--username USERNAME") { |v| @username = v }
+        opts.on("-p", "--password PASSWORD") { |v| @password = v }
+        opts.on("-d", "--database DATABASE") { |v| @database = v }
+        opts.on("-m", "--mode MODE") { |v| @mode = v.to_sym }
+      end.parse!
+
+      unless(@hostname && @username && @password && @database)
+        abort usage
+      end
+
+      unless([:kill, :master, :slave].include?(@mode))
+        abort "specify -m kill OR -m master OR -m slave"
+      end
+
+      connect
+    end
+
+    def usage
+      <<-desc.gsub(/^      /, '')
+        kills queries on the given server after detecting 'lock table% -- lhm'.
+        usage:
+          lhm-kill-queue -h hostname -u username -p password -d database \\
+          (--kill | --master | --slave)
+
+      desc
+    end
+
+    def run
+      case @mode
+      when :kill then kill
+      when :master then master
+      when :slave then slave
+      end
+    end
+
+    def kill
+      lock = trip
+      puts "killing lock process #{ lock }."
+      kill_process(lock)
+    end
+
+    def master
+      lock = trip
+      puts "starting to kill non lhm processes in 1 second"
+      sleep(1)
+
+      [list_non_lhm].flatten.each do |process|
+        puts "killing #{ select_statement(process) }"
+        kill_process(process)
+        sleep(0.05)
+      end
+    end
+
+    def slave
+      lock = trip
+      puts "starting to kill non lhm SELECT processes in 1 second"
+      sleep(1)
+
+      [list_non_lhm].flatten.each do |process|
+        if(select?(process))
+          puts "killing #{ select_statement(process) }"
+          kill_process(process)
+          sleep(0.05)
+        end
+      end
+    end
+
+  private
+
+    def connect
+      ActiveRecord::Base.establish_connection({
+        :adapter => 'mysql',
+        :host => @hostname,
+        :port => @port,
+        :username => @username,
+        :password => @password,
+        :database => @database
+      })
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+
+    def list_non_lhm
+      select_process("info not like '% -- lhm' and time > 0 and command = 'Query'")
+    end
+
+    def trip
+      until res = select_process("info like 'lock table% -- lhm'")
+        sleep 0.2
+        print '.'
+      end
+
+      res
+    end
+
+    def kill_process(process_id)
+      connection.execute("kill #{ process_id }")
+    end
+
+    def select?(process)
+      if statement = select_statement(process)
+
+        case statement
+        when /delete/i then false
+        when /update/i then false
+        when /insert/i then false
+        else
+          !!statement.match(/select/i)
+        end
+      end
+    end
+
+    def select_statement(process)
+      if process
+        value %Q(
+          select info from information_schema.processlist where id = #{ process }
+        )
+      end
+    end
+
+    def select_process(predicate)
+      value %Q(
+        select id from information_schema.processlist
+         where db = '#{ @database }'
+           and user = '#{ @username }'
+           and #{ predicate }
+      )
+    end
+
+    def value(statement)
+      connection.select_value(statement)
+    end
+  end
+end
+
+killer = Lhm::KillQueue.new
+killer.run
+.

data/bin/lhm-spec-clobber

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'open3'
+
+bindir = File.dirname(File.expand_path(__FILE__))
+script = File.join(bindir, File.basename($0) + ".sh 2>&1")
+
+Open3.popen3(script) do |input, output|
+  output.each { |line| print(line) }
+end

data/{spec/config/clobber → bin/lhm-spec-clobber.sh}

@@ -3,7 +3,7 @@
 set -e
 set -u
 
-source .config
+source ~/.lhm
 
 lhmkill() {
   ps -ef | gsed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
@@ -21,7 +21,7 @@ echo removing $basedir
 rm -rf "$basedir"
 
 echo setting up cluster
-spec/config/setup-cluster
+lhm-spec-setup-cluster
 
 echo staring instances
 "$mysqldir"/bin/mysqld --defaults-file="$basedir/master/my.cnf" 2>&1 >$basedir/master/lhm.log &
@@ -29,7 +29,7 @@ echo staring instances
 sleep 5
 
 echo running grants
-spec/config/grants
+lhm-spec-grants
 
 trap lhmkill SIGTERM SIGINT
 

data/bin/lhm-spec-grants

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'open3'
+
+bindir = File.dirname(File.expand_path(__FILE__))
+script = File.join(bindir, File.basename($0) + ".sh 2>&1")
+
+Open3.popen3(script) do |input, output|
+  output.each { |line| print(line) }
+end

data/{spec/config/grants → bin/lhm-spec-grants.sh}

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source .config
+source ~/.lhm
 
 master() { "$mysqldir"/bin/mysql --protocol=TCP -P $master_port -uroot; }
 slave()  { "$mysqldir"/bin/mysql --protocol=TCP -P $slave_port -uroot; }

data/bin/lhm-spec-setup-cluster

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'open3'
+
+bindir = File.dirname(File.expand_path(__FILE__))
+script = File.join(bindir, File.basename($0) + ".sh 2>&1")
+
+Open3.popen3(script) do |input, output|
+  output.each { |line| print(line) }
+end

data/{spec/config/setup-cluster → bin/lhm-spec-setup-cluster.sh}

@@ -7,12 +7,14 @@
 set -e
 set -u
 
-source .config
+source ~/.lhm
 
 #
 # Main
 #
 
+install_bin="$(echo ./*/mysql_install_db)"
+
 mkdir -p "$basedir/master/data" "$basedir/slave/data"
 
 cat <<-CNF > $basedir/master/my.cnf
@@ -59,7 +61,7 @@ CNF
 
 (
   cd "$mysqldir"
-  ./scripts/mysql_install_db --datadir="$basedir/master/data"
-  ./scripts/mysql_install_db --datadir="$basedir/slave/data"
+  $install_bin --datadir="$basedir/master/data"
+  $install_bin --datadir="$basedir/slave/data"
 
 )

data/lhm.gemspec

@@ -16,8 +16,13 @@ Gem::Specification.new do |s|
   s.homepage      = %q{http://github.com/soundcloud/large-hadron-migrator}
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
+  s.executables   = [
+    "lhm-spec-clobber",
+    "lhm-spec-grants",
+    "lhm-spec-setup-cluster",
+    "lhm-kill-queue"
+  ]
 
   # this should be a real dependency, but we're using a different gem in our code
   s.add_development_dependency "mysql", "~> 2.8.1"

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = "1.0.0.rc4"
+  VERSION = "1.0.0.rc5"
 end

data/spec/README.md

@@ -1,26 +1,51 @@
 Preparing for master slave integration tests
 --------------------------------------------
 
+# configuration
+
+create ~/.lhm:
+
+    mysqldir=/usr/local/mysql
+    basedir=/opt/lhm-cluster
+    master_port=3306
+    slave_port=3307
+
+mysqldir specifies the location of your mysql install. basedir is the
+directory master and slave databases will get installed into.
+
+# setup
+
 You can set the integration specs up to run against a master slave setup by
-running the included `setup-cluster` script.
+running the included `lhm-spec-clobber` script. this deletes the configured
+lhm master slave setup and reinstalls and configures a master slave setup.
 
-# set up instances
+Follow the manual instructions if you want more control over this process.
 
-    spec/config/setup-cluster
+# manual setup
 
-# start instances
+## set up instances
 
-    basedir=/opt/lhm-cluster
+    lhm-spec-setup-cluster
+
+## start instances
+
+    basedir=/opt/lhm-luster
     mysqld --defaults-file="$basedir/master/my.cnf"
     mysqld --defaults-file="$basedir/slave/my.cnf"
 
-# run the grants
+## run the grants
 
-    spec/config/grants
+    lhm-spec-grants
 
-# run specs
+## run specs
 
 To run specs in slave mode, set the SLAVE=1 when running tests:
 
     MASTER_SLAVE=1 rake specs
 
+# connecting
+
+you can connect by running (with the respective ports):
+
+    mysql --protocol=TCP -p3307
+

metadata

@@ -6,8 +6,8 @@ version: !ruby/object:Gem::Version
   - 1
   - 0
   - 0
-  - rc4
-  version: 1.0.0.rc4
+  - rc5
+  version: 1.0.0.rc5
 platform: ruby
 authors: 
 - SoundCloud
@@ -18,7 +18,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-29 00:00:00 +01:00
+date: 2012-01-31 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -79,8 +79,11 @@ dependencies:
   version_requirements: *id004
 description: Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.
 email: rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com
-executables: []
-
+executables: 
+- lhm-spec-clobber
+- lhm-spec-grants
+- lhm-spec-setup-cluster
+- lhm-kill-queue
 extensions: []
 
 extra_rdoc_files: []
@@ -92,6 +95,13 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/lhm-kill-queue
+- bin/lhm-spec-clobber
+- bin/lhm-spec-clobber.sh
+- bin/lhm-spec-grants
+- bin/lhm-spec-grants.sh
+- bin/lhm-spec-setup-cluster
+- bin/lhm-spec-setup-cluster.sh
 - gemfiles/ar-2.3.gemfile
 - gemfiles/ar-3.1.gemfile
 - lhm.gemspec
@@ -107,12 +117,9 @@ files:
 - lib/lhm/sql_helper.rb
 - lib/lhm/table.rb
 - lib/lhm/version.rb
+- spec/.lhm.example
 - spec/README.md
 - spec/bootstrap.rb
-- spec/config/.config
-- spec/config/clobber
-- spec/config/grants
-- spec/config/setup-cluster
 - spec/fixtures/destination.ddl
 - spec/fixtures/origin.ddl
 - spec/fixtures/users.ddl
@@ -167,10 +174,6 @@ summary: online schema changer for mysql
 test_files: 
 - spec/README.md
 - spec/bootstrap.rb
-- spec/config/.config
-- spec/config/clobber
-- spec/config/grants
-- spec/config/setup-cluster
 - spec/fixtures/destination.ddl
 - spec/fixtures/origin.ddl
 - spec/fixtures/users.ddl