jetpants 0.7.2 → 0.7.4

data/bin/jetpants

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 jetpants_base_dir = File.expand_path(File.dirname(__FILE__) + '/..')
 $:.unshift File.join(jetpants_base_dir, 'lib')
-%w[thor pry highline/import terminal-table colored].each {|g| require g}
+%w[thor pry highline/import colored].each {|g| require g}
 
 module Jetpants
 
@@ -63,24 +63,25 @@ module Jetpants
         demoted.probe
       else
         demoted = ask_node 'Please enter the IP address of the node to demote:'
-        if demoted.running?
-          error 'Cannot demote a node that has no slaves!' unless demoted.has_slaves?
+      end
+
+      if demoted.running?
+        error 'Cannot demote a node that has no slaves!' unless demoted.has_slaves?
+      else
+        inform "Unable to connect to node #{demoted} to demote"
+        error  "Unable to perform promotion" unless agree "Please confirm that #{demoted} is offline [yes/no]: "
+        
+        # An asset-tracker plugin may have been populated the slave list anyway
+        if demoted.slaves && demoted.slaves.count > 0
+          demoted.slaves.each {|s| s.probe}
         else
-          inform "Unable to connect to node #{demoted} to demote"
-          error  "Unable to perform promotion" unless agree "Please confirm that #{demoted} is offline [yes/no]: "
-          
-          # An asset-tracker plugin may have been populated the slave list anyway
-          if demoted.slaves && demoted.slaves.count > 0
-            demoted.slaves.each {|s| s.probe}
-          else
-            replicas = ask("Please enter a comma-seperated list of IP addresses of all current replicas of #{demoted}: ").split /\s*,\s*/
-            error "No replicas entered" unless replicas && replicas.count > 0
-            error "User supplied list of replicas appears to be invalid - #{replicas}" unless replicas.all? {|replica| is_ip? replica}
-            demoted.instance_eval {@slaves = replicas.map &:to_db}
-            demoted.slaves.each do |replica|
-              # Validate that they are really slaves of demoted
-              error "#{replica} does not appear to be a valid replica of #{demoted}" unless replica.master == demoted
-            end
+          replicas = ask("Please enter a comma-seperated list of IP addresses of all current replicas of #{demoted}: ").split /\s*,\s*/
+          error "No replicas entered" unless replicas && replicas.count > 0
+          error "User supplied list of replicas appears to be invalid - #{replicas}" unless replicas.all? {|replica| is_ip? replica}
+          demoted.instance_eval {@slaves = replicas.map &:to_db}
+          demoted.slaves.each do |replica|
+            # Validate that they are really slaves of demoted
+            error "#{replica} does not appear to be a valid replica of #{demoted}" unless replica.master == demoted
           end
         end
       end
@@ -117,103 +118,51 @@ module Jetpants
     end
     
     
-    desc 'show_slaves', 'show the current slaves of a master'
-    method_option :node, :desc => 'node to query for slaves'
-    def show_slaves
-      node = options[:node] || ask('Please enter IP of node to query for slaves: ')
-      error "node (#{node}) does not appear to be an IP." unless is_ip? node
-      node = Jetpants::DB.new node
-      slaves = node.slaves rescue error("unable to connect to node #{node}")
-    
-      inform "node (#{node}) is a current slave of #{node.master}" if node.is_slave?
-
-      if node.has_slaves?
-        current_slaves = Terminal::Table.new :title => "slaves of master (#{node})", :headings => ["slave", "seconds behind master"] do |rows|
-          slaves.each do |slave|
-            rows << [slave, slave.seconds_behind_master]
-          end
-        end
-        puts current_slaves
-      else
-        inform "node (#{node}) currently has no slaves."
-      end
+    desc 'summary', 'display information about a node in the context of its pool'
+    method_option :node, :desc => 'IP address of node to query'
+    def summary
+      node = ask_node('Please enter node IP: ', options[:node])
+      node.pool(true).probe
+      describe node
+      node.pool(true).summary(true)
     end
     
     
-    desc 'show_master', 'show the current master of a node'
-    method_option :node, :desc => 'node to query for master'
-    method_option :siblings, :desc => 'show nodes current slave siblings'
-    def show_master
-      node = options[:node] || ask('Please enter the IP address of a node to query for master: ')
-      error "node (#{node}) does not appear to be an IP." unless is_ip? node
-      node = Jetpants::DB.new node
-
-      if node.has_slaves?
-        inform "node (#{node}) is a master to the following nodes: #{node.slaves.join(', ')}"
-      end rescue error("unable to connect to node #{node}")
+    desc 'pools', 'display a full summary of every pool in the topology'
+    def pools
+      Jetpants.pools.concurrent_each &:probe
+      Jetpants.pools.each &:summary
       
-      if node.is_slave?
-        inform "node (#{node}) is a slave of master (#{node.master})"
-        if options[:siblings]
-          current_siblings = Terminal::Table.new :title => "siblings of slave (#{node})", :headings => ["sibling slave", "seconds behind master (#{node.master})"] do |rows|
-            node.master.slaves.reject {|slave| slave == node}.each do |slave|
-              rows << [slave, slave.seconds_behind_master]
-            end
-          end
-          puts current_siblings
-        end
-      else
-        inform "node (#{node}) does not appear to be a slave"
+      counts = {master: 0, active_slave: 0, standby_slave: 0, backup_slave: 0}
+      Jetpants.pools.map(&:nodes).flatten.each {|n| counts[n.role] += 1}
+      
+      puts
+      puts "%4d global pools" % Jetpants.functional_partitions.count
+      puts "%4d shard pools" % Jetpants.shards.count
+      puts "---- --------------"
+      puts "%4d total pools" % Jetpants.pools.count
+      puts
+      
+      total = 0
+      counts.each do |role, count|
+        puts "%4d %ss" % [count, role.to_s.tr('_', ' ')]
+        total += count
       end
+      puts "---- --------------"
+      puts "%4d total nodes" % total
+      puts
     end
-    
-    
-    desc 'node_info', 'show information about a given node'
-    method_option :node, :desc => 'node to query for information'
-    def node_info
-      node = options[:node] || ask('Please enter node: ')
-      error "node address (#{node}) does not appear to be an ip" unless is_ip? node
-    
-      node = Jetpants::DB.new node
-      role =  case
-              when node.has_slaves?
-                :master
-              when node.is_slave?
-                :slave
-              else
-                :node
-              end rescue error("unable to connect to node #{node}")
 
-      node_info = Terminal::Table.new :title => "info on #{role}: #{node}"
-    
-      binary_log, binary_log_position = node.binlog_coordinates
-      node_info << [:binary_log, binary_log]
-      node_info << [:binary_log_position, binary_log_position]
-      node_info << [:read_only, node.read_only? ? 'true' : 'false'] 
-      
-      if node.is_slave?
-        slave_status = node.slave_status
-        slave_siblings = node.master.slaves.reject {|slave| slave == node}
-    
-        node_info << [:master, node.master]
-        node_info << [:sibling_slaves, slave_siblings.join(', ')]
-        node_info << [:replicating, node.replicating? ? 'true' : 'false']
-        if node.replicating?
-          node_info << [:seconds_behind_master, node.seconds_behind_master]
-          node_info << [:master_log, slave_status[:master_log_file]]
-          node_info << [:master_position, slave_status[:exec_master_log_pos]]
-        end
+    desc 'pools_compact', 'display a compact summary (master, name, and size) of every pool in the topology'
+    def pools_compact
+      puts
+      Jetpants.shards.each do |s| 
+        puts "[%-12s] %8s to %-11s = %3s GB" % [s.ip, s.min_id, s.max_id, s.data_set_size(true)]
       end
-      puts node_info
-      
-      if node.has_slaves?
-        current_slaves = Terminal::Table.new :title => "slaves of #{node}", :headings => ["slave", "seconds behind master"] do |rows|
-          node.slaves.each do |slave|
-            rows << [slave, slave.seconds_behind_master]
-          end
-        end
-        puts current_slaves
+      Jetpants.functional_partitions.each do |p| 
+        puts "[%-12s] %-23s = %3s GB" % [p.ip, p.name, p.data_set_size(true)]
       end
+      puts
     end
     
     
@@ -228,9 +177,9 @@ module Jetpants
     method_option :target, :desc => 'IP of node to clone to'
     def clone_slave
       puts "This task clones the data set of a standby slave."
-      source = options[:source] || ask('Please enter IP of node to clone from: ')
-      error "source (#{source}) does not appear to be an IP." unless is_ip? source
-      source = Jetpants::DB.new source
+      source = ask_node('Please enter IP of node to clone from: ', options[:source])
+      describe source
+      
       target = options[:target] || ask('Please enter comma-separated list of IP addresses to clone to: ')
       targets = target.split(',').map do |ip|
         ip.strip!
@@ -257,13 +206,14 @@ module Jetpants
     method_option :node, :desc => 'IP of standby slave to activate'
     def activate_slave
       puts "This task turns a standby slave into an active slave, OR alters an active slave's weight."
-      node = options[:node] || ask('Please enter node IP: ')
+      node = ask_node('Please enter node IP: ', options[:node])
+      describe node
+
       weight = options[:weight] || ask('Please enter weight, or ENTER for default of 100: ')
       weight = 100 if weight == ''
       weight = weight.to_i
-      error "node address (#{node}) does not appear to be an IP" unless is_ip? node
       error "Adding a slave of weight 0 makes no sense, use pull_slave instead" if weight == 0
-      node = node.to_db
+      
       node.pool.mark_slave_active(node, weight)
       Jetpants.topology.write_config
     end
@@ -277,9 +227,9 @@ module Jetpants
     method_option :node, :desc => 'IP of active slave to pull'
     def pull_slave
       puts "This task turns an active slave into a standby slave."
-      node = options[:node] || ask('Please enter node IP: ')
-      error "node address (#{node}) does not appear to be an ip" unless is_ip? node
-      node = node.to_db
+      node = ask_node('Please enter node IP: ', options[:node])
+      describe node
+      raise "Node is not an active slave" unless node.role == :active_slave
       node.pool.mark_slave_standby(node)
       Jetpants.topology.write_config
     end
@@ -288,12 +238,11 @@ module Jetpants
     desc 'destroy_slave', 'remove a standby slave from its pool'
     method_option :node, :desc => 'IP of standby slave to remove'
     def destroy_slave
-      puts "This task removes a standby slave from its pool entirely. THIS IS PERMANENT, ie, it does a RESET SLAVE on the target."
-      node = options[:node] || ask('Please enter node IP: ')
-      error "node address (#{node}) does not appear to be an IP" unless is_ip? node
-      node = node.to_db
-      raise "Node is not a standby slave" unless node.is_standby?
-      raise "Aborting" unless ask('Please type YES in all capital letters to confirm: ') == 'YES'
+      puts "This task removes a standby/backup slave from its pool entirely. THIS IS PERMANENT, ie, it does a RESET SLAVE on the target."
+      node = ask_node('Please enter node IP: ', options[:node])
+      describe node
+      raise "Node is not a standby or backup slave" unless (node.is_standby? || node.for_backups?)
+      raise "Aborting" unless ask('Please type YES in all capital letters to confirm removing node from its pool: ') == 'YES'
       node.pool.remove_slave!(node)
     end
     
@@ -302,10 +251,9 @@ module Jetpants
     method_option :node, :desc => 'IP of standby slave to rebuild'
     def rebuild_slave
       puts "This task exports all data on a standby/backup slave and then re-imports it."
-      node = options[:node] || ask('Please enter node IP: ')
-      error "node address (#{node}) does not appear to be an ip" unless is_ip? node
-      node = node.to_db
-      raise "Node is not a standby or backup slave" unless node.is_standby? || node.for_backups?
+      node = ask_node('Please enter node IP: ', options[:node])
+      describe node
+      raise "Node is not a standby or backup slave" unless (node.is_standby? || node.for_backups?)
       raise "Cannot rebuild non-shard slaves from command suite; use jetpants console instead" unless node.pool.is_a?(Shard)
       node.rebuild!
     end
@@ -417,11 +365,7 @@ module Jetpants
     method_option :min_id, :desc => 'Minimum ID of parent shard being split'
     method_option :max_id, :desc => 'Maximum ID of parent shard being split'
     def shard_split_child_writes
-      shard_min = options[:min_id] || ask('Please enter min ID of the parent shard: ')
-      shard_max = options[:max_id] || ask('Please enter max ID of the parent shard: ')
-      s = Jetpants.topology.shard shard_min, shard_max
-      raise "Shard not found" unless s
-      raise "Shard isn't in expected state" unless s.state == :deprecated && s.children.count > 1
+      s = ask_shard_being_split
       s.move_writes_to_children
       Jetpants.topology.write_config
     end
@@ -439,11 +383,7 @@ module Jetpants
     method_option :min_id, :desc => 'Minimum ID of parent shard being split'
     method_option :max_id, :desc => 'Maximum ID of parent shard being split'
     def shard_split_cleanup
-      shard_min = options[:min_id] || ask('Please enter min ID of the parent shard: ')
-      shard_max = options[:max_id] || ask('Please enter max ID of the parent shard: ')
-      s = Jetpants.topology.shard shard_min, shard_max
-      raise "Shard not found" unless s
-      raise "Shard isn't in expected state" unless s.state == :deprecated && s.children.count > 1
+      s = ask_shard_being_split
       s.cleanup!
     end
     def self.after_shard_split_cleanup
@@ -456,6 +396,11 @@ module Jetpants
     desc 'shard_cutover', 'truncate the current last shard range, and add a new shard after it'
     method_option :cutover_id, :desc => 'Minimum ID of new last shard being created'
     def shard_cutover
+      # ensure the spares are available before beginning
+      raise "Not enough total spare machines!" unless Jetpants.topology.count_spares >= Jetpants.standby_slaves_per_pool + 1
+      raise "Not enough standby_slave role spare machines!" unless Jetpants.topology.count_spares(role: 'standby_slave') >= Jetpants.standby_slaves_per_pool
+      raise "Cannot find a spare master-role machine!" unless Jetpants.topology.count_spares(role: 'master') >= 1
+
       cutover_id = options[:cutover_id] || ask('Please enter min ID of the new shard to be created: ')
       cutover_id = cutover_id.to_i
       last_shard = Jetpants.topology.shards.select {|s| s.max_id == 'INFINITY' && s.in_config?}.first
@@ -511,11 +456,36 @@ module Jetpants
         puts message.blue
       end
       
-      def ask_node(prompt)
-        node = ask prompt
+      def describe node
+        puts "Node #{node} (#{node.hostname}:#{node.port}) has role #{node.role} in pool #{node.pool(true)}.".green
+      end
+      
+      def ask_node(prompt, supplied_node=false)
+        node = supplied_node || ask(prompt)
         error "Node (#{node}) does not appear to be an IP address." unless is_ip? node
         node.to_db
       end
+      
+      def ask_shard_being_split
+        shards_being_split = Jetpants.shards.select {|s| s.children.count > 0}
+        if shards_being_split.count == 0
+          raise 'No shards are currently being split. You can only use this task after running "jetpants shard_split".'
+        elsif shards_being_split.count == 1
+          s = shards_being_split[0]
+          puts "Detected #{s} as the only shard currently involved in a split operation."
+          error "Aborting." unless agree "Is this the right shard that you want to perform this action on? [yes/no]: "
+        else
+          puts "The following shards are currently involved in a split operation:"
+          shards_being_split.each {|sbs| puts "* #{sbs}"}
+          puts "Which shard would you like to perform this action on?"
+          shard_min = options[:min_id] || ask('Please enter min ID of the parent shard: ')
+          shard_max = options[:max_id] || ask('Please enter max ID of the parent shard: ')
+          s = Jetpants.topology.shard shard_min, shard_max
+          raise "Shard not found" unless s
+        end
+        raise "Shard isn't in expected state" unless s.state == :deprecated
+        s
+      end
     end
     
     def self.reminders(*strings)

data/doc/commands.rdoc

@@ -103,13 +103,13 @@ With an asset tracker, \Jetpants allows you to mark a shard as read-only or comp
 
 == Informational commands
 
-These commands do not require an asset tracker. They may be fleshed out further in a future release of \Jetpants.
+These commands display status information about a particular node, pool, or the entire topology.
 
-<b><tt>jetpants node_info</tt></b> displays information on a MySQL instance.
+<b><tt>jetpants summary</tt></b> displays information about a node, along with its pool. Does not require an asset tracker.
 
-<b><tt>jetpants show_master</tt></b> tells you what node is the master of a given MySQL instance.
+<b><tt>jetpants pools</tt></b> displays full information about all pools (name, size, full node list including roles), and then displays counts of all nodes by role. Requires an asset tracker to obtain the list of all pools. This command may take a minute or two to run (depending on topology size) since it has to probe every node to determine roles.
 
-<b><tt>jetpants show_slaves</tt></b> displays a list of slaves (along with current slave lag) of a MySQL instance.
+<b><tt>jetpants pools_compact</tt></b> displays condensed information about all pools (name, master IP, size). Requires an asset tracker to obtain the list of all pools.
 
 
 == Miscellaneous

data/doc/faq.rdoc

@@ -7,6 +7,26 @@
 The benefit of a toolkit is that you can still leverage standard MySQL replication, still use InnoDB/XtraDB as a robust storage engine choice, etc. \Jetpants largely doesn't interfere with any of that, and instead just provides tools to help you manage a large MySQL topology and support a range-based sharding scheme.
 
 
+== If it's not a server, how does my application know which pool/shard to send queries to?
+
+\Jetpants doesn't perform query routing; it leaves that to your application. The purpose of Jetpants is to *operationally* manage your database topology.  It provides an object hierarchy that allows you to manipulate and automate your DB topology, and offers efficient implementations of common operational tasks in a sharded environment (bulk data importing/exporting, replica cloning, shard splitting, master promotion).
+
+That said, you could integrate it into an application and implement query routing using its methods like Jetpants::Topology#shard_db_for_id. This method just compares a sharding key value to the list of shard ranges, to figure out which DB the query should go to.  It's simple to port this logic to non-Ruby apps as well, since the list of shard ranges can be expressed in a portable format like YAML or JSON.
+
+Doing this more seamlessly would require some amount of parsing of SQL in your app or service.  This might be a small amount of code (if you're just scanning queries for "WHERE sharding_key_column = ?") or might be a lot (if you're trying to support joins and complex expressions).
+
+
+== Can I use \Jetpants to query some or all shards at once?
+
+Absolutely! You can even use it to implement some simple map/reduce-style distributed computation in MySQL:
+
+   Jetpants.topology.shards.concurrent_map {|s| s.query_return_first_value 'SELECT MAX(id) from foo'}.reduce(0) {|overall_max, val| [val, overall_max].max}
+
+This example concurrently queries all shards to determine the maximum ID in existence for table foo. This might be useful if, say, you're using Memcached as an ID generation service, and need to determine the previous highest-assigned ID after a restart or failover event.
+
+Another simple example would be to obtain counts of rows across all shards. In theory you can apply this same technique to other map/reduce style requirements with arbitrarily complex queries.
+
+
 == Is \Jetpants still useful if my architecture isn't sharded?
 
 Potentially, since \Jetpants fully supports "global" pools, also known as "functional partitions". You can even use \Jetpants to help manage a standard single-pool MySQL topology (1 master and some number of slaves) for handling common operations like slave cloning and master promotions. That said, there are other tools that may be easier to use if your MySQL footprint is smaller than, say, a dozen machines.

data/lib/jetpants.rb

@@ -60,7 +60,9 @@ module Jetpants
     # Returns a hash containing :user => username string, :pass => password string
     # for the MySQL replication user, as found in Jetpants' configuration. Plugins
     # may freely override this if there's a better way to obtain this password --
-    # for example, by parsing master.info on a slave in your topology.
+    # for example, by parsing master.info on a specific slave in your topology.
+    # SEE ALSO: DB#replication_credentials, which only falls back to the global
+    # version when needed.
     def replication_credentials
       {user: @config['mysql_repl_user'], pass: @config['mysql_repl_password']}
     end
@@ -97,4 +99,5 @@ module Jetpants
   
   # Finally, initialize topology object
   @topology = Topology.new
+  @topology.load_pools
 end

data/lib/jetpants/db.rb

@@ -31,6 +31,10 @@ module Jetpants
     @@all_dbs = {}
     @@all_dbs_mutex = Mutex.new
     
+    def self.clear
+      @@all_dbs_mutex.synchronize {@@all_dbs = {}}
+    end
+    
     # Because this class is rather large, methods have been grouped together
     # and moved to separate files in lib/jetpants/db. We load these all now.
     # They each just re-open the DB class and add some methods.
@@ -51,15 +55,19 @@ module Jetpants
     
     def initialize(ip, port=3306)
       @ip, @port = ip, port.to_i
-      @user = false # connections will default to app user
+      @host = Host.new(ip)
+      
+      # These get set upon DB#probe being run
       @master = nil
       @slaves = nil
       @repl_paused = nil
       @running = nil
-      @host = Host.new(ip)
+      
+      # These get set upon DB#connect being run
+      @user = nil
+      @schema = nil
     end
     
-    
     ###### Host methods ########################################################
     
     # Jetpants::DB delegates missing methods to its Jetpants::Host.

data/lib/jetpants/db/client.rb

@@ -5,21 +5,24 @@ module Jetpants
   #++
   
   class DB
-    # Runs the provided SQL statement as root, and returns the response as a single string.
+    # Runs the provided SQL statement as root, locally via an SSH command line, and
+    # returns the response as a single string.
     # Available options:
     # :terminator:: how to terminate the query, such as '\G' or ';'. (default: '\G')
     # :parse:: parse a single-row, vertical-format result (:terminator must be '\G') and return it as a hash
+    # :schema:: name of schema to use, or true to use this DB's default. This may have implications when used with filtered replication! (default: nil, meaning no schema)
     # :attempts:: by default, queries will be attempted up to 3 times. set this to 0 or false for non-idempotent queries.
     def mysql_root_cmd(cmd, options={})
       terminator = options[:terminator] || '\G'
       attempts = (options[:attempts].nil? ? 3 : (options[:attempts].to_i || 1))
+      schema = (options[:schema] == true ? app_schema : options[:schema])
       failures = 0
       
       begin
         raise "MySQL is not running" unless running?
         supply_root_pw = (Jetpants.mysql_root_password ? "-p#{Jetpants.mysql_root_password}" : '')
         supply_port = (@port == 3306 ? '' : "-h 127.0.0.1 -P #{@port}")
-        real_cmd = %Q{mysql #{supply_root_pw} #{supply_port} -ss -e "#{cmd}#{terminator}" #{Jetpants.mysql_schema}}
+        real_cmd = %Q{mysql #{supply_root_pw} #{supply_port} -ss -e "#{cmd}#{terminator}" #{schema}}
         real_cmd.untaint
         result = ssh_cmd!(real_cmd)
         raise result if result && result.downcase.start_with?('error ')
@@ -34,53 +37,91 @@ module Jetpants
       end
     end
     
-    # Returns a Sequel database object
-    def mysql
-      return @db if @db
+    # Returns a Sequel database object for use in sending queries to the DB remotely.
+    # Initializes (or re-initializes) the connection pool upon first use or upon
+    # requesting a different user or schema. Note that we only maintain one connection
+    # pool per DB.
+    # Valid options include :user, :pass, :schema, :max_conns or omit these to use
+    # defaults.
+    def connect(options={})
+      options[:user]    ||= app_credentials[:user]
+      options[:schema]  ||= app_schema
+      
+      return @db if @db && @user == options[:user] && @schema == options[:schema]
+
+      disconnect if @db
+      
       @db = Sequel.connect(
         :adapter          =>  'mysql2',
         :host             =>  @ip,
         :port             =>  @port,
-        :user             =>  @user || Jetpants.app_credentials[:user],
-        :password         =>  Jetpants.app_credentials[:pass],
-        :database         =>  Jetpants.mysql_schema,
-        :max_connections  =>  Jetpants.max_concurrency)
+        :user             =>  options[:user],
+        :password         =>  options[:pass] || app_credentials[:pass],
+        :database         =>  options[:schema],
+        :max_connections  =>  options[:max_conns] || Jetpants.max_concurrency)
+      @user = options[:user]
+      @schema = options[:schema]
+      @db
     end
-    alias init_db_connection_pool mysql
     
-    # Closes existing mysql connection pool and opens a new one. Useful when changing users.
-    # Supply a new user name as the param, or nothing/false to keep old user name, or
-    # a literal true value to switch to the default app user in Jetpants configuration
-    def reconnect(new_user=false)
-      @user = (new_user == true ? Jetpants.app_credentials[:user] : new_user) if new_user
+    # Closes the database connection(s) in the connection pool.
+    def disconnect
       if @db
         @db.disconnect rescue nil
         @db = nil
       end
-      init_db_connection_pool
+      @user = nil
+      @schema = nil
+    end
+    
+    # Disconnects and reconnects to the database.
+    def reconnect(options={})
+      disconnect # force disconnection even if we're not changing user or schema
+      connect(options)
+    end
+    
+    # Returns a Sequel database object representing the current connection. If no
+    # current connection, this will automatically connect with default options.
+    def connection
+      @db || connect
+    end
+    alias mysql connection
+    
+    # Returns a hash containing :user and :pass indicating how the application connects to
+    # this database instance.  By default this just delegates to Jetpants.application_credentials,
+    # which obtains credentials from the Jetpants config file. Plugins may override this
+    # to use different credentials for particular hosts or in certain situations.
+    def app_credentials
+      Jetpants.app_credentials
+    end
+    
+    # Returns the schema name ("database name" in MySQL parlance) to use for connections.
+    # Defaults to just calling Jetpants.mysql_schema, but plugins may override.
+    def app_schema
+      Jetpants.mysql_schema
     end
     
     # Execute a write (INSERT, UPDATE, DELETE, REPLACE, etc) query.
     # If the query is an INSERT, returns the last insert ID (if an auto_increment
     # column is involved).  Otherwise returns the number of affected rows.
     def query(sql, *binds)
-      ds = mysql.fetch(sql, *binds)
-      mysql.execute_dui(ds.update_sql) {|c| return c.last_id > 0 ? c.last_id : c.affected_rows}
+      ds = connection.fetch(sql, *binds)
+      connection.execute_dui(ds.update_sql) {|c| return c.last_id > 0 ? c.last_id : c.affected_rows}
     end
     
     # Execute a read (SELECT) query. Returns an array of hashes.
     def query_return_array(sql, *binds)
-      mysql.fetch(sql, *binds).all
+      connection.fetch(sql, *binds).all
     end
     
     # Execute a read (SELECT) query. Returns a hash of the first row only.
     def query_return_first(sql, *binds)
-      mysql.fetch(sql, *binds).first
+      connection.fetch(sql, *binds).first
     end
     
     # Execute a read (SELECT) query. Returns the value of the first column of the first row only.
     def query_return_first_value(sql, *binds)
-      mysql.fetch(sql, *binds).single_value
+      connection.fetch(sql, *binds).single_value
     end
     
     # Parses the result of a MySQL query run with a \G terminator. Useful when

data/lib/jetpants/db/import_export.rb

@@ -10,7 +10,7 @@ module Jetpants
       output 'Exporting table definitions'
       supply_root_pw = (Jetpants.mysql_root_password ? "-p#{Jetpants.mysql_root_password}" : '')
       supply_port = (@port == 3306 ? '' : "-h 127.0.0.1 -P #{@port}")
-      cmd = "mysqldump #{supply_root_pw} #{supply_port} -d #{Jetpants.mysql_schema} " + tables.join(' ') + " >#{Jetpants.export_location}/create_tables_#{@port}.sql"
+      cmd = "mysqldump #{supply_root_pw} #{supply_port} -d #{app_schema} " + tables.join(' ') + " >#{Jetpants.export_location}/create_tables_#{@port}.sql"
       cmd.untaint
       result = ssh_cmd(cmd)
       output result
@@ -23,7 +23,7 @@ module Jetpants
     # CAUTION IF RUNNING THIS MANUALLY!
     def import_schemata!
       output 'Dropping and re-creating table definitions'
-      result = mysql_root_cmd "source #{Jetpants.export_location}/create_tables_#{@port}.sql", :terminator => ''
+      result = mysql_root_cmd "source #{Jetpants.export_location}/create_tables_#{@port}.sql", terminator: '', schema: true
       output result
     end
     
@@ -44,12 +44,12 @@ module Jetpants
       create_user(import_export_user)
       grant_privileges(import_export_user)               # standard privs
       grant_privileges(import_export_user, '*', 'FILE')  # FILE global privs
-      reconnect(import_export_user)
+      reconnect(user: import_export_user)
       @counts ||= {}
       tables.each {|t| @counts[t.name] = export_table_data t, min_id, max_id}
     ensure
-      reconnect(true) # switches back to default app user
-      drop_user(import_export_user)
+      reconnect(user: app_credentials[:user])
+      drop_user import_export_user
     end
     
     # Exports data for a table. Only includes the data subset that falls
@@ -108,7 +108,7 @@ module Jetpants
       create_user(import_export_user)
       grant_privileges(import_export_user)               # standard privs
       grant_privileges(import_export_user, '*', 'FILE')  # FILE global privs
-      reconnect(import_export_user)
+      reconnect(user: import_export_user)
       
       import_counts = {}
       tables.each {|t| import_counts[t.name] = import_table_data t, min_id, max_id}
@@ -124,7 +124,7 @@ module Jetpants
       end
       
     ensure
-      reconnect(true) # switches back to default app user
+      reconnect(user: app_credentials[:user])
       drop_user(import_export_user)
     end
     
@@ -194,7 +194,7 @@ module Jetpants
     # Supply the ID range (in terms of the table's sharding key)
     # of rows to KEEP.
     def prune_data_to_range(tables, keep_min_id, keep_max_id)
-      reconnect(true)
+      reconnect(user: app_credentials[:user])
       tables.each do |t|
         output "Cleaning up data, pruning to only keep range #{keep_min_id}-#{keep_max_id}", t
         rows_deleted = 0
@@ -260,7 +260,6 @@ module Jetpants
       stop_query_killer
       disable_binary_logging
       restart_mysql
-      reconnect
       pause_replication if is_slave?
       
       # Automatically detect missing min/max. Assumes that all tables' primary keys
@@ -293,15 +292,6 @@ module Jetpants
       enable_monitoring
     end
     
-    # Returns the data set size in bytes (if in_gb is false or omitted) or in gigabytes
-    # (if in_gb is true).  Note that this is actually in gibibytes (2^30) rather than
-    # a metric gigabyte.  This puts it on the same scale as the output to tools like 
-    # "du -h" and "df -h".
-    def data_set_size(in_gb=false)
-      bytes = dir_size("#{mysql_directory}/#{Jetpants.mysql_schema}")
-      in_gb ? (bytes / 1073741824.0).round : bytes
-    end
-    
     # Copies mysql db files from self to one or more additional DBs.
     # WARNING: temporarily shuts down mysql on self, and WILL OVERWRITE CONTENTS
     # OF MYSQL DIRECTORY ON TARGETS.  Confirms first that none of the targets
@@ -321,7 +311,7 @@ module Jetpants
       fast_copy_chain(mysql_directory, 
                       destinations,
                       port: 3306,
-                      files: ['ibdata1', 'mysql', 'test', Jetpants.mysql_schema],
+                      files: ['ibdata1', 'mysql', 'test', app_schema],
                       overwrite: true)
       [self, targets].flatten.concurrent_each {|t| t.start_mysql; t.start_query_killer}
     end

data/lib/jetpants/db/privileges.rb

@@ -9,28 +9,30 @@ module Jetpants
     # configuration will be used instead.  Does not automatically grant any
     # privileges; use DB#grant_privileges for that.
     def create_user(username=false, database=false, password=false)
-      username ||= Jetpants.app_credentials[:user]
-      database ||= Jetpants.mysql_schema
-      password ||= Jetpants.app_credentials[:pass]
+      username ||= app_credentials[:user]
+      database ||= app_schema
+      password ||= app_credentials[:pass]
       commands = []
       Jetpants.mysql_grant_ips.each do |ip|
         commands << "CREATE USER '#{username}'@'#{ip}' IDENTIFIED BY '#{password}'"
       end
       commands << "FLUSH PRIVILEGES"
-      mysql_root_cmd(commands.join '; ')
+      commands = commands.join '; '
+      mysql_root_cmd commands, schema: true
     end
     
     # Drops a user. Can optionally make this statement skip replication, if you
     # want to drop a user on master and not on its slaves.
     def drop_user(username=false, skip_binlog=false)
-      username ||= Jetpants.app_credentials[:user]
+      username ||= app_credentials[:user]
       commands = []
       commands << 'SET sql_log_bin = 0' if skip_binlog
       Jetpants.mysql_grant_ips.each do |ip|
         commands << "DROP USER '#{username}'@'#{ip}'"
       end
       commands << "FLUSH PRIVILEGES"
-      mysql_root_cmd(commands.join '; ')
+      commands = commands.join '; '
+      mysql_root_cmd commands, schema: true
     end
     
     # Grants privileges to the given username for the specified database.
@@ -50,8 +52,8 @@ module Jetpants
     # Helper method that can do grants or revokes.
     def grant_or_revoke_privileges(statement, username, database, privileges)
       preposition = (statement.downcase == 'revoke' ? 'FROM' : 'TO')
-      username ||= Jetpants.app_credentials[:user]
-      database ||= Jetpants.mysql_schema
+      username ||= app_credentials[:user]
+      database ||= app_schema
       privileges = Jetpants.mysql_grant_privs if privileges.empty?
       privileges = privileges.join(',')
       commands = []
@@ -60,14 +62,15 @@ module Jetpants
         commands << "#{statement} #{privileges} ON #{database}.* #{preposition} '#{username}'@'#{ip}'"
       end
       commands << "FLUSH PRIVILEGES"
-      mysql_root_cmd(commands.join '; ')
+      commands = commands.join '; '
+      mysql_root_cmd commands, schema: true
     end
     
     # Disables access to a DB by the application user, and sets the DB to 
     # read-only. Useful when decommissioning instances from a shard that's
     # been split.
     def revoke_all_access!
-      user_name = Jetpants.app_credentials[:user]
+      user_name = app_credentials[:user]
       enable_read_only!
       output "Revoking access for user #{user_name}."
       output(drop_user(user_name, true)) # drop the user without replicating the drop statement to slaves

data/lib/jetpants/db/replication.rb

@@ -12,8 +12,9 @@ module Jetpants
     # If you omit :log_pos or :log_file, uses the current position/file from new_master,
     # though this is only safe if new_master is not receiving writes!
     #
-    # If you omit :user or :password, tries obtaining replication credentials from global
-    # settings, and failing that from the current node (assuming it is already a slave)
+    # If you omit :user and :password, tries obtaining replication credentials from the
+    # current node (assuming it is already a slave) or if that fails then from the global
+    # settings.
     def change_master_to(new_master, option_hash={})
       return disable_replication! unless new_master   # change_master_to(nil) alias for disable_replication!
       return if new_master == master                  # no change
@@ -25,8 +26,8 @@ module Jetpants
         logfile, pos = new_master.binlog_coordinates
       end
       
-      repl_user = option_hash[:user]     || Jetpants.replication_credentials[:user] || replication_credentials[:user]
-      repl_pass = option_hash[:password] || Jetpants.replication_credentials[:pass] || replication_credentials[:pass]
+      repl_user = option_hash[:user]     || replication_credentials[:user]
+      repl_pass = option_hash[:password] || replication_credentials[:pass]
 
       pause_replication if @master && !@repl_paused
       result = mysql_root_cmd "CHANGE MASTER TO " +
@@ -78,12 +79,12 @@ module Jetpants
     # Resumes replication on self afterwards, but does NOT automatically start
     # replication on the targets.
     # You can omit passing in the replication user/pass if this machine is itself
-    # a slave OR already has at least one slave.
+    # a slave OR already has at least one slave OR the global setting is fine to use here.
     # Warning: takes self offline during the process, so don't use on a master that
     # is actively in use by your application!
     def enslave!(targets, repl_user=false, repl_pass=false)
-      repl_user ||= (Jetpants.replication_credentials[:user] || replication_credentials[:user])
-      repl_pass ||= (Jetpants.replication_credentials[:pass] || replication_credentials[:pass])
+      repl_user ||= replication_credentials[:user]
+      repl_pass ||= replication_credentials[:pass]
       disable_monitoring
       pause_replication if master && ! @repl_paused
       file, pos = binlog_coordinates
@@ -113,8 +114,8 @@ module Jetpants
         t.change_master_to( master, 
                             log_file: file,
                             log_pos:  pos,
-                            user:     (Jetpants.replication_credentials[:user] || replication_credentials[:user]),
-                            password: (Jetpants.replication_credentials[:pass] || replication_credentials[:pass])  )
+                            user:     replication_credentials[:user],
+                            password: replication_credentials[:pass]  )
       end
       resume_replication # should already have happened from the clone_to! restart anyway, but just to be explicit
       catch_up_to_master
@@ -205,14 +206,21 @@ module Jetpants
     end
     
     # Reads an existing master.info file on this db or one of its slaves,
-    # propagates the info back to the Jetpants singleton, and returns it
+    # propagates the info back to the Jetpants singleton, and returns it as
+    # a hash containing :user and :pass.
+    # If the node is not a slave and has no slaves, will use the global Jetpants
+    # config instead.
     def replication_credentials
-      unless @master || @slaves.count > 0
-        raise "Cannot obtain replication credentials from orphaned instance -- this instance is not a slave, and has no slaves"
+      user = false
+      pass = false
+      if master || slaves.count > 0
+        target = (@master ? self : @slaves[0])
+        results = target.ssh_cmd("cat #{mysql_directory}/master.info | head -6 | tail -2").split
+        if results.count == 2 && results[0] != 'test'
+          user, pass = results
+        end
       end
-      target = (@master ? self : @slaves[0])
-      user, pass = target.ssh_cmd("cat #{mysql_directory}/master.info | head -6 | tail -2").split
-      {user: user, pass: pass}
+      user && pass ? {user: user, pass: pass} : Jetpants.replication_credentials
     end
     
     # Disables binary logging in my.cnf.  Does not take effect until you restart

data/lib/jetpants/db/server.rb

@@ -31,10 +31,21 @@ module Jetpants
     # Restarts MySQL.
     def restart_mysql
       @repl_paused = false if @master
+      
+      # Disconnect if we were previously connected
+      user, schema = false, false
+      if @db
+        user, schema = @user, @schema
+        disconnect
+      end
+      
       output "Attempting to restart MySQL"
       output service(:restart, 'mysql')
       confirm_listening
       @running = true
+      
+      # Reconnect if we were previously connected
+      connect(user: user, schema: schema) if user || schema
     end
     
     # Has no built-in effect. Plugins can override it, and/or implement

data/lib/jetpants/db/state.rb

@@ -73,7 +73,7 @@ module Jetpants
     # remembers the state from the previous probe and any actions since then.
     def replicating?
       status = slave_status
-      [status[:slave_io_running], status[:slave_sql_running]].all? {|s| s.downcase == 'yes'}
+      [status[:slave_io_running], status[:slave_sql_running]].all? {|s| s && s.downcase == 'yes'}
     end
 
     # Returns true if this instance has a master, false otherwise.
@@ -174,6 +174,16 @@ module Jetpants
       end
     end
     
+    # Returns the data set size in bytes (if in_gb is false or omitted) or in gigabytes
+    # (if in_gb is true).  Note that this is actually in gibibytes (2^30) rather than
+    # a metric gigabyte.  This puts it on the same scale as the output to tools like 
+    # "du -h" and "df -h".
+    def data_set_size(in_gb=false)
+      bytes = dir_size("#{mysql_directory}/#{app_schema}")
+      in_gb ? (bytes / 1073741824.0).round : bytes
+    end
+    
+    
     ###### Private methods #####################################################
     
     private

data/lib/jetpants/host.rb

@@ -8,11 +8,15 @@ module Jetpants
   class Host
     include CallbackHandler
     
+    # IP address of the Host, as a string.
+    attr_reader :ip
+
     @@all_hosts = {}
     @@all_hosts_mutex = Mutex.new
     
-    # IP address of the Host, as a string.
-    attr_reader :ip
+    def self.clear
+      @@all_hosts_mutex.synchronize {@@all_hosts = {}}
+    end
     
     # We override Host.new so that attempting to create a duplicate Host object
     # (that is, one with the same IP as an existing Host object) returns the
@@ -369,6 +373,7 @@ module Jetpants
     
     # Returns the machine's hostname
     def hostname
+      return 'unknown' unless available?
       @hostname ||= ssh_cmd('hostname').chomp
     end
     

data/lib/jetpants/pool.rb

@@ -128,14 +128,15 @@ module Jetpants
     # Note that a plugin may want to override this (or implement after_remove_slave!)
     # to actually sync the change to an asset tracker, depending on how the plugin
     # implements Pool#sync_configuration. (If the implementation makes sync_configuration
-    # work by iterating over the pool's current slaves, it won't see any slaves that have
-    # been removed.)
+    # work by iterating over the pool's current slaves to update their status/role/pool, it 
+    # won't see any slaves that have been removed, and therefore won't update them.)
     def remove_slave!(slave_db)
       raise "Slave is not in this pool" unless slave_db.pool == self
       slave_db.disable_monitoring
       slave_db.stop_replication
       slave_db.repl_binlog_coordinates # displays how far we replicated, in case you need to roll back this change manually
       slave_db.disable_replication!
+      sync_configuration # may or may not be sufficient -- see note above.
     end
     
     # Informs this pool that it has an alias. A pool may have any number of aliases.
@@ -167,7 +168,9 @@ module Jetpants
           elsif s == @master
             details[s] = {coordinates: s.binlog_coordinates(false), lag: 'N/A'}
           else
-            details[s] = {coordinates: s.repl_binlog_coordinates(false), lag: s.seconds_behind_master.to_s + 's'}
+            lag = s.seconds_behind_master
+            lag_str = lag.nil? ? 'NULL' : lag.to_s + 's'
+            details[s] = {coordinates: s.repl_binlog_coordinates(false), lag: lag_str}
           end
         end
       end

data/lib/jetpants/topology.rb

@@ -9,12 +9,16 @@ module Jetpants
     
     def initialize
       @pools  = [] # array of Pool objects
-      load_pools
+      # We intentionally don't call load_pools here. The caller must do that.
+      # This allows Jetpants module to create Jetpants.topology object, and THEN
+      # invoke load_pools, which might then refer back to Jetpants.topology.
     end
     
     ###### Class methods #######################################################
     
     # Metaprogramming hackery to create a "synchronized" method decorator
+    # Note that all synchronized methods share the same mutex, so don't make one
+    # synchronized method call another!
     @lock = Mutex.new
     @do_sync = false
     @synchronized_methods = {} # symbol => true
@@ -84,7 +88,7 @@ module Jetpants
     end
     
     
-    ###### Accessors ###########################################################
+    ###### Instance Methods ####################################################
     
     # Returns array of this topology's Jetpants::Pool objects of type Jetpants::Shard
     def shards
@@ -140,5 +144,19 @@ module Jetpants
       claim_spares(1, options)[0]
     end
     
+    synchronized
+    # Clears the pool list and nukes cached DB and Host object lookup tables
+    def clear
+      @pools = []
+      DB.clear
+      Host.clear
+    end
+    
+    # Empties and then reloads the pool list
+    def refresh
+      clear
+      load_pools
+      true
+    end
   end
 end

data/plugins/simple_tracker/simple_tracker.rb

@@ -43,9 +43,8 @@ module Jetpants
       end
       
       def determine_pool_and_role(ip, port=3306)
-        ip += ":#{port}" if port.to_i != 3306
-        
-        [@global_pools + @shards].each do |h|
+        ip += ":#{port}"
+        (@global_pools + @shards).each do |h|
           pool = (h['name'] ? Jetpants.topology.pool(h['name']) : Jetpants.topology.shard(h['min_id'], h['max_id']))
           return [pool, 'MASTER'] if h['master'] == ip
           h['slaves'].each do |s|
@@ -57,9 +56,9 @@ module Jetpants
       end
       
       def determine_slaves(ip, port=3306)
-        ip += ":#{port}" if port.to_i != 3306
+        ip += ":#{port}"
         
-        [@global_pools + @shards].each do |h|
+        (@global_pools + @shards).each do |h|
           next unless h['master'] == ip
           return h['slaves'].map {|s| s['host'].to_db}
         end
@@ -71,4 +70,4 @@ module Jetpants
 end
 
 # load all the monkeypatches for other Jetpants classes
-%w(pool shard topology).each {|mod| require "simple_tracker/#{mod}"}
+%w(pool shard topology db).each {|mod| require "simple_tracker/#{mod}"}

metadata

@@ -2,7 +2,7 @@
 name: jetpants
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.7.2
+  version: 0.7.4
 platform: ruby
 authors: 
 - Evan Elias
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-18 00:00:00 Z
+date: 2012-09-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mysql2
@@ -80,7 +80,7 @@ dependencies:
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
-  name: terminal-table
+  name: colored
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
@@ -90,17 +90,6 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: colored
-  prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-  type: :runtime
-  version_requirements: *id008
 description: Jetpants is an automation toolkit for handling monstrously large MySQL database topologies. It is geared towards common operational tasks like cloning slaves, rebalancing shards, and performing master promotions. It features a command suite for easy use by operations staff, though it's also a full Ruby library for use in developing custom migration scripts and database automation.
 email: 
 - me@evanelias.com
@@ -111,40 +100,40 @@ extensions: []
 
 extra_rdoc_files: 
 - README.rdoc
+- doc/plugins.rdoc
 - doc/configuration.rdoc
 - doc/faq.rdoc
-- doc/requirements.rdoc
 - doc/commands.rdoc
-- doc/plugins.rdoc
+- doc/requirements.rdoc
 files: 
 - Gemfile
 - README.rdoc
+- doc/plugins.rdoc
 - doc/configuration.rdoc
 - doc/faq.rdoc
-- doc/requirements.rdoc
 - doc/commands.rdoc
-- doc/plugins.rdoc
-- lib/jetpants/monkeypatch.rb
+- doc/requirements.rdoc
+- lib/jetpants/callback.rb
+- lib/jetpants/topology.rb
+- lib/jetpants/db/server.rb
+- lib/jetpants/db/state.rb
 - lib/jetpants/db/import_export.rb
 - lib/jetpants/db/privileges.rb
 - lib/jetpants/db/client.rb
 - lib/jetpants/db/replication.rb
-- lib/jetpants/db/server.rb
-- lib/jetpants/db/state.rb
-- lib/jetpants/db.rb
 - lib/jetpants/shard.rb
+- lib/jetpants/db.rb
+- lib/jetpants/host.rb
 - lib/jetpants/pool.rb
+- lib/jetpants/monkeypatch.rb
 - lib/jetpants/table.rb
-- lib/jetpants/topology.rb
-- lib/jetpants/callback.rb
-- lib/jetpants/host.rb
 - lib/jetpants.rb
 - bin/jetpants
-- plugins/simple_tracker/db.rb
+- plugins/simple_tracker/topology.rb
 - plugins/simple_tracker/shard.rb
-- plugins/simple_tracker/pool.rb
 - plugins/simple_tracker/simple_tracker.rb
-- plugins/simple_tracker/topology.rb
+- plugins/simple_tracker/db.rb
+- plugins/simple_tracker/pool.rb
 - etc/jetpants.yaml.sample
 homepage: https://github.com/tumblr/jetpants/
 licenses: []