jetpants 0.7.0

data/lib/jetpants/callback.rb

@@ -0,0 +1,131 @@
+module Jetpants
+  # Exception class used to halt further processing in callback chain.  See
+  # description in CallbackHandler.
+  class CallbackAbortError < StandardError; end
+  
+  # If you include CallbackHandler as a mix-in, it grants the base class support
+  # for Jetpants callbacks, as defined here:
+  #
+  # If you invoke a method "foo", Jetpants will first 
+  # automatically call any "before_foo" methods that exist in the class or its
+  # superclasses. You can even define multiple methods named before_foo (in the
+  # same class!) and they will each be called. In other words, Jetpants
+  # callbacks "stack" instead of overriding each other.
+  # 
+  # After calling any/all before_foo methods, the foo method is called, followed
+  # by all after_foo methods in the same manner.
+  #
+  # If any before_foo method raises a CallbackAbortError, subsequent before_foo
+  # methods will NOT be called, NOR will foo itself nor any after_foo methods.
+  #
+  # If any after_foo method raises a CallbackAbortError, subsequent after_foo
+  # methods will NOT be called.
+  #
+  # You may preceed the definition of a callback method with "callback_priority 123"
+  # to set an explicit priority (higher = called first) for subsequent callbacks.
+  # The default priority is 100.
+  module CallbackHandler
+    def self.included(base)
+      base.class_eval do
+        class << self
+          # Set the priority (higher = called first) for any subsequent callbacks defined in the current class.
+          def callback_priority(value)
+            @callback_priority = value
+          end
+
+          def method_added(name)
+            # Intercept before_* and after_* methods and create corresponding Callback objects
+            if name.to_s.start_with? 'before_', 'after_'
+              Callback.new self, name.to_s.split('_', 2)[1].to_sym, name.to_s.split('_', 2)[0].to_sym, @callback_priority
+            
+            # Intercept redefinitions of methods we've already wrapped, so we can
+            # wrap them again
+            elsif Callback.wrapped? self, name
+              Callback.wrap_method self, name
+            end
+          end
+        end
+        
+        # Default priority for callbacks is 100
+        @callback_priority = 100
+      end
+    end
+  end
+  
+  # Generic representation of a before-method or after-method callback.
+  # Used internally by CallbackHandler; you won't need to interact with Callback directly.
+  class Callback
+    @@all_callbacks = {}        # hash of class obj -> method_name symbol -> type string -> array of callbacks
+    @@currently_wrapping = {}   # hash of class obj -> method_name symbol -> bool
+    
+    attr_reader :for_class    # class object
+    attr_reader :method_name  # symbol containing method name (the one being callback-wrapped)
+    attr_reader :type         # :before or :after
+    attr_reader :priority     # high numbers get triggered first
+    attr_reader :my_alias     # method name alias OF THE CALLBACK
+    
+    def initialize(for_class, method_name, type=:after, priority=100)
+      @for_class = for_class
+      @method_name = method_name
+      @type = type
+      @priority = priority
+      
+      @@all_callbacks[for_class] ||= {}
+      @@all_callbacks[for_class][method_name] ||= {}
+      already_wrapped = Callback.wrapped?(for_class, method_name)
+      @@all_callbacks[for_class][method_name][type] ||= []
+
+      next_method_id = @@all_callbacks[for_class][method_name][type].count + 1
+      old_name = "#{type.to_s}_#{method_name.to_s}".to_sym
+      @my_alias = new_name = ("real_callback_#{old_name}_" + for_class.to_s.sub('::', '_') + "_#{next_method_id}").to_sym
+      for_class.class_eval do
+        alias_method new_name, old_name
+      end
+      Callback.wrap_method(for_class, method_name) unless already_wrapped
+      
+      @@all_callbacks[for_class][method_name][type] << self
+    end
+    
+    def self.wrap_method(for_class, method_name)
+      @@currently_wrapping[for_class] ||= {}
+      @@currently_wrapping[for_class][method_name] ||= false
+      return if @@currently_wrapping[for_class][method_name] # prevent infinite recursion from the alias_method call
+      @@currently_wrapping[for_class][method_name] = true
+      
+      for_class.class_eval do
+        alias_method "#{method_name}_without_callbacks".to_sym, method_name
+        define_method method_name do |*args|
+          begin
+            Callback.trigger(self, method_name, :before, *args)
+          rescue CallbackAbortError
+            return
+          end
+          result = send "#{method_name}_without_callbacks".to_sym, *args
+          begin
+            Callback.trigger(self, method_name, :after, *args)
+          rescue CallbackAbortError
+          end
+          result
+        end
+      end
+      
+      @@currently_wrapping[for_class][method_name] = false
+    end
+    
+    def self.trigger(for_object, method_name, type, *args)
+      my_callbacks = []
+      for_object.class.ancestors.each do |for_class|
+        if @@all_callbacks[for_class] && @@all_callbacks[for_class][method_name] && @@all_callbacks[for_class][method_name][type]
+          my_callbacks.concat(@@all_callbacks[for_class][method_name][type])
+        end
+      end
+      my_callbacks.sort_by! {|c| -1 * c.priority}
+      my_callbacks.each {|c| for_object.send(c.my_alias, *args)}
+    end
+    
+    def self.wrapped?(for_class, method_name)
+      return false unless @@all_callbacks[for_class] && @@all_callbacks[for_class][method_name]
+      @@all_callbacks[for_class][method_name].count > 0
+    end
+  end
+end

data/lib/jetpants/db.rb

@@ -0,0 +1,122 @@
+require 'sequel'
+require 'json'
+
+module Jetpants
+  
+  # A Jetpants::DB is a specific mysql instance running on a particular IP and port.
+  # It also contains a Jetpants::Host object corresponding to the IP; any missing
+  # method calls get delegated to the Host.
+  #
+  # This class has been split across several files due to its size.  Please see
+  # lib/jetpants/db/*.rb for the bulk of its logic, which has been divided along
+  # functional lines.
+  class DB
+    include CallbackHandler
+    
+    # IP address (as a string) of the MySQL instance
+    attr_reader :ip
+    
+    # Port number of the MySQL instance. The base Jetpants implementation only supports
+    # port 3306, since this is necessary to crawl a replication hierarchy using SHOW
+    # PROCESSLIST, which does not include slave port numbers. However, plugins may
+    # override this behavior to support nonstandard ports and multi-instance-per-host
+    # topologies.
+    attr_reader :port
+    
+    # Jetpants::Host object that this MySQL instance runs on.
+    attr_reader :host
+    
+    # We keep track of DB instances to prevent DB.new from every returning
+    # duplicates.
+    @@all_dbs = {}
+    @@all_dbs_mutex = Mutex.new
+    
+    # Because this class is rather large, methods have been grouped together
+    # and moved to separate files in lib/jetpants/db. We load these all now.
+    # They each just re-open the DB class and add some methods.
+    Dir[File.join File.dirname(__FILE__), 'db', '*'].each {|f| require f}
+    
+    # We override DB.new so that attempting to create a duplicate DB object
+    # (that is, one with the same IP and port as an existing DB object)
+    # returns the original object.
+    def self.new(ip, port=3306)
+      ip, embedded_port = ip.split(':', 2)
+      port = embedded_port.to_i if embedded_port
+      addr = "#{ip}:#{port}"
+      @@all_dbs_mutex.synchronize do
+        @@all_dbs[addr] = nil unless @@all_dbs[addr].is_a? self
+        @@all_dbs[addr] ||= super
+      end
+    end
+    
+    def initialize(ip, port=3306)
+      @ip, @port = ip, port.to_i
+      @user = false # connections will default to app user
+      @master = nil
+      @slaves = nil
+      @repl_paused = nil
+      @running = nil
+      @host = Host.new(ip)
+    end
+    
+    
+    ###### Host methods ########################################################
+    
+    # Jetpants::DB delegates missing methods to its Jetpants::Host.
+    def method_missing(name, *args, &block)
+      if @host.respond_to? name
+        @host.send name, *args, &block
+      else
+        super
+      end
+    end
+    
+    # Alters respond_to? logic to account for delegation of missing methods
+    # to the instance's Host.
+    def respond_to?(name, include_private=false)
+      super || @host.respond_to?(name)
+    end
+    
+    # Returns true if the supplied Jetpants::DB is on the same Jetpants::Host
+    # as self.
+    def same_host_as?(db)
+      @ip == db.ip
+    end
+    
+    ###### Misc methods ########################################################
+    
+    # Displays the provided output, along with information about the current time,
+    # self, and optionally a Jetpants::Table name.
+    def output(str, table=nil)
+      str = str.to_s.strip
+      str = nil if str && str.length == 0
+      str ||= "Completed (no output)"
+      output = Time.now.strftime("%H:%M:%S") + " [#{self}] "
+      output << table.name << ': ' if table
+      output << str
+      print output + "\n"
+      output
+    end
+    
+    # DB objects are sorted as strings, ie, by calling to_s
+    def <=> other
+      to_s <=> other.to_s
+    end
+    
+    # Returns a string in the form "ip:port"
+    def to_s
+      "#{@ip}:#{@port}"
+    end
+    
+    # Returns self, since self is already a Jetpants::DB.
+    def to_db
+      self
+    end
+    
+    # Returns the instance's Jetpants::Host.
+    def to_host
+      @host
+    end
+    
+  end
+end

data/lib/jetpants/db/client.rb

@@ -0,0 +1,103 @@
+module Jetpants
+  
+  #--
+  # Connection and query methods ###############################################
+  #++
+  
+  class DB
+    # Runs the provided SQL statement as root, and returns the response as a single string.
+    # Available options:
+    # :terminator:: how to terminate the query, such as '\G' or ';'. (default: '\G')
+    # :parse:: parse a single-row, vertical-format result (:terminator must be '\G') and return it as a hash
+    # :attempts:: by default, queries will be attempted up to 3 times. set this to 0 or false for non-idempotent queries.
+    def mysql_root_cmd(cmd, options={})
+      terminator = options[:terminator] || '\G'
+      attempts = (options[:attempts].nil? ? 3 : (options[:attempts].to_i || 1))
+      failures = 0
+      
+      begin
+        raise "MySQL is not running" unless running?
+        supply_root_pw = (Jetpants.mysql_root_password ? "-p#{Jetpants.mysql_root_password}" : '')
+        supply_port = (@port == 3306 ? '' : "-h 127.0.0.1 -P #{@port}")
+        real_cmd = %Q{mysql #{supply_root_pw} #{supply_port} -ss -e "#{cmd}#{terminator}" #{Jetpants.mysql_schema}}
+        real_cmd.untaint
+        result = ssh_cmd!(real_cmd)
+        raise result if result && result.downcase.start_with?('error ')
+        result = parse_vertical_result(result) if options[:parse] && terminator == '\G'
+        return result
+      rescue => ex
+        failures += 1
+        raise if failures >= attempts
+        output "Root query \"#{cmd}\" failed: #{ex.message}, re-trying after delay"
+        sleep 3 * failures
+        retry
+      end
+    end
+    
+    # Returns a Sequel database object
+    def mysql
+      return @db if @db
+      @db = Sequel.connect(
+        :adapter          =>  'mysql2',
+        :host             =>  @ip,
+        :port             =>  @port,
+        :user             =>  @user || Jetpants.app_credentials[:user],
+        :password         =>  Jetpants.app_credentials[:pass],
+        :database         =>  Jetpants.mysql_schema,
+        :max_connections  =>  Jetpants.max_concurrency)
+    end
+    alias init_db_connection_pool mysql
+    
+    # Closes existing mysql connection pool and opens a new one. Useful when changing users.
+    # Supply a new user name as the param, or nothing/false to keep old user name, or
+    # a literal true value to switch to the default app user in Jetpants configuration
+    def reconnect(new_user=false)
+      @user = (new_user == true ? Jetpants.app_credentials[:user] : new_user) if new_user
+      if @db
+        @db.disconnect rescue nil
+        @db = nil
+      end
+      init_db_connection_pool
+    end
+    
+    # Execute a write (INSERT, UPDATE, DELETE, REPLACE, etc) query.
+    # If the query is an INSERT, returns the last insert ID (if an auto_increment
+    # column is involved).  Otherwise returns the number of affected rows.
+    def query(sql, *binds)
+      ds = mysql.fetch(sql, *binds)
+      mysql.execute_dui(ds.update_sql) {|c| return c.last_id > 0 ? c.last_id : c.affected_rows}
+    end
+    
+    # Execute a read (SELECT) query. Returns an array of hashes.
+    def query_return_array(sql, *binds)
+      mysql.fetch(sql, *binds).all
+    end
+    
+    # Execute a read (SELECT) query. Returns a hash of the first row only.
+    def query_return_first(sql, *binds)
+      mysql.fetch(sql, *binds).first
+    end
+    
+    # Execute a read (SELECT) query. Returns the value of the first column of the first row only.
+    def query_return_first_value(sql, *binds)
+      mysql.fetch(sql, *binds).single_value
+    end
+    
+    # Parses the result of a MySQL query run with a \G terminator. Useful when
+    # interacting with MySQL via the command-line client (for secure access to
+    # the root user) instead of via the MySQL protocol.
+    def parse_vertical_result(text)
+      results = {}
+      return results unless text
+      raise text.chomp if text =~ /^ERROR/
+      lines = text.split("\n")
+      lines.each do |line|
+        col, val = line.split ':'
+        next unless val
+        results[col.strip.downcase.to_sym] = val.strip
+      end
+      results
+    end
+    
+  end
+end

data/lib/jetpants/db/import_export.rb

@@ -0,0 +1,330 @@
+module Jetpants
+  
+  #--
+  # Import, export, and data set methods #######################################
+  #++
+  
+  class DB
+    # Exports the DROP TABLE + CREATE TABLE statements for the given tables via mysqldump
+    def export_schemata(tables)
+      output 'Exporting table definitions'
+      supply_root_pw = (Jetpants.mysql_root_password ? "-p#{Jetpants.mysql_root_password}" : '')
+      supply_port = (@port == 3306 ? '' : "-h 127.0.0.1 -P #{@port}")
+      cmd = "mysqldump #{supply_root_pw} #{supply_port} -d #{Jetpants.mysql_schema} " + tables.join(' ') + " >#{Jetpants.export_location}/create_tables_#{@port}.sql"
+      cmd.untaint
+      result = ssh_cmd(cmd)
+      output result
+    end
+  
+    # Executes a .sql file previously created via export_schemata.
+    # Warning: this will DESTROY AND RECREATE any tables contained in the file.
+    # DO NOT USE ON A DATABASE THAT CONTAINS REAL DATA!!! This method doesn't
+    # check first! The statements will replicate to any slaves! PROCEED WITH
+    # CAUTION IF RUNNING THIS MANUALLY!
+    def import_schemata!
+      output 'Dropping and re-creating table definitions'
+      result = mysql_root_cmd "source #{Jetpants.export_location}/create_tables_#{@port}.sql", :terminator => ''
+      output result
+    end
+    
+    # Has no built-in effect. Plugins can override this and/or use before_alter_schemata
+    # and after_alter_schemata callbacks to provide an implementation.
+    # Also sometimes useful to override this as a singleton method on specific DB objects
+    # in a migration script.
+    def alter_schemata
+    end
+    
+    # Exports data for the supplied tables. If min/max ID supplied, only exports
+    # data where at least one of the table's sharding keys falls within this range.
+    # Creates a 'jetpants' db user with FILE permissions for the duration of the
+    # export.
+    def export_data(tables, min_id=false, max_id=false)
+      pause_replication if @master && ! @repl_paused
+      import_export_user = 'jetpants'
+      create_user(import_export_user)
+      grant_privileges(import_export_user)               # standard privs
+      grant_privileges(import_export_user, '*', 'FILE')  # FILE global privs
+      reconnect(import_export_user)
+      @counts ||= {}
+      tables.each {|t| @counts[t.name] = export_table_data t, min_id, max_id}
+    ensure
+      reconnect(true) # switches back to default app user
+      drop_user(import_export_user)
+    end
+    
+    # Exports data for a table. Only includes the data subset that falls
+    # within min_id and max_id. The export files will be located according
+    # to the export_location configuration setting.
+    # Returns the number of rows exported.
+    def export_table_data(table, min_id=false, max_id=false)
+      unless min_id && max_id && table.chunks > 0
+        output "Exporting all data", table
+        rows_exported = query(table.sql_export_all)
+        output "#{rows_exported} rows exported", table
+        return rows_exported
+      end
+      
+      output "Exporting data for ID range #{min_id}..#{max_id}", table
+      lock = Mutex.new
+      rows_exported = 0
+      chunks_completed = 0
+      
+      (min_id..max_id).in_chunks(table.chunks) do |min, max|
+        attempts = 0
+        begin
+          sql = table.sql_export_range(min, max)
+          result = query sql
+          lock.synchronize do
+            rows_exported += result
+            chunks_completed += 1
+            percent_finished = 100 * chunks_completed / table.chunks
+            output("Export #{percent_finished}% complete.", table) if table.chunks >= 40 && chunks_completed % 20 == 0
+          end
+        rescue => ex
+          if attempts >= 10
+            output "EXPORT ERROR: #{ex.message}, chunk #{min}-#{max}, giving up", table
+            raise
+          end
+          attempts += 1
+          output "EXPORT ERROR: #{ex.message}, chunk #{min}-#{max}, attempt #{attempts}, re-trying after delay", table
+          ssh_cmd("rm -f " + table.export_file_path(min, max))
+          sleep(1.0 * attempts)
+          retry
+        end
+      end
+      output "#{rows_exported} rows exported", table
+      rows_exported
+    end
+    
+    # Imports data for a table that was previously exported using export_data. 
+    # Only includes the data subset that falls within min_id and max_id.  If
+    # run after export_data (in the same process), import_data will 
+    # automatically confirm that the import counts match the previous export
+    # counts.
+    # Creates a 'jetpants' db user with FILE permissions for the duration of the
+    # import.
+    def import_data(tables, min_id=false, max_id=false)
+      import_export_user = 'jetpants'
+      create_user(import_export_user)
+      grant_privileges(import_export_user)               # standard privs
+      grant_privileges(import_export_user, '*', 'FILE')  # FILE global privs
+      reconnect(import_export_user)
+      
+      import_counts = {}
+      tables.each {|t| import_counts[t.name] = import_table_data t, min_id, max_id}
+      
+      # Verify counts
+      @counts ||= {}
+      @counts.each do |name, exported|
+        if exported == import_counts[name]
+          output "Verified import count matches export count for table #{name}"
+        else
+          raise "Import count (#{import_counts[name]}) does not match export count (#{exported}) for table #{name}"
+        end
+      end
+      
+    ensure
+      reconnect(true) # switches back to default app user
+      drop_user(import_export_user)
+    end
+    
+    # Imports the data subset previously dumped thorugh export_data.
+    # Returns number of rows imported.
+    def import_table_data(table, min_id=false, max_id=false)
+      unless min_id && max_id && table.chunks > 0
+        output "Importing all data", table
+        rows_imported = query(table.sql_import_all)
+        output "#{rows_imported} rows imported", table
+        return rows_imported
+      end
+      
+      output "Importing data for ID range #{min_id}..#{max_id}", table
+      lock = Mutex.new
+      rows_imported = 0
+      chunks_completed = 0
+      
+      (min_id..max_id).in_chunks(table.chunks) do |min, max|
+        attempts = 0
+        begin
+          sql = table.sql_import_range(min, max)
+          result = query sql
+          lock.synchronize do
+            rows_imported += result
+            chunks_completed += 1
+            percent_finished = 100 * chunks_completed / table.chunks
+            output("Import #{percent_finished}% complete.", table) if table.chunks >= 40 && chunks_completed % 20 == 0
+            chunk_file_name = table.export_file_path(min, max)
+            ssh_cmd "rm -f #{chunk_file_name}"
+          end
+        rescue => ex
+          if attempts >= 10
+            output "IMPORT ERROR: #{ex.message}, chunk #{min}-#{max}, giving up", table
+            raise
+          end
+          attempts += 1
+          output "IMPORT ERROR: #{ex.message}, chunk #{min}-#{max}, attempt #{attempts}, re-trying after delay", table
+          sleep(3.0 * attempts)
+          retry
+        end
+      end
+      output "#{rows_imported} rows imported", table
+      rows_imported
+    end
+    
+    # Counts rows falling between min_id and max_id for the supplied tables.
+    # Returns a hash mapping table names to counts.
+    # Note: runs 10 concurrent queries to perform the count quickly. This is
+    # MUCH faster than doing a single count, but far more I/O intensive, so
+    # don't use this on a master or active slave.
+    def row_counts(tables, min_id, max_id)
+      lock = Mutex.new
+      row_count = {}
+      tables.each do |t|
+        row_count[t.name] = 0
+        (min_id..max_id).in_chunks(t.chunks, 10) do |min, max|
+          result = query_return_first_value(t.sql_count_rows(min, max))
+          lock.synchronize {row_count[t.name] += result}
+        end
+        output "#{row_count[t.name]} rows counted", t
+      end
+      row_count
+    end
+    
+    # Cleans up all rows that should no longer be on this db.
+    # Supply the ID range (in terms of the table's sharding key)
+    # of rows to KEEP.
+    def prune_data_to_range(tables, keep_min_id, keep_max_id)
+      reconnect(true)
+      tables.each do |t|
+        output "Cleaning up data, pruning to only keep range #{keep_min_id}-#{keep_max_id}", t
+        rows_deleted = 0
+        [:asc, :desc].each {|direction| rows_deleted += delete_table_data_outside_range(t, keep_min_id, keep_max_id, direction)}
+        output "Done cleanup; #{rows_deleted} rows deleted", t
+      end
+    end
+    
+    # Helper method used by prune_data_to_range. Deletes data for the given table that falls
+    # either below the supplied keep_min_id (if direction is :desc) or falls above the 
+    # supplied keep_max_id (if direction is :asc).
+    def delete_table_data_outside_range(table, keep_min_id, keep_max_id, direction)
+      rows_deleted = 0
+      
+      if direction == :asc
+        dir_english = "Ascending"
+        boundary = keep_max_id
+        output "Removing rows with ID > #{boundary}", table
+      elsif direction == :desc
+        dir_english = "Descending"
+        boundary = keep_min_id
+        output "Removing rows with ID < #{boundary}", table
+      else
+        raise "Unknown order parameter #{order}"
+      end
+      
+      table.sharding_keys.each do |col|
+        deleter_sql = table.sql_cleanup_delete(col, keep_min_id, keep_max_id)
+        
+        id = boundary
+        iter = 0
+        while id do
+          finder_sql = table.sql_cleanup_next_id(col, id, direction)
+          id = query_return_first_value(finder_sql)
+          break unless id
+          rows_deleted += query(deleter_sql, id)
+          iter += 1
+          output("#{dir_english} deletion progress: through #{col} #{id}, deleted #{rows_deleted} rows so far", table) if iter % 50000 == 0
+        end
+      end
+      rows_deleted
+    end
+    
+    # Exports and re-imports data for the specified tables, optionally bounded by the
+    # given range. Useful for defragmenting a node. Also useful for doing fast schema
+    # alterations, if alter_schemata (or its callbacks) has been implemented.
+    #
+    # You can omit all params for a shard, in which case the method will use the list
+    # of sharded tables in the Jetpants config file, and will use the shard's min and
+    # max ID.
+    def rebuild!(tables=false, min_id=false, max_id=false)
+      raise "Cannot rebuild an active node" unless is_standby? || for_backups?
+      
+      p = pool
+      if p.is_a?(Shard)
+        tables ||= Table.from_config 'sharded_tables'
+        min_id ||= p.min_id
+        max_id ||= p.max_id if p.max_id != 'INFINITY'
+      end
+      raise "No tables supplied" unless tables && tables.count > 0
+      
+      disable_monitoring
+      stop_query_killer
+      disable_binary_logging
+      restart_mysql
+      reconnect
+      pause_replication if is_slave?
+      
+      # Automatically detect missing min/max. Assumes that all tables' primary keys
+      # are on the same scale, so this may be non-ideal, but better than just erroring.
+      unless min_id
+        tables.each do |t|
+          my_min = query_return_first_value "SELECT MIN(#{t.sharding_keys[0]}) FROM #{t.name}"
+          min_id = my_min if !min_id || my_min < min_id
+        end
+      end
+      unless max_id
+        @found_max_ids = {} # we store the detected maxes in case DB#alter_schemata needs them later
+        tables.each do |t|
+          my_max = @found_max_ids[t.name] = query_return_first_value("SELECT MAX(#{t.sharding_keys[0]}) FROM #{t.name}")
+          max_id = my_max if !max_id || my_max > max_id
+        end
+      end
+      
+      export_schemata tables
+      export_data tables, min_id, max_id
+      import_schemata!
+      alter_schemata if respond_to? :alter_schemata
+      import_data tables, min_id, max_id
+
+      resume_replication if is_slave?
+      enable_binary_logging
+      restart_mysql
+      catch_up_to_master
+      start_query_killer
+      enable_monitoring
+    end
+    
+    # Returns the data set size in bytes (if in_gb is false or omitted) or in gigabytes
+    # (if in_gb is true).  Note that this is actually in gibibytes (2^30) rather than
+    # a metric gigabyte.  This puts it on the same scale as the output to tools like 
+    # "du -h" and "df -h".
+    def data_set_size(in_gb=false)
+      bytes = dir_size("#{mysql_directory}/#{Jetpants.mysql_schema}")
+      in_gb ? (bytes / 1073741824.0).round : bytes
+    end
+    
+    # Copies mysql db files from self to one or more additional DBs.
+    # WARNING: temporarily shuts down mysql on self, and WILL OVERWRITE CONTENTS
+    # OF MYSQL DIRECTORY ON TARGETS.  Confirms first that none of the targets
+    # have over 100MB of data in the schema directory or in ibdata1.
+    # MySQL is restarted on source and targets afterwards. 
+    def clone_to!(*targets)
+      targets.flatten!
+      raise "Cannot clone an instance onto its master" if master && targets.include?(master)
+      destinations = {}
+      targets.each do |t| 
+        destinations[t] = t.mysql_directory
+        existing_size = t.data_set_size + t.dir_size("#{t.mysql_directory}/ibdata1")
+        raise "Over 100 MB of existing MySQL data on target #{t}, aborting copy!" if existing_size > 100000000
+      end
+      [self, targets].flatten.concurrent_each {|t| t.stop_query_killer; t.stop_mysql}
+      targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_logfile*"}
+      fast_copy_chain(mysql_directory, 
+                      destinations,
+                      port: 3306,
+                      files: ['ibdata1', 'mysql', 'test', Jetpants.mysql_schema],
+                      overwrite: true)
+      [self, targets].flatten.concurrent_each {|t| t.start_mysql; t.start_query_killer}
+    end
+    
+  end
+end