jetpants 0.8.0 → 0.8.2

data/plugins/online_schema_change/topology.rb

@@ -0,0 +1,56 @@
+module Jetpants
+  class Topology
+    
+    # run an alter table on all the sharded pools
+    # if you specify dry run it will run a dry run on all the shards
+    # otherwise it will run on the first shard and ask if you want to
+    # continue on the rest of the shards, 10 shards at a time
+    def alter_table_shards(database, table, alter, dry_run=true)
+      my_shards = shards.dup
+      first_shard = my_shards.shift
+      print "Will run on first shard and prompt for going past the dry run only on the first shard\n\n"
+      print "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n"
+      unless first_shard.alter_table(database, table, alter, dry_run, false)
+        print "First shard had an error, please check output\n"
+        return
+      end
+
+      continue = 'no'
+      continue = ask('First shard complete would you like to continue with the rest of the shards?: (YES/no) - YES has to be in all caps and fully typed')
+      if continue == 'YES'
+        errors = []
+
+        my_shards.limited_concurrent_map(10) do |shard|
+          print "[#{Time.now.to_s.blue}] #{shard.pool.to_s}\n"
+          errors << shard unless shard.alter_table(database, table, alter, dry_run, true)
+        end
+
+        errors.each do |shard|
+          print "check #{shard.name} for errors during online schema change\n"
+        end
+      end
+    end 
+
+    # will drop old table from the shards after a alter table
+    # this is because we do not drop the old table in the osc
+    # also I will do the first shard and ask if you want to
+    # continue, after that it will do each table serially
+    def drop_old_alter_table_shards(database, table)
+      my_shards = shards.dup
+      first_shard = my_shards.shift
+      print "Will run on first shard and prompt before going on to the rest\n\n"
+      print "[#{Time.now.to_s.blue}] #{first_shard.pool.to_s}\n"
+      first_shard.drop_old_alter_table(database, table)
+
+      continue = 'no'
+      continue = ask('First shard complete would you like to continue with the rest of the shards?: (YES/no) - YES has to be in all caps and fully typed')
+      if continue == 'YES'
+        my_shards.each do |shard|
+          print "[#{Time.now.to_s.blue}] #{shard.pool.to_s}\n"
+          shard.drop_old_alter_table(database, table)
+        end
+      end
+    end
+
+  end
+end

data/plugins/simple_tracker/shard.rb

@@ -62,7 +62,7 @@ module Jetpants
         # read-only shards, and offline shards appropriately.
         return me.merge case state
                  when :ready, :needs_cleanup then {'host' => master.ip}
-                 when :child then {'host_read' => master.ip, 'host_write' => parent.master.ip}
+                 when :child then {'host_read' => master.ip, 'host_write' => master.master.ip}
                  when :read_only then {'host_read' => master.ip, 'host_write' => false}
                  when :offline then {'host' => false}
                  end

data/plugins/upgrade_helper/commandsuite.rb

@@ -0,0 +1,212 @@
+# additional commands added by this plugin
+
+require 'thor'
+
+module Jetpants
+  class CommandSuite < Thor
+
+    desc 'upgrade_clone_slave', 'clone a standby slave to target node(s) running a newer version of MySQL'
+    method_option :source, :desc => 'IP of node to clone from'
+    method_option :target, :desc => 'IP of node(s) to clone to'
+    def upgrade_clone_slave
+      puts "This task clones the data set of a standby slave to target node(s) that have a"
+      puts "newer version of MySQL already installed."
+      source = ask_node('Please enter IP of node to clone from: ', options[:source])
+      source.master.probe if source.master # fail early if there are any replication issues in this pool
+      describe source
+      
+      puts "You may clone to particular IP address(es), or can type \"spare\" to claim a node from the spare pool."
+      target = options[:target] || ask('Please enter comma-separated list of targets (IPs or "spare") to clone to: ')
+      spares_needed = target.split(',').count {|t| t.strip.upcase == 'SPARE'}
+      target = 'spare' if target.strip == '' || target.split(',').length == 0
+      if spares_needed > 0
+        spares_available = Jetpants.topology.count_spares(role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version)
+        raise "Not enough upgraded spares with role of standby slave! Requested #{spares_needed} but only have #{spares_available} available." if spares_needed > spares_available
+        claimed_spares = Jetpants.topology.claim_spares(spares_needed, role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version)
+      end
+
+      targets = target.split(',').map do |ip|
+        ip.strip!
+        if is_ip? ip
+          ip.to_db
+        elsif ip == '' || ip.upcase == 'SPARE'
+          claimed_spares.shift
+        else
+          error "target (#{ip}) does not appear to be an IP."
+        end
+      end
+      
+      source.start_mysql if ! source.running?
+      error "source (#{source}) is not a standby slave" unless source.is_standby?
+      
+      targets.each do |t|
+        error "target #{t} already has a master; please clear out node (including in asset tracker) before proceeding" if t.master
+      end
+      
+      # Disable fast shutdown on the source
+      source.mysql_root_cmd 'SET GLOBAL innodb_fast_shutdown = 0'
+      
+      # Flag the nodes as needing upgrade, which will get triggered when
+      # enslave_siblings restarts them
+      targets.each {|t| t.needs_upgrade = true}
+      
+      # Remove ib_lru_dump if present on targets
+      targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_lru_dump"}
+      
+      source.enslave_siblings!(targets)
+      targets.concurrent_each {|t| t.resume_replication; t.catch_up_to_master}
+      source.pool.sync_configuration
+      
+      puts "Clone-and-upgrade complete."
+      Jetpants.topology.write_config
+    end
+    
+    
+    desc 'upgrade_promotion', 'demote and destroy a master running an older version of MySQL'
+    method_option :demote,  :desc => 'node to demote'
+    def upgrade_promotion
+      demoted = ask_node 'Please enter the IP address of the node to demote:', options[:demote]
+      demoted.probe
+      
+      # This task should not be used for emergency promotions (master failures)
+      # since the regular "jetpants promotion" logic is actually fine in that case.
+      error "Unable to connect to node #{demoted} to demote" unless demoted.running?
+      
+      # Before running this task, the pool should already have an extra standby slave,
+      # since we're going to be removing the master from the pool.
+      standby_slaves_needed = Jetpants.standby_slaves_per_pool + 1
+      error "Only run this task on a pool with 3 standby slaves!" unless demoted.pool(true).standby_slaves.size >= standby_slaves_needed
+      
+      # Verify that all nodes except the master are running the same version, and
+      # are higher version than the master
+      unless demoted.slaves.all? {|s| s.version_cmp(demoted.slaves.first) == 0 && s.version_cmp(demoted) > 0}
+        error "This task can only be used when all slaves are running the same version of MySQL,"
+        error "and the master's version is older than that of all the slaves."
+      end
+      
+      puts
+      inform "Summary of affected pool"
+      inform "Binary log positions and slave lag shown below are just a snapshot taken at the current time." if demoted.running?
+      puts
+      demoted.pool(true).summary(true)
+      puts
+      
+      promoted = ask_node 'Please enter the IP address of a standby slave to promote: '
+      
+      error "Node to promote #{promoted} is not a standby slave of node to demote #{demoted}" unless promoted.master == demoted && promoted.role == :standby_slave
+      error "The chosen node cannot be promoted. Please choose another." unless promoted.promotable_to_master?(false)
+      
+      inform "Going to DEMOTE AND DESTROY existing master #{demoted} and PROMOTE new master #{promoted}."
+      error "Aborting." unless agree "Proceed? [yes/no]: "
+      
+      # Perform the promotion, but without making the old master become a slave of the new master
+      # We then rely on the built-in call to Pool#sync_configuration or Pool#after_master_promotion!
+      # to remove the old master from the pool in the same way it would handle a failed master (which
+      # is entirely asset-tracker-plugin specific)
+      demoted.pool(true).master_promotion!(promoted, false)
+    end
+    def self.after_upgrade_promotion
+      reminders(
+        'Commit/push the configuration in version control.',
+        'Deploy the configuration to all machines.',
+      )
+    end
+    
+    
+    desc 'shard_upgrade', 'upgrade a shard via four-step lockless process'
+    method_option :min_id,  :desc => 'Minimum ID of shard to upgrade'
+    method_option :max_id,  :desc => 'Maximum ID of shard to ugprade'
+    method_option :reads,   :desc => 'Move reads to the new master', :type => :boolean
+    method_option :writes,  :desc => 'Move writes to new master', :type => :boolean
+    method_option :cleanup, :desc => 'Tear down the old-version nodes', :type => :boolean
+    def shard_upgrade
+      if options[:reads]
+        raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:writes] || options[:cleanup]
+        s = ask_shard_being_upgraded :reads
+        s.branched_upgrade_move_reads
+        Jetpants.topology.write_config
+        self.class.reminders(
+          'Commit/push the configuration in version control.',
+          'Deploy the configuration to all machines.',
+          'Wait for reads to stop on the old shard master.',
+          'Proceed to next step: jetpants shard_upgrade --writes'
+        )
+      elsif options[:writes]
+        raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:reads] || options[:cleanup]
+        s = ask_shard_being_upgraded :writes
+        s.branched_upgrade_move_writes
+        Jetpants.topology.write_config
+        self.class.reminders(
+          'Commit/push the configuration in version control.',
+          'Deploy the configuration to all machines.',
+          'Wait for writes to stop on the old parent master.',
+          'Proceed to next step: jetpants shard_upgrade --cleanup',
+        )
+        
+      elsif options[:cleanup]
+        raise 'The --reads, --writes, and --cleanup options are mutually exclusive' if options[:reads] || options[:writes]
+        s = ask_shard_being_upgraded :cleanup
+        s.cleanup!
+        
+      else
+        self.class.reminders(
+          'This process may take an hour or two. You probably want to run this from a screen session.',
+          'Be especially careful if you are relying on SSH Agent Forwarding for your root key, since this is not screen-friendly.'
+        )
+        s = ask_shard_being_upgraded :prep
+        s.branched_upgrade_prep
+        self.class.reminders(
+          'Proceed to next step: jetpants shard_upgrade --reads'
+        )
+      end
+    end
+    
+    
+    desc 'checksum_pool', 'Run pt-table-checksum on a pool to verify data consistency after an upgrade of one slave'
+    method_option :pool,  :desc => 'name of pool'
+    def checksum_pool
+      pool_name = options[:pool] || ask('Please enter name of pool to checksum: ')
+      pool = Jetpants.topology.pool(pool_name) or raise "Pool #{pool_name} does not exist"
+      pool.checksum_tables
+    end
+    
+    
+    desc 'check_pool_queries', 'Runs pt-upgrade on a pool to verify query performance and results between different MySQL versions'
+    method_option :pool, :desc => 'name of pool'
+    method_option :dumptime, :desc => 'number of seconds of tcpdump data to consider'
+    def check_pool_queries
+      pool_name = options[:pool] || ask('Please enter name of pool to checksum: ')
+      dump_time = options[:dumptime].to_i if options[:dumptime]
+      dump_time ||= 30
+      
+      pool = Jetpants.topology.pool(pool_name) or raise "Pool #{pool_name} does not exist"
+      pool.collect_and_compare_queries!(dump_time)
+    end
+    
+    no_tasks do
+      def ask_shard_being_upgraded(stage=:prep)
+        shards_being_upgraded = Jetpants.shards.select {|s| [:child, :needs_cleanup].include?(s.state) && !s.parent && s.master.master}
+        if stage == :writes || stage == :cleanup
+          if shards_being_upgraded.size == 0
+            raise 'No shards are currently being upgraded. You can only use this task after running "jetpants shard_upgrade".'
+          elsif shards_being_upgraded.size == 1
+            s = shards_being_upgraded.first
+            puts "Detected #{s} as the only shard currently involved in an upgrade operation."
+            error "Aborting." unless agree "Is this the right shard that you want to perform this action on? [yes/no]: "
+            return s
+          else
+            puts "The following shards are already involved in an upgrade operation:"
+            shards_being_upgraded.each {|sbu| puts "* #{sbu}"}
+          end
+        end
+        puts "Which shard would you like to perform this action on?"
+        shard_min = options[:min_id] || ask('Please enter min ID of the shard: ')
+        shard_max = options[:max_id] || ask('Please enter max ID of the shard: ')
+        s = Jetpants.topology.shard shard_min, shard_max
+        raise 'Shard not found' unless s
+        s
+      end
+    end
+    
+  end
+end

data/plugins/upgrade_helper/db.rb

@@ -0,0 +1,78 @@
+module Jetpants
+  class DB
+    attr_accessor :needs_upgrade
+    
+    ##### CALLBACKS ############################################################
+    
+    # Handle upgrading mysql if needed
+    def before_start_mysql(*options)
+      return unless @needs_upgrade
+      
+      @repl_paused = false if @master
+      running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
+      raise "[#{@ip}] Failed to start MySQL: Something is already listening on port #{@port}" unless running.chomp == '0'
+        
+      output "Attempting to start MySQL with --skip-networking --skip-grant-tables in prep for upgrade"
+      
+      # Can't use start_mysql here without causing infinite recursion! Also don't need
+      # to do all the same checks here, nor do we need to store these to @options.
+      output service(:start, 'mysql', '--skip-networking --skip-grant-tables')
+      
+      output "Attempting to run mysql_upgrade"
+      output ssh_cmd('mysql_upgrade')
+      
+      output "Upgrade complete"
+      @needs_upgrade = false
+      
+      # Now shut down mysql, so that start_mysql can restart it without the --skip-* options
+      stop_mysql
+    end
+    
+    ##### NEW METHODS ##########################################################
+    
+    # Creates a temporary user for use of pt-table-checksum, yields to the 
+    # supplied block, and then drops the user.
+    # The user will have a randomly-generated 50-character password, and will
+    # have elevated permissions (ALL PRIVILEGES on the application schema, and
+    # a few global privs as well) since these are necessary to run the tools.
+    # The block will be passed the randomly-generated password.
+    def with_pt_checksum_user(username='pt-checksum')
+      password = DB.random_password
+      create_user username, password
+      grant_privileges username, '*', 'PROCESS', 'REPLICATION CLIENT',  'REPLICATION SLAVE'
+      grant_privileges username, app_schema, 'ALL PRIVILEGES'
+      begin
+        yield username, password
+      rescue
+        drop_user username
+        raise
+      end
+      drop_user username
+    end
+    
+    # Captures mysql traffic with tcpdump for the specified amount of time, in seconds.
+    # The dumpfile will be saved to #{Jetpants.export_location} with filename 
+    # #{hostname}.dumpfile, and the filename portion will be returned by this method.
+    #
+    # Not all traffic will be included -- uses a method by Devananda van der Veen described in 
+    # http://www.mysqlperformanceblog.com/2011/04/18/how-to-use-tcpdump-on-very-busy-hosts/
+    # to sample the traffic.
+    #
+    # Requires that tcpdump is available in root's PATH. Also assumes root's shell is bash
+    # or supports equivalent syntax. Currently only works if mysqld running on port 3306.
+    #
+    # Warning: tcpdump can be taxing on the server, and also can generate rather large
+    # amounts of output! Also, will overwrite any previous file at the destination path!
+    def tcpdump!(duration=30, interface=false)
+      interface ||= Jetpants.private_interface
+      output "Using tcpdump to capture sample of MySQL traffic for #{duration} seconds"
+      tcpdump_options = "-i #{interface} -s 65535 -x -n -q -tttt 'port 3306 and tcp[1] & 7 == 2 and tcp[3] & 7 == 2'"
+      outfile = "#{Jetpants.export_location}/#{hostname}.dumpfile"
+      ssh_cmd "tcpdump #{tcpdump_options} > #{outfile} & export DUMP_PID=$! && sleep #{duration} && kill $DUMP_PID"
+      output "Completed capturing traffic sample"
+      "#{hostname}.dumpfile"
+    end
+    
+    
+  end
+end

data/plugins/upgrade_helper/host.rb

@@ -0,0 +1,22 @@
+module Jetpants
+  class Host
+    
+    ##### NEW METHODS ##########################################################
+    
+    # Converts tcpdump output into slowlog format using pt-query-digest. Requires that
+    # pt-query-digest is installed and in root's path. Returns the full path to the
+    # slowlog. Does not delete or remove the tcpdump output file.
+    #
+    # This is in Host instead of DB because it may be preferable to run this on
+    # the host running Jetpants, as opposed to the DB where the dumpfile came from,
+    # because pt-query-digest may be taxing to run on the server.
+    def dumpfile_to_slowlog(tcpdump_output_file_path, delete_tcpdumpfile=true)
+      slowlog_file_path = tcpdump_output_file_path.sub('.dumpfile', '') + '.slowlog'
+      ssh_cmd "pt-query-digest #{tcpdump_output_file_path} --type tcpdump --no-report --print >#{slowlog_file_path}"
+      ssh_cmd "rm #{tcpdump_output_file_path}" if delete_tcpdumpfile
+      slowlog_file_path
+    end
+    
+    
+  end
+end

data/plugins/upgrade_helper/pool.rb

@@ -0,0 +1,259 @@
+require 'open3'
+
+module Jetpants
+  class Pool
+    collins_attr_accessor :checksum_running
+    
+    # Runs pt-table-checksum on the pool.
+    # Returns true if no problems found, false otherwise.
+    # If problems were found, the 'checksums' table will be
+    # left in the pool - the user must review and manually delete.
+    def checksum_tables
+      schema = master.app_schema
+      success = false
+      output_lines = []
+      
+      # check if already running, or a previous run died
+      previous_run = collins_checksum_running
+      previous_run = nil if previous_run == ''
+      if previous_run
+        run_data = JSON.parse(previous_run.downcase)  # hash with 'from_host', 'from_pid', 'timestamp'
+        previous_host = run_data['from_host'].to_host
+        previous_pid = run_data['from_pid'] or die 'No previous pid found in previous rundata?'
+        still_running = previous_host.pid_running?(previous_pid, 'pt-table-checksum')
+        raise "Checksum already in progress from #{previous_host}, pid=#{previous_pid}" if still_running
+        output "Previous failed run detected, will use --resume parameter"
+      end
+      
+      # Determine what to pass to --max-load
+      master.output "Polling for normal max threads_running, please wait"
+      max_threads_running = master.max_threads_running
+      limit_threads_running = [(max_threads_running * 1.2).ceil, 50].max
+      master.output "Found max threads_running=#{max_threads_running}, will use limit of #{limit_threads_running}"
+      
+      # Operate with a temporary user that has elevated permissions
+      master.with_pt_checksum_user do |username, password|
+        # Build command line
+        command_line = ['pt-table-checksum',
+          '--no-check-replication-filters',
+          "--databases #{schema}",
+          "--host #{master.ip}",
+          "--port #{master.port}",
+          "--max-load Threads_running:#{limit_threads_running}",
+          "--replicate #{schema}.checksums",
+          "--replicate-database #{schema}",
+          "--user #{username}",
+          "--password #{password}"
+        ].join ' '
+        command_line += ' --resume' if previous_run
+        
+        # Spawn the process
+        Open3.popen3(command_line) do |stdin, stdout, stderr, wait_thread|
+          exit_code = nil
+          pid = wait_thread.pid
+          puts "Running pt-table-checksum targetting #{master}, pid on Jetpants host is #{pid}"
+        
+          self.collins_checksum_running = {
+            'from_host' => Host.local.ip,
+            'from_pid'  => pid,
+            'timestamp' => Time.now.to_i,
+          }.to_json
+        
+          # Display STDERR output in real-time, via a separate thread
+          Thread.new do
+            begin
+              stderr.each {|line| puts line}
+            rescue IOError, Interrupt
+              nil
+            end
+          end
+          
+          # Capture STDOUT and buffer it; since this is the main thread, also
+          # watch out for broken pipe or ctrl-c
+          begin
+            stdout.each {|line| output_lines << line}
+            exit_code = wait_thread.value.to_i
+          rescue IOError, Interrupt => ex
+            puts "Caught exception #{ex.message}"
+            exit_code = 130  # by unix convention, return 128 + SIGINT
+          end
+          
+          # Dump out stdout: first anything we buffered on our end, plus anything
+          # that Perl or the OS had buffered on its end
+          puts
+          output_lines.each {|line| puts line}
+          unless stdout.eof?
+            stdout.each {|line| puts line} rescue nil
+          end
+          puts
+          
+          puts "Checksum completed with exit code #{exit_code}.\n"
+          success = (exit_code == 0)
+          
+          # Run again with --replicate-check-only to display ALL diffs, including ones from
+          # prior runs of the tool.
+          puts 'Verifying all results via --replicate-check-only...'
+          output, diff_success = `#{command_line} --replicate-check-only`, $?.success?
+          if diff_success
+            puts 'No diffs found in any tables.'
+            puts output
+          else
+            puts 'Found diffs:'
+            puts output
+            success = false
+          end
+          
+          # Drop the checksums table, but only if there were no diffs
+          if success
+            output "Dropping table #{schema}.checksums..."
+            master.connect(user: username, pass: password)
+            master.query('DROP TABLE checksums')
+            output "Table dropped."
+            master.disconnect
+            self.collins_checksum_running = ''
+          else
+            output 'Keeping checksums table in place for your review.'
+            output 'Please manually drop it when done.'
+          end
+          puts
+        end # popen3
+      end # with_pt_checksum_user
+      success
+    end
+    
+    
+    # Uses pt-upgrade to compare query performance and resultsets among nodes
+    # in a pool. Supply params:
+    # * a full path to a slowlog file
+    # * a boolean indicating whether or not you want to do an initial silent
+    #   run (results discarded) to populate the buffer pools on the nodes
+    # * Two or more nodes, or no nodes if you want to default to using the
+    #   pool's standby slaves
+    #
+    # Requires that pt-upgrade is in root's PATH on the node running Jetpants.
+    def compare_queries(slowlog_path, silent_run_first, *compare_nodes)
+      if compare_nodes.size == 0
+        compare_nodes = standby_slaves
+      else
+        compare_nodes.flatten!
+        raise "Supplied nodes must all be in this pool" unless compare_nodes.all? {|n| n == master || n.master == master}
+      end
+      
+      # We need to create a temporary SUPER user on the nodes to compare
+      # Also attempt to silence warning 1592 about unsafe-for-replication statements if
+      # using Percona Server 5.5.10+ which supports this.
+      username = 'pt-upgrade'
+      password = DB.random_password
+      remove_suppress_1592 = []
+      compare_nodes.each do |node|
+        node.create_user username, password
+        node.grant_privileges username, '*', 'SUPER'
+        node.grant_privileges username, node.app_schema, 'ALL PRIVILEGES'
+        
+        # We only want to try this if (a) the node supports log_warnings_suppress,
+        # and (b) the node isn't already suppressing warning 1592
+        if node.global_variables[:log_warnings_suppress] == ''
+          node.mysql_root_cmd "SET GLOBAL log_warnings_suppress = '1592'"
+          remove_suppress_1592 << node
+        end
+      end
+      
+      node_text = compare_nodes.map {|s| s.to_s + ' (v' + s.normalized_version(3) + ')'}.join ' vs '
+      dsn_text = compare_nodes.map {|n| "h=#{n.ip},P=#{n.port},u=#{username},p=#{password},D=#{n.app_schema}"}.join ' '
+      
+      # Do silent run if requested (to populate buffer pools)
+      if silent_run_first
+        output "Doing a silent run of pt-upgrade with slowlog #{slowlog_path} to populate buffer pool."
+        output "Comparing nodes #{node_text}..."
+        stdout, exit_code = `pt-upgrade --set-vars wait_timeout=10000 #{slowlog_path} #{dsn_text} 2>&1`, $?.to_i
+        output "pt-upgrade silent run completed with exit code #{exit_code}"
+        puts
+        puts
+      end
+      
+      # Run pt-upgrade for real. Note that we only compare query times and results, NOT warnings,
+      # due to issues with warning 1592 causing a huge amount of difficult-to-parse output.
+      output "Running pt-upgrade with slowlog #{slowlog_path}"
+      output "Comparing nodes #{node_text}..."
+      stdout, exit_code = `pt-upgrade --set-vars wait_timeout=10000 --compare query_times,results #{slowlog_path} #{dsn_text} 2>&1`, $?.to_i
+      output stdout
+      puts
+      output "pt-upgrade completed with exit code #{exit_code}"
+      
+      # Drop the SUPER user and re-enable logging of warning 1592
+      compare_nodes.each {|node| node.drop_user username}
+      remove_suppress_1592.each {|node| node.mysql_root_cmd "SET GLOBAL log_warnings_suppress = ''"}
+    end
+    
+    
+    # Collects query slowlog on the master (and one active slave, if there are any)
+    # using tcpdump, copies over to the host Jetpants is running on, converts to a
+    # slowlog, and then uses Pool#compare_queries to run pt-upgrade.
+    #
+    # The supplied *compare_nodes should be standby slaves, and you may omit them
+    # to automatically select two standby slaves (of different versions, if available)
+    # 
+    # When comparing exactly two nodes, we stop replication on the nodes temporarily
+    # to ensure a consistent dataset for comparing query results. Otherwise, async
+    # replication can naturally result in false-positives.
+    def collect_and_compare_queries!(tcpdump_time=30, *compare_nodes)
+      # Sample traffic and convert to slowlog for master
+      master_dump_filename = master.tcpdump!(tcpdump_time)
+      local = Host.local # node where we're running Jetpants from
+      local.ssh_cmd "mkdir -p #{Jetpants.export_location}"
+      master.fast_copy_chain(Jetpants.export_location, local, files: master_dump_filename, overwrite: true)
+      master.ssh_cmd "rm #{Jetpants.export_location}/#{master_dump_filename}"
+      master_slowlog_path = local.dumpfile_to_slowlog("#{Jetpants.export_location}/#{master_dump_filename}")
+      
+      # If we also have an active slave running, grab sampled slowlog from there too
+      active_slowlog_path = nil
+      if active_slaves.size > 0
+        active_slave = active_slaves.first
+        active_dump_filename = active_slave.tcpdump!(tcpdump_time)
+        active_slave.fast_copy_chain(Jetpants.export_location, local, files: active_dump_filename, overwrite: true)
+        active_slave.ssh_cmd "rm #{Jetpants.export_location}/#{active_dump_filename}"
+        active_slowlog_path = local.dumpfile_to_slowlog("#{Jetpants.export_location}/#{active_dump_filename}")
+      end
+      
+      # Gather our comparison nodes
+      if compare_nodes.size == 0
+        higher_ver_standby = standby_slaves.select {|s| s.version_cmp(master) > 0}.first
+        same_ver_standby = standby_slaves.select {|s| s.version_cmp(master) == 0}.first
+        if higher_ver_standby && same_ver_standby
+          compare_nodes = [same_ver_standby, higher_ver_standby]
+        else
+          compare_nodes = standby_slaves[0, 2]
+        end
+      end
+      
+      # Disable monitoring on our comparison nodes, and then stop replication
+      # at the same position. We only proceed with this if we're comparing
+      # exactly two nodes; this may be improved in a future release.
+      if compare_nodes.size == 2
+        compare_nodes.each {|n| n.disable_monitoring}
+        compare_nodes.first.pause_replication_with(compare_nodes.last)
+      end
+      
+      # Run pt-upgrade using the master dumpfile
+      puts
+      output "COMPARISON VIA QUERY LOG FROM MASTER"
+      compare_queries(master_slowlog_path, true, *compare_nodes)
+      
+      if active_slowlog_path
+        puts
+        output "COMPARISON VIA QUERY LOG FROM ACTIVE SLAVE"
+        compare_queries(active_slowlog_path, true, *compare_nodes)
+      end
+      
+      # If we previously paused replication and disabled monitoring, un-do this
+      if compare_nodes.size == 2
+        compare_nodes.concurrent_each do |n| 
+          n.resume_replication
+          n.catch_up_to_master
+          n.enable_monitoring
+        end
+      end
+    end
+    
+  end
+end