jetpants 0.8.0 → 0.8.2

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA1: 
+  metadata.gz: 942f23ed6af75e4ee9536e0833bdb5ff733ee947
+  data.tar.gz: 9938e5eccd30352d14a8871ea7aa28513413d100
+SHA512: 
+  metadata.gz: 42ab57cd08c826101ed1c3416e50bd6f50c48f00cbd8120fd6c6db521a23e0e66f66d7b8f5fcf5041d757f1531dcdb67cb2c273efdc8354b5df9e15284518b69
+  data.tar.gz: ed6db33c0b814c284e59e5528c8e456dbe699b4657b5dc25ec4bcbe77eaf5202eee29396a08f69ae1f79520eb87d8e98d7351dc6ce5afb5c83813c4d52d25b8e

data/README.rdoc

@@ -8,16 +8,11 @@
 
 == MOTIVATION:
 
-\Jetpants was created by {Tumblr}[http://www.tumblr.com/] to help manage our database infrastructure. It handles automation tasks for our entire database topology, which as of October 2012 consists of approximately:
-* 200 dedicated database servers
-* 5 global (unsharded) functional pools 
-* 58 shard pools
-* 28 terabytes total of unique relational data on masters
-* 100 billion total unique relational rows on masters
+\Jetpants was created by {Tumblr}[http://www.tumblr.com/] to help manage our database infrastructure. It handles automation tasks for our entire database topology, which as of May 2013 consists of over 215 dedicated database servers and nearly 200 billion total distinct relational rows.
 
-One of the primary requirements for \Jetpants was speed. On our hardware, <b>\Jetpants can divide a 750GB, billion-row shard in half in about six hours</b> -- or even faster if you're diving into thirds or fourths. It can also <b>clone slaves at line speed on gigabit ethernet</b>, including to multiple destinations at once, using a novel "chained copy" approach.
+One of the primary requirements for \Jetpants was speed. On our hardware, <b>\Jetpants can divide a 750GB, billion-row shard in half in about six hours</b> -- or even faster if you're diving into thirds or fourths.
 
-For more background on the initial motivations behind \Jetpants, please see {Evan Elias's presentation at Percona Live NYC 2012}[https://github.com/tumblr/jetpants/blob/master/doc/PerconaLiveNYC2012Presentation.pdf?raw=true].
+For more background on the motivations behind \Jetpants, please see {Evan Elias's presentation at Percona Live 2013}[https://github.com/tumblr/jetpants/blob/master/doc/PerconaLive2013Presentation.pdf?raw=true].
 
 == COMMAND SUITE FEATURES:
 
@@ -76,7 +71,7 @@ If you have a question that isn't covered here, please feel free to email the au
 
 == CREDITS:
 
-* <b>Evan Elias</b>: Lead developer
+* <b>Evan Elias</b>: Creator and developer
 * <b>Dallas Marlow</b>: Developer
 * <b>Bob Patterson Jr</b>: Developer
 * <b>Tom Christ</b>: Developer

data/bin/jetpants

@@ -286,7 +286,7 @@ module Jetpants
       end
       raise "Aborting" unless ask('Please type YES in all capital letters to confirm removing node from its pool: ') == 'YES'
       node.pool.remove_slave!(node)
-      node.revoke_all_access!
+      node.revoke_all_access! if node.running? && node.available?
     end
     
     
@@ -367,10 +367,9 @@ module Jetpants
         end
         children = supply_ranges.count
         raise "Supplied range does not cover parent completely!" if supply_ranges.first[0] != shard_min.to_i || supply_ranges.last[1] != shard_max.to_i
-        s.init_children(children, supply_ranges)
-        s.split!
+        s.split!(children, supply_ranges)
       else
-        children = options[:count]  || ask('Optionally enter how many children to split into [default=2]: ')
+        children = options[:count] || ask('Optionally enter how many children to split into [default=2]: ')
         children = 2 if children == ''
         s.split!(children.to_i)
       end
@@ -450,7 +449,7 @@ module Jetpants
       raise "Specified cutover ID is too low!" unless cutover_id > last_shard.min_id
       
       # Ensure enough spare nodes are available before beginning.
-      # We supply the *previous* last shard as context for counting/claiming spares
+      # We supply the *previous* last shard as context for counting spares
       # because the new last shard can't be created yet (chicken-and-egg problem -- master
       # must exist before we create the pool). The assumption is the hardware spec
       # of the new last shard and previous last shard will be the same.
@@ -472,7 +471,9 @@ module Jetpants
       new_last_shard_master = Jetpants.topology.claim_spare(role: :master, like: last_shard_master)
       new_last_shard_master.disable_read_only! if new_last_shard_master.running?
       if Jetpants.standby_slaves_per_pool > 0
-        new_last_shard_slaves = Jetpants.topology.claim_spares(Jetpants.standby_slaves_per_pool, role: :standby_slave, like: last_shard_master)
+        # Verify spare count again, now that we can actually supply the new master as the :like context
+        raise "Not enough standby_slave role spare machines!" unless Jetpants.topology.count_spares(role: :standby_slave, like: new_last_shard_master) >= Jetpants.standby_slaves_per_pool
+        new_last_shard_slaves = Jetpants.topology.claim_spares(Jetpants.standby_slaves_per_pool, role: :standby_slave, like: new_last_shard_master)
         new_last_shard_slaves.each do |x| 
           x.change_master_to new_last_shard_master
           x.resume_replication

data/doc/capacity_plan.rdoc

@@ -0,0 +1,77 @@
+= capacity_plan
+
+== OVERVIEW:
+
+This \Jetpants plugin gathers and processes space usage data from your machines and then creates a report with the categories 'Usage and time left', 'Day over day usage', 'Data outliers', 'Hardware status'.
+
+== CONFIGURATION:
+
+For this plugin the configuration is multi step.
+
+First you want to run the following create table statment on a database you would like to store your capacity data
+
+    CREATE TABLE `storage` (
+      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+      `timestamp` int(11) NOT NULL,
+      `pool` varchar(255) NOT NULL,
+      `total` BIGINT UNSIGNED NOT NULL,
+      `used` BIGINT UNSIGNED NOT NULL,
+      `available` BIGINT UNSIGNED NOT NULL,
+      `db_sizes` BIGINT UNSIGNED NOT NULL,
+      PRIMARY KEY (`id`),
+      INDEX (`timestamp`)
+    ) ENGINE=InnoDB;
+
+Next you want to fill out the \Jetpants configuration file (either <tt>/etc/jetpants.yaml</tt> or <tt>~/.jetpants.yaml</tt>). For example you configuration might look like this:
+
+    # ... rest of Jetpants config here
+    
+    plugins:
+        capacity_plan:
+            critical_mount: 0.85
+            warning_mount: 0.80
+            pool_name: platform
+            user: jetpants_cap
+            schema: capacity_plan
+            pass: xxxxxxxxxxxxxxx
+        # ... other plugins configured here
+
+critical_mount::    the disk used percent of when the machine will be marked as critical
+
+warning_mount::     the disk used percent of when the machine will be marked as warning
+
+pool_name::         pool you will be writing too for the historical data
+
+user::              user to connect to mysql with for historical data
+
+schema::            table name for historical data
+
+pass::              password for user for historical data
+
+
+Next you want to is create a cron to capture the historical data
+
+    0 * * * * /your_bin_path/jetpants capacity_snapshot 2>&1 > /dev/null
+
+Then you want create a cron that will email you the report everyday (if you want that)
+    
+    0 10 * * * /your_bin_path/jetpants capacity_plan --email=your_email@example.com 2>&1 > /dev/null
+
+If you want the hardware stats part of the email you have to create a function in Jetpants.topology.machine_status_counts that returns a hash that will be used to output the email
+
+== ASSUMPTIONS AND REQUIREMENTS:
+
+Use of this plugin assumes that you are using \Collins and the jetpants_collins plugin
+
+Also you should have the pony gem installed
+
+== USAGE:
+
+If you want to run the capacity plan you can do 
+    
+    jetpants capacity_plan
+
+To capture a one off snapshot of your data usage
+    
+    jetpants capacity_snapshot
+

data/doc/commands.rdoc

@@ -75,7 +75,7 @@ Or if you're deploying a brand new pool in an existing topology:
 5. Stop replicating writes from the parent shard, and then take the parent pool offline entirely.
 6. Remove rows that replicated to the wrong child shard. This data will be sparse, since it's only the writes that were made since the shard split process started.
 
-For more information, including diagrams of each step, please see {our presentation at Percona Live NYC 2012}[https://github.com/tumblr/jetpants/blob/master/doc/PerconaLiveNYC2012Presentation.pdf?raw=true].
+For more information, including diagrams of each step, please see {our presentation at Percona Live  2013}[https://github.com/tumblr/jetpants/blob/master/doc/PerconaLive2013Presentation.pdf?raw=true].
 
 Separately, \Jetpants also allows you to alter the range of the last shard in your topology. In a range-based sharding scheme, the last shard has a range of X to infinity; eventually this will be too large of a range, so you need to truncate that shard range and create a "new" last shard after it.  We call this process "shard cutover".
 

data/doc/jetpants_collins.rdoc

@@ -38,9 +38,10 @@ This plugin also makes some assumptions about the way in which you use \Collins,
 * All MySQL database server hosts that are in-use will have a STATUS of either ALLOCATED or MAINTENANCE.
 * All MySQL database server hosts that are in-use will have a POOL set matching the name of their pool/shard, and a SECONDARY_ROLE set matching their \Jetpants role within the pool (MASTER, ACTIVE_SLAVE, STANDBY_SLAVE, or BACKUP_SLAVE).
 * You can initially assign PRIMARY_ROLE, STATUS, POOL, and SECONDARY_ROLE to database servers somewhat automatically; see GETTING STARTED, below.
-* All database server hosts that are "spares" (not yet in use, but ready for use in shard splits, shard cutover, or slave cloning) need to have a STATUS of PROVISIONED. These nodes must meet the requirements of spares as defined by the REQUIREMENTS doc that comes with \Jetpants. They should NOT have a POOL or SECONDARY_ROLE set in advance; if they do, it will be ignored -- we treat all spares as identical. That said, you can implement custom logic to respect POOL or SECONDARY_ROLE (or any other Collins attribute) by overriding Topology#process_spare_selector_options in a custom plugin loaded after jetpants_collins.
+* All database server hosts that are "spares" (not yet in use, but ready for use in shard splits, shard cutover, or slave cloning) need to have a STATUS of ALLOCATED and a STATE OF SPARE. These nodes must meet the requirements of spares as defined by the REQUIREMENTS doc that comes with \Jetpants. They should NOT have a POOL or SECONDARY_ROLE set in advance; if they do, it will be ignored -- we treat all spares as identical. That said, you can implement custom logic to respect POOL or SECONDARY_ROLE (or any other Collins attribute) by overriding Topology#process_spare_selector_options in a custom plugin loaded after jetpants_collins.
 * Database server hosts may optionally have an attribute called SLAVE_WEIGHT. The default weight, if omitted, is 100. This field has no effect in \Jetpants, but can be used by your custom configuration generator as needed, if your application supports a notion of different weights for slave selection.
 * Arbitrary metadata regarding pools and shards will be stored in assets with a TYPE of CONFIGURATION. These assets will have a POOL matching the pool's name, a TAG matching the pool's name but prefixed with 'mysql-', a STATUS reflecting the pool's state, and a PRIMARY_ROLE of either MYSQL_POOL or MYSQL_SHARD depending on the type of pool. You can make jetpants_collins create these automatically; see GETTING STARTED, below.
+* Your jetpants user in Collins must also have permission to set state in collins otherwise you will have to manually create the STATUS:ALLOCATED STATE:SPARE and STATE:ALLOCATED STATE:CLAIMED
 
 Please note that jetpants_collins does not generate application configuration files, because every web app/framework uses a different format. You will need to write a custom plugin to generate a configuration file for your application as needed, by overriding the Topology#write_config method.
 

data/doc/online_schema_change.rdoc

@@ -0,0 +1,45 @@
+= online_schema_change
+
+== OVERVIEW:
+
+This \Jetpants plugin allows the use of pt-online-schema-change in the \Jetpants suite of tools.  With this plugin there will be checks to see if there is already an online schema change going on.  Also you can run a online schema change against all your shards.
+
+== CONFIGURATION:
+
+This plugin has no extra options, just add the name to your plugins section and that is it.
+
+To enable this plugin, add it to your \Jetpants configuration file (either <tt>/etc/jetpants.yaml</tt> or <tt>~/.jetpants.yaml</tt>). For example you configuration might look like this:
+
+    # ... rest of Jetpants config here
+    
+    plugins:
+        online_schema_change:
+        # ... other plugins configured here
+
+== ASSUMPTIONS AND REQUIREMENTS:
+
+Use of this plugin assumes that you already have pt-online-schema-change installed
+
+Also you should be using \Collins and the jetpants_collins plugin
+
+== EXAMPLES:
+
+dry run of an alter on a single pool
+    jetpants alter_table  --database=allmydata  --table=somedata --pool=users --dry-run --alter='ADD COLUMN c1 INT' 
+
+alter a single pool
+    jetpants alter_table  --database=allmydata  --table=somedata --pool=users --alter='ADD COLUMN c1 INT'
+
+dry run of an alter of all the shards of your topology
+    jetpants alter_table  --database=allmydata  --table=somedata --all_shards --dry-run --alter='ADD COLUMN c1 INT'
+
+alter all the shards of your topology
+    jetpants alter_table  --database=allmydata  --table=somedata --all_shards --alter='ADD COLUMN c1 INT'
+
+
+the alter table does not drop the old table automatically, so to remove the table on a sigle pool
+    jetpants alter_table_drop --database=allmydata  --table=somedata --pool=users
+
+to drop the tables on all your shards
+    jetpants alter_table_drop --database=allmydata  --table=somedata --all_shards
+

data/doc/plugins.rdoc

@@ -104,4 +104,10 @@ Callbacks may interrupt the control flow by raising a Jetpants::CallbackAbortErr
 
 * If this is raised in a before_foo method, all lower-priority before_foo stacked callbacks will be skipped, as will the call to foo and any after_foo callbacks. The exception is automatically caught and is not fatal. The return value of foo will be nil.
 
-* If this is raised in an after_foo method, all subsequent lower-priority after_foo stacked callbacks will be skipped. The return value of foo will be unaffected.
+* If this is raised in an after_foo method, all subsequent lower-priority after_foo stacked callbacks will be skipped. The return value of foo will be unaffected.
+
+== Other bundled plugins
+
+=== upgrade_helper
+
+This plugin adds additional Jetpants commands to aid in performing a major MySQL upgrade. For full documentation, please see upgrade_helper.rdoc ({view on GitHub}[https://github.com/tumblr/jetpants/blob/master/doc/upgrade_helper.rdoc]).

data/doc/requirements.rdoc

@@ -64,4 +64,4 @@ By default, the <tt>standby_slaves_per_pool</tt> config option is set to 2. This
 
 A "spare" machine in \Jetpants should be in a clean-slate state: MySQL should be installed and have the proper grants and root password, but there should be no data on these machines, and they should not be slaving. \Jetpants will set up replication appropriately when it assigns the nodes to their appropriate pools.
 
-For more information on the shard split process implemented by \Jetpants, including diagrams of each stage of the process, please see {Evan Elias's presentation at Percona Live NYC 2012}[https://github.com/tumblr/jetpants/blob/master/doc/PerconaLiveNYC2012Presentation.pdf?raw=true], starting at slide 20.
+For more information on the shard split process implemented by \Jetpants, including diagrams of each stage of the process, please see {Evan Elias's presentation at Percona Live 2013}[https://github.com/tumblr/jetpants/blob/master/doc/PerconaLive2013Presentation.pdf?raw=true], starting at slide 38.

data/doc/upgrade_helper.rdoc

@@ -0,0 +1,68 @@
+= upgrade_helper
+
+== OVERVIEW:
+
+This is a plugin to aid in performing a major MySQL upgrade, such as from 5.1 to 5.6. It includes the following functionality:
+
+* New Jetpants command to clone a lower-version standby slave to a higher-version spare, and perform the upgrade process on the spare
+* New Jetpants commands to do a master promotion that evicts the final lower-version node from a pool. You can choose to do either a standard locking master promotion (with read_only period), or on shards you may do a promotion that "ejects" the old master in a multi-step process without any locking.
+* New Jetpants commands that wrap two Percona Toolkit tools (pt-upgrade and pt-query-digest) to verify the upgrade process did not cause any data drift or major performance degradations.
+
+== CONFIGURATION:
+
+This plugin has only one configuration option, which is mandatory:
+
+new_version::         major.minor string of MySQL version being upgraded to, for example "5.5". Be sure to wrap in quotes so that it is not misinterpreted as a float. (required)
+
+Example usage:
+
+    # ... rest of Jetpants config here
+    
+    plugins:
+        jetpants_collins:
+            # config for jetpants_collins here
+            
+        upgrade_helper:
+            new_version: "5.5"
+        
+        # ... other plugins configured here
+
+
+== ASSUMPTIONS AND REQUIREMENTS:
+
+This plugin currently requires that you are using jetpants_collins plugin and \Collins as your asset tracker. This may be made more generic in a future release.
+
+This plugin never upgrades nodes in-place; instead, it operates on spare nodes. The plugin must have the ability to grab spare nodes which already have the newer version of MySQL installed, as opposed to regular spares which have your old/existing version. In order for this to be possible, a custom plugin loaded AFTER jetpants_collins must override its method Topology#process_spare_selector_options to support these options:
+
+:version::  if supplied, will be a major.minor string ('5.1', '5.5', etc). Spares should use the exact specified version.
+:like:: if supplied, will be a Jetpants::DB. Spares should use the same version as this node, unless :version was also supplied, in which case :like is ignored and :version takes precedence.
+
+In order to use new commands "jetpants checksum_pool" and "jetpants upgrade_check_pool", Percona Toolkit must be installed on the same machine as Jetpants, and be in your PATH. This plugin has been tested most extensively with Percona Toolkit 2.1.7. It may require some modifications to full work with Percona Toolkit 2.2.x, since pt-upgrade has been completely rewritten.
+
+In order to use "jetpants upgrade_check_pool", tcpdump must be installed on all database nodes, and be in root's PATH.
+
+== USAGE:
+
+=== Upgrading functional partitions
+
+1. Use "jetpants upgrade_clone_slave" to clone an existing (older-MySQL-version) standby to a machine that already has your newer version of MySQL. This task will then run mysql_upgrade properly on the node to complete the upgrade process.
+2. Use "jetpants check_pool_queries" to verify query performance. This will collect a read query log from the master (and from an active slave, if any exist) and replay them against an older-version and newer-version standby slave using pt-upgrade. We automatically run pt-upgrade twice, ignoring the results the first time. The first run is to help populate the buffer pool.
+3. After the upgraded slave has been replicating for some time (we recommend 24 hours), use "jetpants checksum_pool" to detect data drift on the upgraded slave. This command uses pt-table-checksum.
+4. Use "jetpants clone_slave" repeatedly to clone the upgraded slave as needed. If the pool has active slaves, you will eventually want to use "jetpants activate_slave" and "jetpants pull_slave" to convert upgraded standbys to actives, and vice versa to pull out the older-version actives. Then use "jetpants destroy_slave" as needed to eliminate older-version standby slaves. Your end result of this step should be that all nodes except the master are upgraded, and you have one extra upgraded standby slave in the pool. (So if you normally have N standbys per pool, you should now have N+1 instead; and all slaves -- standby or active -- are running the newer version of MySQL.)
+5. Use "jetpants upgrade_promotion" to perform a master promotion. This is identical a normal "jetpants promotion" except that eliminates the old master, instead of enslaving it. This is necessary because older-version nodes cannot have a higher-version master.
+
+Repeat this entire process for each functional partition.
+
+
+=== Upgrading shards
+
+The above process for functional partitions also works perfectly fine for shards. The first time you upgrade a shard, you should use the above process, including the steps to verify query performance and confirm lack of data drift.
+
+For subsequent shard upgrades, you may optionally use this simplified process.
+
+1. Use "jetpants shard_upgrade" to build new upgraded standby slaves of the shard, in a hierarchical replication setup. One new standby slave will directly replicate from the master, and the others will be slaves of that one. In total, if a shard in your environment normally has N standby slaves, this command will create N+1 upgraded standby slaves. (It's essentially creating an upgraded mirror of the shard, except the mirror's master is actually a slave of the true top-level master.)
+2. Use "jetpants shard_upgrade --reads" to regenerate your application configuration in a way that moves read queries to the upgraded mirror shard's master, but keeps write queries going to the true master. (Your custom app config generator plugin must already do this for shards in the :child state, in order for "jetpants shard_split_child_reads" to work.)
+3. Use "jetpants shard_upgrade --writes" to regenerate your application configuration in a way that moves read AND write queries to the upgraded mirror shard's master.
+4. Use "jetpants shard_upgrade --cleanup" to eject all non-upgraded nodes from the pool entirely. This will tear down replication between the version of the shard and the old version.
+
+Using a custom Ruby script, this process can be automated to perform each step on several shards at once.

data/lib/jetpants/db/client.rb

@@ -67,6 +67,7 @@ module Jetpants
         :after_connect    =>  options[:after_connect] )
       @user = options[:user]
       @schema = options[:schema]
+      @db.convert_tinyint_to_bool = false
       @db
     end
     
@@ -147,4 +148,4 @@ module Jetpants
     end
     
   end
-end
+end

data/lib/jetpants/db/import_export.rb

@@ -192,13 +192,18 @@ module Jetpants
     # MUCH faster than doing a single count, but far more I/O intensive, so
     # don't use this on a master or active slave.
     def row_counts(tables, min_id, max_id)
+      tables = [tables] unless tables.is_a? Array
       lock = Mutex.new
       row_count = {}
       tables.each do |t|
         row_count[t.name] = 0
-        (min_id..max_id).in_chunks(t.chunks, 10) do |min, max|
-          result = query_return_first_value(t.sql_count_rows(min, max))
-          lock.synchronize {row_count[t.name] += result}
+        if min_id && max_id && t.chunks > 1
+          (min_id..max_id).in_chunks(t.chunks, Jetpants.max_concurrency) do |min, max|
+            result = query_return_first_value(t.sql_count_rows(min, max))
+            lock.synchronize {row_count[t.name] += result}
+          end
+        else
+          row_count[t.name] = query_return_first_value(t.sql_count_rows(false, false))
         end
         output "#{row_count[t.name]} rows counted", t
       end
@@ -246,6 +251,10 @@ module Jetpants
           id = query_return_first_value(finder_sql)
           break unless id
           rows_deleted += query(deleter_sql, id)
+          
+          # Slow down on multi-col sharding key tables, due to queries being far more expensive
+          sleep(0.0001) if table.sharding_keys.size > 1
+          
           iter += 1
           output("#{dir_english} deletion progress: through #{col} #{id}, deleted #{rows_deleted} rows so far", table) if iter % 50000 == 0
         end

data/lib/jetpants/db/replication.rb

@@ -22,7 +22,7 @@ module Jetpants
       logfile = option_hash[:log_file]
       pos     = option_hash[:log_pos]
       if !(logfile && pos)
-        raise "Cannot use coordinates of a new master that is receiving updates" if new_master.master && ! new_master.repl_paused
+        raise "Cannot use coordinates of a new master that is receiving updates" if new_master.master && ! new_master.repl_paused?
         logfile, pos = new_master.binlog_coordinates
       end
       
@@ -126,10 +126,12 @@ module Jetpants
       repl_user ||= replication_credentials[:user]
       repl_pass ||= replication_credentials[:pass]
       disable_monitoring
+      targets.each {|t| t.disable_monitoring}
       pause_replication if master && ! @repl_paused
       file, pos = binlog_coordinates
       clone_to!(targets)
-      targets.each do |t| 
+      targets.each do |t|
+        t.enable_monitoring
         t.change_master_to( self, 
                             log_file: file, 
                             log_pos:  pos, 
@@ -148,10 +150,12 @@ module Jetpants
     def enslave_siblings!(targets)
       raise "Can only call enslave_siblings! on a slave instance" unless master
       disable_monitoring
+      targets.each {|t| t.disable_monitoring}
       pause_replication unless @repl_paused
       file, pos = repl_binlog_coordinates
       clone_to!(targets)
       targets.each do |t| 
+        t.enable_monitoring
         t.change_master_to( master, 
                             log_file: file,
                             log_pos:  pos,

data/lib/jetpants/db/schema.rb

@@ -0,0 +1,40 @@
+require 'table'
+
+module Jetpants
+
+  #--
+  # Table probing methods
+  #++
+
+  class DB
+    # Query the database for information about the table schema, including
+    # primary key, create statement, and columns
+    def detect_table_schema(table_name)
+      table_sql = "SHOW CREATE TABLE `#{table_name}`"
+      create_statement = query_return_first(table_sql).values.last
+      pk_sql = "SHOW INDEX IN #{table_name} WHERE Key_name = 'PRIMARY'"
+      pk_fields = query_return_array(pk_sql)
+      pk_fields.sort_by!{|pk| pk[:Seq_in_index]}
+
+      params = {
+        'primary_key' => pk_fields.map{|pk| pk[:Column_name] },
+        'create_table' => create_statement,
+        'indexes' => connection.indexes(table_name),
+        'pool' => pool,
+        'columns' => connection.schema(table_name).map{|schema| schema[0]} 
+      }
+
+      Table.new(table_name, params)
+    end
+
+    # Deletages check for a table existing by name up to the pool
+    def has_table?(table)
+      pool.has_table? table
+    end
+
+    # List of tables (as defined by the pool)
+    def tables
+      pool(true).tables
+    end
+  end
+end

data/lib/jetpants/db/server.rb

@@ -11,7 +11,7 @@ module Jetpants
       output "Attempting to shutdown MySQL"
       disconnect if @db
       output service(:stop, 'mysql')
-      running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
+      running = ssh_cmd "netstat -ln | grep ':#{@port}' | wc -l"
       raise "[#{@ip}] Failed to shut down MySQL: Something is still listening on port #{@port}" unless running.chomp == '0'
       @options = []
       @running = false
@@ -26,7 +26,7 @@ module Jetpants
       if @master
         @repl_paused = options.include?('--skip-slave-start')
       end
-      running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
+      running = ssh_cmd "netstat -ln | grep ':#{@port}' | wc -l"
       raise "[#{@ip}] Failed to start MySQL: Something is already listening on port #{@port}" unless running.chomp == '0'
       if options.size == 0
         output "Attempting to start MySQL, no option overrides supplied"