jetpants 0.8.0 → 0.8.2

data/plugins/upgrade_helper/shard.rb

@@ -0,0 +1,61 @@
+module Jetpants
+  class Shard
+    # Builds a set of upgraded slaves, and then makes one of the new slaves become the
+    # master for the other new slaves
+    def branched_upgrade_prep
+      raise "Shard #{self} in wrong state to perform this action! expected :ready, found #{@state}" unless @state == :ready
+      raise "Not enough standby slaves of this shard!" unless standby_slaves.size >= Jetpants.standby_slaves_per_pool
+      source = standby_slaves.last
+      spares_available = Jetpants.topology.count_spares(role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version)
+      raise "Not enough spares available!" unless spares_available >= 1 + Jetpants.standby_slaves_per_pool
+      
+      targets = Jetpants.topology.claim_spares(1 + Jetpants.standby_slaves_per_pool, role: :standby_slave, like: source, version: Plugin::UpgradeHelper.new_version)
+      
+      # Disable fast shutdown on the source
+      source.mysql_root_cmd 'SET GLOBAL innodb_fast_shutdown = 0'
+      
+      # Flag the nodes as needing upgrade, which will get triggered when
+      # enslave_siblings restarts them
+      targets.each {|t| t.needs_upgrade = true}
+      
+      # Remove ib_lru_dump if present on targets
+      targets.concurrent_each {|t| t.ssh_cmd "rm -rf #{t.mysql_directory}/ib_lru_dump"}
+      
+      source.enslave_siblings!(targets)
+      targets.concurrent_each {|t| t.resume_replication; t.catch_up_to_master}
+      source.pool.sync_configuration
+      
+      # Make the 1st new slave be the "future master" which the other new
+      # slaves will replicate from
+      future_master = targets.shift
+      targets.each do |t|
+        future_master.pause_replication_with t
+        t.change_master_to future_master
+        [future_master, t].each {|db| db.resume_replication; db.catch_up_to_master}
+      end
+    end
+    
+    # Hack the pool configuration to send reads to the new master, but still send
+    # writes to the old master (they'll replicate over)
+    def branched_upgrade_move_reads
+      raise "Shard #{self} in wrong state to perform this action! expected :ready, found #{@state}" unless @state == :ready
+      future_master = nil
+      slaves.each do |s|
+        future_master = s if s.version_cmp(@master) == 1 && s.slaves.size == Jetpants.standby_slaves_per_pool
+      end
+      raise "Shard #{self} does not have correct hierarchical replication setup to proceed" unless future_master
+      @master = future_master
+      @state = :child
+      sync_configuration
+    end
+    
+    # Move writes over to the new master
+    def branched_upgrade_move_writes
+      raise "Shard #{self} in wrong state to perform this action! expected :child, found #{@state}" unless @state == :child
+      @master.disable_read_only!
+      @state = :needs_cleanup
+      sync_configuration
+    end
+    
+  end
+end

data/plugins/upgrade_helper/upgrade_helper.rb

@@ -0,0 +1,21 @@
+# Entrypoint for upgrade_helper plugin
+
+module Jetpants
+  module Plugin
+    module UpgradeHelper
+      class << self
+        # Shortcut for returning the configured new_version value
+        def new_version
+          Jetpants.plugins['upgrade_helper']['new_version']
+        end
+      end
+    end
+  end
+end
+
+
+# Verify mandatory config options
+raise "No new_version specified in plugin config!" unless Jetpants::Plugin::UpgradeHelper.new_version
+
+# load all the monkeypatches for other Jetpants classes
+%w(pool shard host db commandsuite).each {|mod| require "upgrade_helper/#{mod}"}

data/scripts/global_rowcount.rb

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+# This is a script to count all rows in your database topology. It
+# performs concurrent chunked counts on standby slaves, and can serve
+# as an example of writing scripts using Jetpants as a library.
+
+require 'jetpants'
+
+# Count on shards first
+sharded_tables = Jetpants::Table.from_config 'sharded_tables'
+sharded_counts = Jetpants.shards.limited_concurrent_map(8) do |p|
+  node = p.standby_slaves.last
+  node.stop_query_killer
+  node.output "Starting counts for #{p}"
+  my_max = p.max_id
+  if my_max == 'INFINITY'
+    maxes = sharded_tables.map do |t|
+      node.query_return_first_value "SELECT MAX(#{t.sharding_keys.first}) FROM #{t.name}"
+    end
+    my_max = maxes.map(&:to_i).max
+  end
+  counts = node.row_counts(sharded_tables, p.min_id, my_max)
+  p.output "Found counts for #{p}: #{counts}"
+  node.start_query_killer
+  counts
+end
+
+total_counts = {'rollup' => 0}
+sharded_counts.each do |cnt|
+  cnt.each do |table_name, row_count|
+    total_counts[table_name] ||= 0
+    total_counts[table_name] += row_count
+    total_counts['rollup'] += row_count
+  end
+end
+
+# Count on functional partitions next
+global_counts = Jetpants.functional_partitions.limited_concurrent_map(8) do |p|
+  begin
+    node = p.standby_slaves.last
+    node.stop_query_killer
+    node.output "Starting counts for #{p}"
+    counts = {}
+    p.tables.each do |t|
+      min, max = false, false
+      if t.first_pk_col
+        vals = node.query_return_first "SELECT MIN(#{t.first_pk_col}) AS minval, MAX(#{t.first_pk_col}) AS maxval FROM #{t.name}"
+        min, max = vals[:minval], vals[:maxval]
+        if max.to_i - min.to_i > 100
+          t.chunks = 100
+        else
+          min, max = false, false
+        end
+      end
+      counts.merge! node.row_counts(t, min, max)
+    end
+    p.output "Found counts for #{p}: #{counts}"
+    node.start_query_killer
+    counts
+  rescue => ex
+    p.output "Unable to obtain counts for #{p} -- #{ex.message}"
+    node.start_query_killer
+    {}
+  end
+end
+
+global_counts.each do |cnt|
+  cnt.each do |table_name, row_count|
+    total_counts[table_name] ||= 0
+    total_counts[table_name] += row_count
+    total_counts['rollup'] += row_count
+  end
+end
+
+puts total_counts

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jetpants
 version: !ruby/object:Gem::Version 
-  prerelease: 
-  version: 0.8.0
+  version: 0.8.2
 platform: ruby
 authors: 
 - Evan Elias
@@ -13,13 +12,12 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-03-12 00:00:00 Z
+date: 2013-05-17 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mysql2
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -30,7 +28,6 @@ dependencies:
   name: sequel
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -41,7 +38,6 @@ dependencies:
   name: net-ssh
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -52,7 +48,6 @@ dependencies:
   name: pry
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -63,7 +58,6 @@ dependencies:
   name: thor
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -74,7 +68,6 @@ dependencies:
   name: highline
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -85,7 +78,6 @@ dependencies:
   name: colored
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -96,7 +88,6 @@ dependencies:
   name: collins_client
   prerelease: false
   requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -116,26 +107,33 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 - doc/configuration.rdoc
+- doc/upgrade_helper.rdoc
+- doc/capacity_plan.rdoc
 - doc/faq.rdoc
 - doc/commands.rdoc
 - doc/requirements.rdoc
 - doc/jetpants_collins.rdoc
 - doc/plugins.rdoc
+- doc/online_schema_change.rdoc
 files: 
 - Gemfile
 - README.rdoc
 - doc/configuration.rdoc
+- doc/upgrade_helper.rdoc
+- doc/capacity_plan.rdoc
 - doc/faq.rdoc
 - doc/commands.rdoc
 - doc/requirements.rdoc
 - doc/jetpants_collins.rdoc
 - doc/plugins.rdoc
+- doc/online_schema_change.rdoc
 - lib/jetpants/shard.rb
 - lib/jetpants/table.rb
 - lib/jetpants/db.rb
 - lib/jetpants/pool.rb
 - lib/jetpants/db/privileges.rb
 - lib/jetpants/db/state.rb
+- lib/jetpants/db/schema.rb
 - lib/jetpants/db/import_export.rb
 - lib/jetpants/db/server.rb
 - lib/jetpants/db/client.rb
@@ -146,6 +144,7 @@ files:
 - lib/jetpants/host.rb
 - lib/jetpants.rb
 - bin/jetpants
+- scripts/global_rowcount.rb
 - plugins/jetpants_collins/jetpants_collins.rb
 - plugins/jetpants_collins/shard.rb
 - plugins/jetpants_collins/db.rb
@@ -153,16 +152,32 @@ files:
 - plugins/jetpants_collins/asset.rb
 - plugins/jetpants_collins/topology.rb
 - plugins/jetpants_collins/host.rb
+- plugins/capacity_plan/commandsuite.rb
+- plugins/capacity_plan/monkeypatch.rb
+- plugins/capacity_plan/capacity_plan.rb
+- plugins/upgrade_helper/commandsuite.rb
+- plugins/upgrade_helper/shard.rb
+- plugins/upgrade_helper/db.rb
+- plugins/upgrade_helper/upgrade_helper.rb
+- plugins/upgrade_helper/pool.rb
+- plugins/upgrade_helper/host.rb
 - plugins/simple_tracker/simple_tracker.rb
 - plugins/simple_tracker/commandsuite.rb
 - plugins/simple_tracker/shard.rb
 - plugins/simple_tracker/db.rb
 - plugins/simple_tracker/pool.rb
 - plugins/simple_tracker/topology.rb
+- plugins/online_schema_change/commandsuite.rb
+- plugins/online_schema_change/db.rb
+- plugins/online_schema_change/pool.rb
+- plugins/online_schema_change/online_schema_change.rb
+- plugins/online_schema_change/topology.rb
 - etc/jetpants.yaml.sample
 homepage: https://github.com/tumblr/jetpants/
 licenses: []
 
+metadata: {}
+
 post_install_message: 
 rdoc_options: 
 - --line-numbers
@@ -173,13 +188,11 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
@@ -187,9 +200,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: "Jetpants: a MySQL automation toolkit by Tumblr"
 test_files: []