jetpants 0.8.0 → 0.8.2

data/plugins/capacity_plan/commandsuite.rb

@@ -0,0 +1,19 @@
+require 'thor'
+
+module Jetpants
+  class CommandSuite < Thor
+
+    desc 'capacity_snapshot', 'create a snapshot of the current useage'
+    def capacity_snapshot
+      Plugin::Capacity.new().snapshot
+    end
+
+    desc 'capacity_plan', 'capacity plan'
+    method_option :email, :email => 'email address to send capacity plan report to'
+    def capacity_plan
+      email = options[:email] || false
+      Plugin::Capacity.new().plan(email)
+    end
+
+  end
+end

data/plugins/capacity_plan/monkeypatch.rb

@@ -0,0 +1,20 @@
+module Enumerable
+  def sum
+    self.inject(0){|accum, i| accum + i }
+  end
+
+  def mean
+    self.sum/self.length.to_f
+  end
+
+  def sample_variance
+    m = self.mean
+    sum = self.inject(0){|accum, i| accum +(i-m)**2 }
+    sum/(self.length - 1).to_f
+  end
+
+  def standard_deviation
+    return Math.sqrt(self.sample_variance)
+  end
+
+end

data/plugins/jetpants_collins/db.rb

@@ -66,6 +66,11 @@ module Jetpants
       end
     end
     
+    # After changing the status of a node, clear its list of spare-node-related
+    # validation errors, so that we will re-probe when necessary
+    def after_collins_status=(value)
+      @spare_validation_errors = nil
+    end
     
     ##### NEW METHODS ##########################################################
     
@@ -76,14 +81,48 @@ module Jetpants
     end
     
     # Returns true if this database is a spare node and looks ready for use, false otherwise.
-    # The default implementation just ensures a collins status of Provisioned.
-    # Downstream plugins may override this to do additional checks to ensure the node is
-    # in a sane state. (The caller of this method already checks that the node is SSHable,
-    # and that MySQL is running, and the node isn't already in a pool -- so no need to
-    # check any of those here.)
+    # Normally no need for plugins to override this (as of Jetpants 0.8.1), they should 
+    # override DB#validate_spare instead.
     def usable_spare?
-      collins_status.downcase == 'provisioned'
+      if @spare_validation_errors.nil?
+        @spare_validation_errors = []
+        
+        # The order of checks is important -- if the node isn't even reachable by SSH,
+        # don't run any of the other checks, for example.
+        # Note that we probe concurrently in Topology#query_spare_assets, ahead of time
+        if !probed?
+          @spare_validation_errors << 'Attempt to probe node failed'
+        elsif !available?
+          @spare_validation_errors << 'Node is not reachable via SSH'
+        elsif !running?
+          @spare_validation_errors << 'MySQL is not running'
+        elsif pool
+          @spare_validation_errors << 'Node already has a pool'
+        else
+          validate_spare
+        end
+        
+        unless @spare_validation_errors.empty?
+          error_text = @spare_validation_errors.join '; '
+          output "Removed from spare pool for failing checks: #{error_text}"
+        end
+      end
+      @spare_validation_errors.empty?
     end
     
+    # Performs validation checks on this node to see whether it is a usable spare.
+    # The default implementation just ensures a collins status of Allocated and state
+    # of SPARE.
+    # Downstream plugins may override this to do additional checks to ensure the node is
+    # in a sane condition.
+    # No need to check whether the node is SSHable, MySQL is running, or not already in
+    # a pool -- DB#usable_spare? already does that automatically.
+    def validate_spare
+      # Confirm node is in Allocated:SPARE status:state. (Because Collins find API hits a
+      # search index which isn't synchronously updated with all writes, there's potential
+      # for a find call to return assets that just transitioned to a different status or state.)
+      status_state = collins_status_state
+      @spare_validation_errors << "Unexpected status:state value: #{status_state}" unless status_state == 'allocated:spare'
+    end
   end
 end

data/plugins/jetpants_collins/jetpants_collins.rb

@@ -156,7 +156,7 @@ module Jetpants
 
       # Pass in a hash mapping field name symbols to values to set
       #   Symbol   => String         -- optionally set any Collins attribute
-      #   :status  => String         -- optionally set the status value for the asset
+      #   :status  => String         -- optionally set the status value for the asset. Can optionally be a "status:state" string too.
       #   :asset   => Collins::Asset -- optionally pass this in to avoid an extra Collins API lookup, if asset already obtained
       #
       # Alternatively, pass in 2 strings (field_name, value) to set just a single Collins attribute (or status)
@@ -180,26 +180,30 @@ module Jetpants
               output "WARNING: unable to set Collins status to #{val}"
               next
             end
-            if attrs[:state]
-              previous_state = asset.state.name
-              previous_status = asset.status
-              if previous_state != attrs[:state].to_s || previous_status != attrs[:status].to_s
-                success = Jetpants::Plugin::JetCollins.set_status!(asset, attrs[:status], 'changed through jetpants', attrs[:state])
-                unless success
-                  Jetpants::Plugin::JetCollins.state_create!(attrs[:state], attrs[:state], attrs[:state], attrs[:status])
-                  success = Jetpants::Plugin::JetCollins.set_status!(asset, attrs[:status], 'changed through jetpants', attrs[:state])
-                end
-                raise "#{self}: Unable to set Collins state to #{attrs[:state]} and Unable to set Collins status to #{attrs[:status]}" unless success
-                output "Collins state changed from #{previous_state} to #{attrs[:state]}"
-                output "Collins status changed from #{previous_status} to #{attrs[:status]}"
-              end              
-            else
-              previous_value = asset.status
-              if previous_value != val.to_s
-                success = Jetpants::Plugin::JetCollins.set_status!(asset, val)
-                raise "#{self}: Unable to set Collins status to #{val}" unless success
-                output "Collins status changed from #{previous_value} to #{val}"
+            state_val = attrs[:state]
+            previous_status = asset.status.capitalize
+            # Allow setting status:state at once via foo.collins_status = 'allocated:running'
+            if val.include? ':'
+              raise "Attempting to set state in two places" if state_val
+              vals = val.split(':', 2)
+              val       = vals.first.capitalize
+              state_val = vals.last.upcase
+            end
+            if state_val
+              previous_state = asset.state.name.upcase
+              next unless previous_state != state_val.to_s.upcase || previous_status != val.to_s.capitalize
+              success = Jetpants::Plugin::JetCollins.set_status!(asset, val, 'changed through jetpants', state_val)
+              unless success
+                # If we failed to set to this state, try creating the state as new
+                Jetpants::Plugin::JetCollins.state_create!(state_val, state_val, state_val, val)
+                success = Jetpants::Plugin::JetCollins.set_status!(asset, val, 'changed through jetpants', state_val)
               end
+              raise "#{self}: Unable to set Collins state to #{state_val} and Unable to set Collins status to #{val}" unless success
+              output "Collins status:state changed from #{previous_status}:#{previous_state} to #{val.capitalize}:#{state_val.upcase}"
+            elsif previous_status != val.to_s.capitalize
+              success = Jetpants::Plugin::JetCollins.set_status!(asset, val)
+              raise "#{self}: Unable to set Collins status to #{val}" unless success
+              output "Collins status changed from #{previous_status} to #{val}"
             end
           when :state
             unless asset && asset.status && attrs[:status]
@@ -231,9 +235,16 @@ module Jetpants
         
       end
       
+      # Returns a single downcased "status:state" string, useful when trying to compare both fields
+      # at once.
+      def collins_status_state
+        values = collins_get :status, :state
+        "#{values[:status]}:#{values[:state]}".downcase
+      end
+      
     end # module JetCollins
   end # module Plugin
-end
+end # module Jetpants
 
 
 # load all the monkeypatches for other Jetpants classes

data/plugins/jetpants_collins/pool.rb

@@ -70,7 +70,7 @@ module Jetpants
                   master_read_weight: master_read_weight
       [@master, slaves].flatten.each do |db|
         current_status = (db.collins_status || '').downcase
-        db.collins_status = 'Allocated' unless current_status == 'maintenance'
+        db.collins_status = 'Allocated:RUNNING' unless current_status == 'maintenance'
         db.collins_pool = @name
       end
       @master.collins_secondary_role = 'MASTER'
@@ -128,11 +128,32 @@ module Jetpants
           end
         end
       end
+      
+      # Clean up any slaves that are no longer slaving (again only looking at current datacenter)
+      assets = Jetpants.topology.server_node_assets(@name, :slave)
+      assets.reject! {|a| a.location && a.location.upcase != Plugin::JetCollins.datacenter}
+      assets.map(&:to_db).each do |db|
+        if !db.running? || db.pool != self
+          db.output "Not replicating from new master, removing from pool #{self}"
+          db.collins_pool = ''
+          db.collins_secondary_role = ''
+          db.collins_status = 'Unallocated'
+        end
+      end
     end
     
     
     ##### NEW METHODS ##########################################################
     
+    # Returns the pool's creation time (as a unix timestamp) according to Collins.
+    # (note: may be off by a few hours until https://github.com/tumblr/collins/issues/80
+    # is resolved)
+    # Not called from anything in jetpants_collins, but available to your own
+    # custom automation if useful
+    def collins_creation_timestamp
+      collins_asset.created.to_time.to_i
+    end
+    
     # Called from DB#after_probe_master and DB#after_probe_slave for machines
     # that are unreachable via SSH, or reachable but MySQL isn't running.
     def slaves_according_to_collins

data/plugins/jetpants_collins/shard.rb

@@ -77,13 +77,13 @@ module Jetpants
           db.collins_secondary_role = ''
           db.collins_status = 'Unallocated'
         end
-      else
+      elsif @state != :initializing
         # Note that we don't call Pool#slaves here to get all 3 types in one shot,
         # because that potentially includes child shards, and we don't want to overwrite
         # their pool setting...
         [@master, active_slaves, standby_slaves, backup_slaves].flatten.each do |db|
           current_status = (db.collins_status || '').downcase
-          db.collins_status = 'Allocated' unless current_status == 'maintenance'
+          db.collins_status = 'Allocated:RUNNING' unless current_status == 'maintenance'
           db.collins_pool = @name
         end
         @master.collins_secondary_role = 'MASTER'
@@ -91,6 +91,13 @@ module Jetpants
         standby_slaves.each {|db| db.collins_secondary_role = 'STANDBY_SLAVE'}
         backup_slaves.each  {|db| db.collins_secondary_role = 'BACKUP_SLAVE'}
       end
+      
+      # handle lockless master migration situations
+      if @state == :child && master.master && !@parent
+        to_be_ejected_master = master.master
+        to_be_ejected_master.collins_secondary_role = :standby_slave # not accurate, but no better option for now
+      end
+      
       true
     end
     

data/plugins/jetpants_collins/topology.rb

@@ -73,7 +73,7 @@ module Jetpants
     end
     
     
-    # Returns (count) DB objects.  Pulls from machines in the Provisioned status
+    # Returns (count) DB objects.  Pulls from machines in the spare state
     # and converts them to the Allocated status.
     # You can pass in :role to request spares with a particular secondary_role
     def claim_spares(count, options={})
@@ -84,7 +84,7 @@ module Jetpants
         db.collins_pool = ''
         db.collins_secondary_role = ''
         db.collins_slave_weight = ''
-        db.collins_status = 'Allocated'
+        db.collins_status = 'Allocated:CLAIMED'
         db
       end
     end
@@ -181,6 +181,7 @@ module Jetpants
 
       if primary_roles.count == 1
         selector[:type] = '^CONFIGURATION$'
+        selector[:primary_role] = primary_roles.first
       else
         values = primary_roles.map {|r| "primary_role = ^#{r}$"}
         selector[:query] = 'type = ^CONFIGURATION$ AND (' + values.join(' OR ') + ')'
@@ -239,7 +240,7 @@ module Jetpants
     
     private
     
-    # Helper method to query Collins for spare provisioned DBs.
+    # Helper method to query Collins for spare DBs.
     def query_spare_assets(count, options={})
       # Intentionally no remoteLookup=true here.  We only want to grab spare nodes
       # from the datacenter that Jetpants is running in.
@@ -247,7 +248,8 @@ module Jetpants
         operation:        'and',
         details:          true,
         type:             'SERVER_NODE',
-        status:           'Provisioned',
+        status:           'Allocated',
+        state:            'SPARE',
         primary_role:     'DATABASE',
         size:             100,
       }
@@ -257,15 +259,12 @@ module Jetpants
       keep_nodes = []
       
       # Probe concurrently for speed reasons
-      nodes.map(&:to_db).concurrent_each(&:probe)
+      nodes.map(&:to_db).concurrent_each {|db| db.probe rescue nil}
       
       # Now iterate in a single-threaded way for simplicity
       nodes.each do |node|
         db = node.to_db
-        db.probe
-        if node.pool || !db.available? || !db.running? || !db.usable_spare?
-          db.output "Removed from potential spare pool for failing checks"
-        else
+        if db.usable_spare?
           keep_nodes << node
           break if keep_nodes.size >= count
         end

data/plugins/online_schema_change/commandsuite.rb

@@ -0,0 +1,56 @@
+require 'thor'
+
+module Jetpants
+  class CommandSuite < Thor
+
+    desc 'alter_table', 'perform an alter table'
+    method_option :pool, :desc => 'Name of pool to run the alter table on'
+    method_option :dry_run, :desc => 'Dry run of the alter table', :type => :boolean
+    method_option :alter, :desc => 'The alter statment (eg ADD COLUMN c1 INT)'
+    method_option :database, :desc => 'Database to run the alter table on'
+    method_option :table, :desc => 'Table to run the alter table on'
+    method_option :all_shards, :desc => 'To run on all the shards', :type => :boolean
+    def alter_table
+      unless options[:all_shards]
+        pool_name = options[:pool] || ask('Please enter a name of a pool: ')
+        pool = Jetpants.topology.pool(pool_name)
+        raise "#{pool_name} is not a pool name" unless pool
+      end
+
+      database = options[:database] || false
+      table = options[:table] || ask('Please enter a name of a table: ')
+      alter = options[:alter] || ask('Please enter a alter table statment (eg ADD COLUMN c1 INT): ')
+
+      if options[:all_shards]
+        Jetpants.topology.alter_table_shards(database, table, alter, options[:dry_run])
+      else
+        unless pool.alter_table(database, table, alter, options[:dry_run])
+          print "check for errors during online schema change\n"
+        end
+      end
+    end
+
+    desc 'alter_table_drop', 'drop the old table after the alter table is complete'
+    method_option :pool, :desc => 'Name of pool that your ran the alter table on'
+    method_option :table, :desc => 'Table you ran the alter table on'
+    method_option :database, :desc => 'Database you ran the alter table on'
+    method_option :all_shards, :desc => 'To run on all the shards', :type => :boolean
+    def alter_table_drop
+      unless options[:all_shards]
+        pool_name = options[:pool] || ask('Please enter a name of a pool: ')
+        pool = Jetpants.topology.pool(pool_name)
+        raise "#{pool_name} is not a pool name" unless pool
+      end
+
+      database = options[:database] || false
+      table = options[:table] || ask('Please enter a name of a table: ')
+
+      if options[:all_shards]
+        Jetpants.topology.drop_old_alter_table_shards(database, table)
+      else
+        pool.drop_old_alter_table(database, table)
+      end
+    end
+
+  end
+end

data/plugins/online_schema_change/db.rb

@@ -0,0 +1,33 @@
+module Jetpants
+  class DB
+
+    # Creates a temporary user for use of pt-table-checksum and pt-upgrade,
+    # yields to the supplied block, and then drops the user.
+    # The user will have a randomly-generated 50-character password, and will
+    # have elevated permissions (ALL PRIVILEGES on the application schema, and
+    # a few global privs as well) since these are necessary to run the tools.
+    # The block will be passed the randomly-generated password.
+    def with_online_schema_change_user(username, database)
+      password = DB.random_password
+      create_user username, password
+      grant_privileges username, '*', 'PROCESS', 'REPLICATION CLIENT', 'REPLICATION SLAVE', 'SUPER'
+      grant_privileges username, database, 'ALL PRIVILEGES'
+      begin
+        yield password
+      rescue StandardError, Interrupt, IOError
+        drop_user username
+        raise
+      end
+      drop_user username
+    end
+
+    # make sure there is enough space to do an online schema change
+    def has_space_for_alter?(table_name, database_name=nil)
+      database_name ||= app_schema
+      table_size = dir_size("#{mysql_directory}/#{database_name}/#{table_name}.ibd")
+
+      table_size < mount_stats['available']
+    end
+
+  end
+end

data/plugins/online_schema_change/online_schema_change.rb

@@ -0,0 +1,5 @@
+require 'json'
+
+
+# load all the monkeypatches for other Jetpants classes
+%w(pool db topology commandsuite).each {|mod| require "online_schema_change/#{mod}"}

data/plugins/online_schema_change/pool.rb

@@ -0,0 +1,105 @@
+# JetCollins monkeypatches to add Collins integration
+
+module Jetpants
+  class Pool
+    collins_attr_accessor :online_schema_change
+
+    def alter_table(database, table, alter, dry_run=true, force=false)
+      database ||= app_schema
+      error = false
+
+      raise "not enough space to run alter table on #{table}" unless master.has_space_for_alter?(table, database)
+
+      unless(check_collins_for_alter)
+        raise "alter table already running on #{@name}"
+      end
+
+      max_threads = max_threads_running(30,1)
+      max_threads = 50 unless max_threads > 50
+
+      critical_threads_running = 2 * max_threads > 500 ? 2 * max_threads : 500
+
+      update_collins_for_alter(database, table, alter)
+
+      master.with_online_schema_change_user('pt-osc', database) do |password|
+
+        command = "pt-online-schema-change --nocheck-replication-filters --max-load='Threads_running:#{max_threads}' --critical-load='Threads_running:#{critical_threads_running}' --nodrop-old-table --retries=10 --set-vars='wait_timeout=100000' --dry-run --print --alter '#{alter}' D=#{database},t=#{table},h=#{master.ip},u=#{'pt-osc'},p=#{password}"
+
+        print "[#{@name.to_s.red}][#{Time.now.to_s.blue}]---------------------------------------------------------------------------------------\n"
+        print "[#{@name.to_s.red}][#{Time.now.to_s.blue}] #{command}\n"
+        print "[#{@name.to_s.red}][#{Time.now.to_s.blue}]---------------------------------------------------------------------------------------\n"
+
+        IO.popen command do |io|
+          io.each do |line|
+            print "[#{@name.to_s.red}][#{Time.now.to_s.blue}] #{line.gsub("\n","")}\n"
+          end
+          error = true if $?.to_i > 0
+        end
+
+        if !(dry_run || error)
+          continue = 'no'
+          unless force
+            continue = ask('Dry run complete would you like to continue?: (YES/no)')
+          end
+
+          if force || continue == 'YES'
+            command = "pt-online-schema-change --nocheck-replication-filters --max-load='Threads_running:#{max_threads}' --critical-load='Threads_running:#{critical_threads_running}' --nodrop-old-table --retries=10 --set-vars='wait_timeout=100000' --execute --print --alter '#{alter}' D=#{database},t=#{table},h=#{master.ip},u=#{'pt-osc'},p=#{password}"
+            
+            print "[#{@name.to_s.red}][#{Time.now.to_s.blue}]---------------------------------------------------------------------------------------\n\n\n"
+            print "[#{@name.to_s.red}][#{Time.now.to_s.blue}] #{command}\n"
+            print "[#{@name.to_s.red}][#{Time.now.to_s.blue}]\n\n---------------------------------------------------------------------------------------\n\n\n"
+
+            IO.popen command do |io|
+              io.each do |line|
+                print "[#{@name.to_s.red}][#{Time.now.to_s.blue}] #{line.gsub("\n","")}\n"
+              end
+              error = true if $?.to_i > 0
+            end #end execute
+
+          end #end continue
+        
+        end #end if ! dry run
+
+      end #end user grant block
+
+      clean_up_collins_for_alter
+
+      ! error
+    end
+
+    # update collins for tracking alters, so there is only one running at a time
+    def update_collins_for_alter(database, table, alter)
+      self.collins_online_schema_change = JSON.pretty_generate({
+                                'running' => true,
+                                'started' => Time.now.to_i,
+                                'database' => database,
+                                'table' => table,
+                                'alter' => alter 
+                              })
+
+    end
+
+    # check if a alter is already running
+    def check_collins_for_alter()
+      return true if self.collins_online_schema_change.empty?
+      meta = JSON.parse(self.collins_online_schema_change)
+      if(meta['running'])
+        return false 
+      end
+      return true
+    end
+
+    # clean up collins after alter
+    def clean_up_collins_for_alter()
+      self.collins_online_schema_change = ''
+    end
+
+    # drop old table after an alter, this is because
+    # we do not drop the table after an alter
+    def drop_old_alter_table(database, table)
+      database ||= app_schema
+      master.mysql_root_cmd("USE #{database}; DROP TABLE _#{table}_old")
+    end
+
+  end
+end