jetpants 0.7.8 → 0.7.10

data/plugins/jetpants_collins/shard.rb

@@ -0,0 +1,106 @@
+# JetCollins monkeypatches to add Collins integration
+
+module Jetpants
+  class Shard < Pool
+    
+    ##### JETCOLLINS MIX-IN ####################################################
+    
+    include Plugin::JetCollins
+    
+    collins_attr_accessor :shard_min_id, :shard_max_id, :shard_state, :shard_parent
+    
+    # Returns a Collins::Asset for this pool
+    def collins_asset(create_if_missing=false)
+      selector = {
+        operation:    'and',
+        details:      true,
+        type:         'CONFIGURATION', 
+        primary_role: 'MYSQL_SHARD',
+        shard_min_id: "^#{@min_id}$",
+        shard_max_id: "^#{@max_id}$",
+      }
+      selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
+      
+      results = Plugin::JetCollins.find selector
+      
+      # If we got back multiple results, try ignoring the remote datacenter ones
+      if results.count > 1
+        filtered_results = results.select {|a| a.location.nil? || a.location.upcase == Plugin::JetCollins.datacenter}
+        results = filtered_results if filtered_results.count > 0
+      end
+      
+      if results.count > 1
+        raise "Multiple configuration assets found for pool #{name}"
+      elsif results.count == 0 && create_if_missing
+        output "Could not find configuration asset for pool; creating now"
+        new_tag = 'mysql-' + @name
+        asset = Collins::Asset.new type: 'CONFIGURATION', tag: new_tag, status: 'Allocated'
+        begin
+          Plugin::JetCollins.create!(asset)
+        rescue
+          collins_set asset:  asset,
+                      status: 'Allocated'
+        end
+        collins_set asset: asset, 
+                    primary_role: 'MYSQL_SHARD', 
+                    pool: @name.upcase,
+                    shard_min_id: @min_id,
+                    shard_max_id: @max_id
+        Plugin::JetCollins.get new_tag
+      elsif results.count == 0 && !create_if_missing
+        raise "Could not find configuration asset for pool #{name}"
+      else
+        results.first
+      end
+    end
+    
+    
+    ##### METHOD OVERRIDES #####################################################
+    
+    # Examines the current state of the pool (as known to Jetpants) and updates
+    # Collins to reflect this, in terms of the pool's configuration asset as
+    # well as the individual hosts.
+    def sync_configuration
+      status =  case
+                when in_config? then 'Allocated'
+                when @state == :deprecated then 'Cancelled'
+                when @state == :recycle  then 'Decommissioned'
+                else 'Provisioning'
+                end
+      collins_set asset: collins_asset(true),
+                  status: status,
+                  shard_state: @state.to_s.upcase,
+                  shard_parent: @parent ? @parent.name : ''
+      if @state == :recycle
+        [@master, @master.slaves].flatten.each do |db|
+          db.collins_pool = ''
+          db.collins_secondary_role = ''
+          db.collins_status = 'Unallocated'
+        end
+      else
+        # Note that we don't call Pool#slaves here to get all 3 types in one shot,
+        # because that potentially includes child shards, and we don't want to overwrite
+        # their pool setting...
+        [@master, active_slaves, standby_slaves, backup_slaves].flatten.each do |db|
+          current_status = (db.collins_status || '').downcase
+          db.collins_status = 'Allocated' unless current_status == 'maintenance'
+          db.collins_pool = @name
+        end
+        @master.collins_secondary_role = 'MASTER'
+
+        standby_slaves.each {|db| db.collins_secondary_role = 'STANDBY_SLAVE'}
+        backup_slaves.each  {|db| db.collins_secondary_role = 'BACKUP_SLAVE'}
+      end
+      true
+    end
+    
+    
+    ##### CALLBACKS ############################################################
+    
+    # After altering the state of a shard, sync the change to Collins immediately.
+    def after_state=(value)
+      sync_configuration
+    end
+    
+  end
+end

data/plugins/jetpants_collins/topology.rb

@@ -0,0 +1,239 @@
+# JetCollins monkeypatches to add Collins integration
+
+require 'yaml'
+require 'json'
+
+module Jetpants
+  class Topology
+    
+    ##### METHOD OVERRIDES #####################################################
+    
+    # IMPORTANT NOTE
+    # This plugin does NOT implement write_config, since this format of
+    # your app configuration file entirely depends on your web framework!
+    #
+    # You will have to implement this yourself in a separate plugin; recommended
+    # approach is to add serialization methods to Pool and Shard, and call it
+    # on each @pool, writing out to a file or pinging a config service, depending
+    # on whatever your application uses.
+    
+    
+    # Initializes list of pools + shards from Collins
+    def load_pools
+      # We keep a cache of Collins::Asset objects, organized as pool_name => role => [asset, asset, ...]
+      @pool_role_assets = {}
+
+      # Populate the cache for all master and active_slave nodes. (We restrict to these types
+      # because this is sufficient for creating Pool objects and generating app config files.)
+      Jetpants.topology.server_node_assets(false, :master, :active_slave)
+      
+      functional_partition_assets = []
+      shard_assets = []
+      configuration_assets('MYSQL_POOL', 'MYSQL_SHARD').each do |asset|
+        if asset.primary_role.upcase == 'MYSQL_SHARD'
+          shard_assets << asset
+        else
+          functional_partition_assets << asset
+        end
+      end
+      
+      functional_partition_assets.sort_by! {|a| (a.config_sort_order || 0).to_i}
+      shard_assets.sort_by! {|a| a.shard_min_id.to_i}
+      @pools = functional_partition_assets.map(&:to_pool) + shard_assets.map(&:to_pool)
+      @pools.compact! # remove nils from pools that had no master
+      
+      # Set up parent/child relationships between shards currently being split.
+      # We do this in a separate step afterwards so that Topology#pool can find the parent
+      # by name, regardless of ordering in Collins
+      @pools.select {|p| p.has_parent}.each do |p|
+        parent = pool(p.has_parent) or raise "Cannot find parent shard named #{p.has_parent}"
+        parent.add_child(p)
+      end
+      true
+    end
+    
+    
+    # Returns (count) DB objects.  Pulls from machines in the Provisioned status
+    # and converts them to the Allocated status.
+    # You can pass in :role to request spares with a particular secondary_role
+    def claim_spares(count, options={})
+      assets = query_spare_assets(count, options)
+      raise "Not enough spare machines available! Found #{assets.count}, needed #{count}" if assets.count < count
+      assets.map do |asset|
+        db = asset.to_db
+        db.collins_pool = ''
+        db.collins_secondary_role = ''
+        db.collins_slave_weight = ''
+        db.collins_status = 'Allocated'
+        db
+      end
+    end
+    
+    
+    # This method won't ever return a number higher than 100, but that's
+    # not a problem, since no single operation requires that many spares
+    def count_spares(options={})
+      query_spare_assets(100, options).count
+    end
+    
+    
+    ##### NEW METHODS ##########################################################
+    
+    # Returns an array of Collins::Asset objects meeting the given criteria.
+    # Caches the result for subsequent use.
+    # Optionally supply a pool name to restrict the result to that pool.
+    # Optionally supply one or more role symbols (:master, :active_slave,
+    # :standby_slave, :backup_slave) to filter the result to just those
+    # SECONDARY_ROLE values in Collins.
+    def server_node_assets(pool_name=false, *roles)
+      roles = normalize_roles(roles) if roles.count > 0
+      
+      # Check for previously-cached result. (Only usable if a pool_name supplied.)
+      if pool_name && @pool_role_assets[pool_name]
+        if roles.count > 0 && roles.all? {|r| @pool_role_assets[pool_name].has_key? r}
+          return roles.map {|r| @pool_role_assets[pool_name][r]}.flatten
+        elsif roles.count == 0 && valid_roles.all? {|r| @pool_role_assets[pool_name].has_key? r}
+          return @pool_role_assets[pool_name].values.flatten
+        end
+      end
+      
+      per_page = 200
+      selector = {
+        operation:    'and',
+        details:      true,
+        size:         per_page,
+        query:        'primary_role = ^DATABASE$ AND type = ^SERVER_NODE$'
+      }
+      selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
+      selector[:query] += " AND pool = ^#{pool_name}$" if pool_name
+      if roles.count == 1
+        selector[:query] += " AND secondary_role = ^#{roles.first}$"
+      elsif roles.count > 1
+        values = roles.map {|r| "secondary_role = ^#{r}$"}
+        selector[:query] += ' AND (' + values.join(' OR ') + ')'
+      end
+      
+      assets = []
+      done = false
+      page = 0
+
+      # Query Collins one or more times, until we've seen all the results
+      until done do
+        selector[:page] = page
+        results = Plugin::JetCollins.find selector.dup # find apparently alters the selector object now, so we dup it
+        done = results.count < per_page
+        page += 1
+        assets.concat results
+      end
+      
+      # Next we need to update our @pool_role_assets cache. But first let's set it to [] for each pool/role
+      # that we queried. This intentionally nukes any previous cached data, and also allows us to differentiate
+      # between an empty result and a cache miss.
+      roles = valid_roles if roles.count == 0
+      seen_pools = assets.map {|a| a.pool.downcase}
+      seen_pools << pool_name if pool_name
+      seen_pools.uniq.each do |p|
+        @pool_role_assets[p] ||= {}
+        roles.each {|r| @pool_role_assets[p][r] = []}
+      end
+      
+      # Filter
+      assets.select! {|a| a.pool && a.secondary_role && %w(allocated maintenance).include?(a.status.downcase)}
+      
+      # Cache
+      assets.each {|a| @pool_role_assets[a.pool.downcase][a.secondary_role.downcase.to_sym] << a}
+      
+      # Return
+      assets
+    end
+    
+    
+    # Returns an array of configuration assets with the supplied primary role(s)
+    def configuration_assets(*primary_roles)
+      raise "Must supply at least one primary_role" if primary_roles.count < 1
+      per_page = 200
+      
+      selector = {
+        operation:    'and',
+        details:      true,
+        size:         per_page,
+      }
+
+      if primary_roles.count == 1
+        selector[:type] = '^CONFIGURATION$'
+      else
+        values = primary_roles.map {|r| "primary_role = ^#{r}$"}
+        selector[:query] = 'type = ^CONFIGURATION$ AND (' + values.join(' OR ') + ')'
+      end
+      
+      selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
+      
+      done = false
+      page = 0
+      assets = []
+      until done do
+        selector[:page] = page
+        page_of_results = Plugin::JetCollins.find selector.dup # find apparently alters the selector object now, so we dup it
+        assets += page_of_results
+        done = page_of_results.count < per_page
+        page += 1
+      end
+      
+      # If remote lookup is enabled, remove the remote copy of any pool that exists
+      # in both local and remote datacenters.
+      if Jetpants.plugins['jetpants_collins']['remote_lookup']
+        dc_pool_map = {Plugin::JetCollins.datacenter => {}}
+        
+        assets.each do |a|
+          location = a.location || Plugin::JetCollins.datacenter
+          pool = a.pool ? a.pool.downcase : a.tag[6..-1].downcase  # if no pool, strip 'mysql-' off front and use that
+          dc_pool_map[location] ||= {}
+          dc_pool_map[location][pool] = a
+        end
+        
+        # Grab everything from current DC first (higher priority over other
+        # datacenters), then grab everything from remote DCs.
+        final_assets = dc_pool_map[Plugin::JetCollins.datacenter].values
+        dc_pool_map.each do |dc, pool_to_assets|
+          next if dc == Plugin::JetCollins.datacenter
+          pool_to_assets.each do |pool, a|
+            final_assets << a unless dc_pool_map[Plugin::JetCollins.datacenter][pool]
+          end
+        end
+        assets = final_assets
+      end
+      
+      # Remove decommissioned nodes
+      assets.reject {|a| a.status == 'Decommissioned'}
+    end
+    
+    
+    def clear_asset_cache(pool_name=false)
+      if pool_name
+        @pool_role_assets.delete pool_name
+      else
+        @pool_role_assets = {}
+      end
+    end
+    
+    
+    private
+    
+    # Helper method to query Collins for spare provisioned DBs.
+    def query_spare_assets(count, options={})
+      # Intentionally no remoteLookup=true here.  We only want to grab spare nodes
+      # from the datacenter that Jetpants is running in.
+      selector = {
+        operation:        'and',
+        details:          true,
+        type:             'SERVER_NODE',
+        status:           'Provisioned',
+        primary_role:     'DATABASE',
+        size:             count,
+      }
+      selector[:secondary_role] = options[:role].to_s.downcase if options[:role]
+      Plugin::JetCollins.find selector
+    end
+    
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: jetpants
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.7.8
+  version: 0.7.10
 platform: ruby
 authors: 
 - Evan Elias
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-09-25 00:00:00 Z
+date: 2012-10-31 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mysql2
@@ -90,6 +90,17 @@ dependencies:
         version: "1.2"
   type: :runtime
   version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: collins_client
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.2.7
+  type: :runtime
+  version_requirements: *id008
 description: Jetpants is an automation toolkit for handling monstrously large MySQL database topologies. It is geared towards common operational tasks like cloning slaves, rebalancing shards, and performing master promotions. It features a command suite for easy use by operations staff, though it's also a full Ruby library for use in developing custom migration scripts and database automation.
 email: 
 - me@evanelias.com
@@ -100,41 +111,50 @@ extensions: []
 
 extra_rdoc_files: 
 - README.rdoc
+- doc/faq.rdoc
 - doc/plugins.rdoc
+- doc/jetpants_collins.rdoc
+- doc/requirements.rdoc
 - doc/configuration.rdoc
-- doc/faq.rdoc
 - doc/commands.rdoc
-- doc/requirements.rdoc
 files: 
 - Gemfile
 - README.rdoc
+- doc/faq.rdoc
 - doc/plugins.rdoc
+- doc/jetpants_collins.rdoc
+- doc/requirements.rdoc
 - doc/configuration.rdoc
-- doc/faq.rdoc
 - doc/commands.rdoc
-- doc/requirements.rdoc
-- lib/jetpants/callback.rb
+- lib/jetpants.rb
 - lib/jetpants/topology.rb
-- lib/jetpants/db/server.rb
-- lib/jetpants/db/state.rb
+- lib/jetpants/db.rb
+- lib/jetpants/host.rb
+- lib/jetpants/table.rb
+- lib/jetpants/callback.rb
 - lib/jetpants/db/import_export.rb
-- lib/jetpants/db/privileges.rb
+- lib/jetpants/db/state.rb
+- lib/jetpants/db/server.rb
 - lib/jetpants/db/client.rb
+- lib/jetpants/db/privileges.rb
 - lib/jetpants/db/replication.rb
 - lib/jetpants/shard.rb
-- lib/jetpants/db.rb
-- lib/jetpants/host.rb
 - lib/jetpants/pool.rb
 - lib/jetpants/monkeypatch.rb
-- lib/jetpants/table.rb
-- lib/jetpants.rb
 - bin/jetpants
+- plugins/simple_tracker/commandsuite.rb
 - plugins/simple_tracker/topology.rb
-- plugins/simple_tracker/shard.rb
-- plugins/simple_tracker/simple_tracker.rb
 - plugins/simple_tracker/db.rb
+- plugins/simple_tracker/shard.rb
 - plugins/simple_tracker/pool.rb
-- plugins/simple_tracker/commandsuite.rb
+- plugins/simple_tracker/simple_tracker.rb
+- plugins/jetpants_collins/jetpants_collins.rb
+- plugins/jetpants_collins/topology.rb
+- plugins/jetpants_collins/db.rb
+- plugins/jetpants_collins/host.rb
+- plugins/jetpants_collins/asset.rb
+- plugins/jetpants_collins/shard.rb
+- plugins/jetpants_collins/pool.rb
 - etc/jetpants.yaml.sample
 homepage: https://github.com/tumblr/jetpants/
 licenses: []