jetpants 0.7.8 → 0.7.10

data/lib/jetpants/topology.rb

@@ -87,6 +87,15 @@ module Jetpants
       raise "Plugin must override Topology#count_spares"
     end
     
+    # Returns a list of valid role symbols in use in Jetpants.
+    def valid_roles
+      [:master, :active_slave, :standby_slave, :backup_slave]
+    end
+    
+    # Returns a list of valid role symbols which indicate a slave status
+    def slave_roles
+      valid_roles.reject {|r| r == :master}
+    end
     
     ###### Instance Methods ####################################################
     
@@ -144,6 +153,19 @@ module Jetpants
       claim_spares(1, options)[0]
     end
     
+    # Returns if the supplied role is valid
+    def valid_role? role
+      valid_roles.include? role.to_s.downcase.to_sym
+    end
+    
+    # Converts the supplied roles (strings or symbols) into lowercase symbol versions
+    # Will expand out special role of :slave to be all slave roles.
+    def normalize_roles(*roles)
+      roles = roles.flatten.map {|r| r.to_s.downcase == 'slave' ? slave_roles.map(&:to_s) : r.to_s.downcase}.flatten
+      roles.each {|r| raise "#{r} is not a valid role" unless valid_role? r}
+      roles.uniq.map &:to_sym
+    end
+    
     synchronized
     # Clears the pool list and nukes cached DB and Host object lookup tables
     def clear

data/plugins/jetpants_collins/asset.rb

@@ -0,0 +1,77 @@
+# Adds conversion methods to the Collins::Asset class for obtaining Jetpants equivalents
+
+module Collins
+  class Asset
+    
+    # Convert a Collins::Asset to a Jetpants::DB. Requires asset TYPE to be SERVER_NODE.
+    def to_db
+      raise "Can only call to_db on SERVER_NODE assets, but #{self} has type #{type}" unless type.upcase == 'SERVER_NODE'
+      backend_ip_address.to_db
+    end
+    
+    
+    # Convert a Collins::Asset to a Jetpants::Host. Requires asset TYPE to be SERVER_NODE.
+    def to_host
+      raise "Can only call to_host on SERVER_NODE assets, but #{self} has type #{type}" unless type.upcase == 'SERVER_NODE'
+      backend_ip_address.to_host
+    end
+    
+    
+    # Convert a Collins::Asset to either a Jetpants::Pool or a Jetpants::Shard, depending
+    # on the value of PRIMARY_ROLE.  Requires asset TYPE to be CONFIGURATION.
+    def to_pool
+      raise "Can only call to_pool on CONFIGURATION assets, but #{self} has type #{type}" unless type.upcase == 'CONFIGURATION'
+      raise "Unknown primary role #{primary_role} for configuration asset #{self}" unless ['MYSQL_POOL', 'MYSQL_SHARD'].include?(primary_role.upcase)
+      raise "No pool attribute set on asset #{self}" unless pool && pool.length > 0
+      
+      # Find the master(s) for this pool. If we got back multiple masters, first
+      # try ignoring the remote datacenter ones
+      master_assets = Jetpants.topology.server_node_assets(pool.downcase, :master)
+      if master_assets.count > 1
+        results = master_assets.select {|a| a.location.nil? || a.location.upcase == Plugin::JetCollins.datacenter}
+        master_assets = results if results.count > 0
+      end
+      puts "WARNING: multiple masters found for pool #{pool}; using first match" if master_assets.count > 1
+      
+      if master_assets.count == 0
+        puts "WARNING: no masters found for pool #{pool}; ignoring pool entirely"
+        result = nil
+      
+      elsif primary_role.upcase == 'MYSQL_POOL'
+        result = Jetpants::Pool.new(pool.downcase, master_assets.first.to_db)
+        if aliases
+          aliases.split(',').each {|a| result.add_alias(a.downcase)}
+        end
+        result.slave_name = slave_pool_name if slave_pool_name
+        result.master_read_weight = master_read_weight if master_read_weight
+
+        # We intentionally only look for active slaves in the current datacenter, since we
+        # treat other datacenters' slaves as backup slaves to prevent promotion or cross-DC usage
+        active_slave_assets = Jetpants.topology.server_node_assets(pool.downcase, :active_slave)
+        active_slave_assets.reject! {|a| a.location && a.location.upcase != Plugin::JetCollins.datacenter}
+        active_slave_assets.each do |asset|
+          weight = asset.slave_weight && asset.slave_weight.to_i > 0 ? asset.slave_weight.to_i : 100
+          result.has_active_slave(asset.to_db, weight)
+        end
+        
+      elsif primary_role.upcase == 'MYSQL_SHARD'
+        result = Jetpants::Shard.new(shard_min_id.to_i, 
+                                   shard_max_id == 'INFINITY' ? 'INFINITY' : shard_max_id.to_i, 
+                                   master_assets.first.to_db, 
+                                   shard_state.downcase.to_sym)
+        
+        # We'll need to set up the parent/child relationship if a shard split is in progress,
+        # BUT we need to wait to do that later since the shards may have been returned by
+        # Collins out-of-order, so the parent shard object might not exist yet.
+        # For now we just remember the NAME of the parent shard.
+        result.has_parent = shard_parent if shard_parent
+        
+      else
+        raise "Unknown configuration asset primary role #{primary_role} for asset #{self}"
+      end
+      
+      result
+    end
+    
+  end
+end

data/plugins/jetpants_collins/db.rb

@@ -0,0 +1,77 @@
+# JetCollins monkeypatches to add Collins integration
+
+module Jetpants
+  class DB
+    
+    ##### JETCOLLINS MIX-IN ####################################################
+    
+    include Plugin::JetCollins
+    
+    collins_attr_accessor :slave_weight
+    
+    # Because we only support 1 mysql instance per machine for now, we can just
+    # delegate this over to the host
+    def collins_asset
+      @host.collins_asset
+    end
+    
+    
+    ##### METHOD OVERRIDES #####################################################
+    
+    # Add an actual collins check to confirm a machine is a standby
+    def is_standby?
+      !(running?) || (is_slave? && !taking_connections? && collins_secondary_role == 'standby_slave')
+    end
+    
+    # Treat any node outside of current data center as being for backups.
+    # This prevents inadvertent cross-data-center master promotion.
+    def for_backups?
+      hostname.start_with?('backup') || in_remote_datacenter?
+    end
+    
+    
+    ##### CALLBACKS ############################################################
+    
+    # Determine master from Collins if machine is unreachable or MySQL isn't running.
+    def after_probe_master
+      unless @running
+        if collins_secondary_role == 'master'
+          @master = false
+        else
+          pool = Jetpants.topology.pool(collins_pool)
+          @master = pool.master if pool
+        end
+      end
+      
+      # We completely ignore cross-data-center master unless inter_dc_mode is enabled.
+      # This may change in a future Jetpants release, once we support tiered replication more cleanly.
+      if @master && @master.in_remote_datacenter? && !Jetpants::Plugin::JetCollins.inter_dc_mode?
+        @remote_master = @master # keep track of it, in case we need to know later
+        @master = false
+      elsif !@master
+        in_remote_datacenter? # just calling to cache for current node, before we probe its slaves, so that its slaves don't need to query Collins
+      end
+    end
+    
+    # Determine slaves from Collins if machine is unreachable or MySQL isn't running
+    def after_probe_slaves
+      # If this machine has a master AND has slaves of its own AND is in another data center,
+      # ignore its slaves entirely unless inter_dc_mode is enabled.
+      # This may change in a future Jetpants release, once we support tiered replication more cleanly.
+      @slaves = [] if @running && @master && @slaves.count > 0 && in_remote_datacenter? && !Jetpants::Plugin::JetCollins.inter_dc_mode?
+      
+      unless @running
+        p = Jetpants.topology.pool(self)
+        @slaves = (p ? p.slaves_according_to_collins : [])
+      end
+    end
+    
+    
+    ##### NEW METHODS ##########################################################
+    
+    def in_remote_datacenter?
+      @host.collins_location != Plugin::JetCollins.datacenter
+    end
+    
+  end
+end

data/plugins/jetpants_collins/host.rb

@@ -0,0 +1,41 @@
+# JetCollins monkeypatches to add Collins integration
+
+module Jetpants
+  class Host
+    
+    ##### JETCOLLINS MIX-IN ####################################################
+    
+    include Plugin::JetCollins
+    
+    def collins_asset
+      # try IP first; failing that, try hostname
+      selector = {ip_address: ip, details: true}
+      selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
+      assets = Plugin::JetCollins.find selector
+
+      if (!assets || assets.count == 0) && available?
+        selector = {hostname: "^#{hostname}$", details: true}
+        selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
+        assets = Plugin::JetCollins.find selector
+      end
+      
+      raise "Multiple assets found for #{self}" if assets.count > 1
+      if ! assets || assets.count == 0
+        output "WARNING: no Collins assets found for this host"
+        nil
+      else
+        assets.first
+      end
+    end
+    
+    # Returns which datacenter this host is in. Only a getter, intentionally no setter.
+    def collins_location
+      return @collins_location if @collins_location
+      ca = collins_asset
+      @collins_location ||= (ca ? ca.location || Plugin::JetCollins.datacenter : 'unknown')
+      @collins_location.upcase!
+      @collins_location
+    end
+    
+  end
+end

data/plugins/jetpants_collins/jetpants_collins.rb

@@ -0,0 +1,214 @@
+require 'collins_client'
+
+# Entrypoint for jetpants_collins plugin (namespace Jetpants::Plugin::JetCollins),
+# which offers integration with the Collins hardware asset tracking system.
+# This particular file accomplishes the following:
+#
+#   * Provides a JetCollins mixin module. Any class including this should also
+#     implement a collins_asset method to convert objects to Collins assets;
+#     the class can then use the provided collins_get and collins_set wrappers,
+#     along with the collins_attr_accessor class method.
+#
+#   * Jetpants::Plugin::JetCollins can also be used as a global Collins API
+#     client -- the module itself will delegate all missing methods to a 
+#     Collins::Client object.
+#
+#   * Loads monkeypatches for Jetpants classes DB, Host, Pool, Shard, Topology,
+#     and Collins class Asset.
+#
+# Configuration options in Jetpants config file include:
+#   user          =>  collins account username (required)
+#   password      =>  collins account password (required)
+#   url           =>  collins URL (required)
+#   timeout       =>  collins client timeout, in seconds (default: 30)
+#   datacenter    =>  collins data center name that we're running Jetpants in the context of (required if multi-datacenter)
+#   remote_lookup =>  if true, supply remoteLookup parameter to search multiple datacenters (default: false)
+
+
+module Jetpants
+  module Plugin
+    module JetCollins
+      @collins_service = nil
+      
+      ##### CLASS METHODS ######################################################
+
+      class << self
+        
+        # We delegate missing class (module) methods to the collins API client,
+        # if it responds to them.
+        def method_missing(name, *args, &block)
+          if service.respond_to? name
+            service.send name, *args, &block
+          else
+            super
+          end
+        end
+        
+        # Eigenclass mix-in for collins_attr_accessor
+        # Calling "collins_attr_accessor :foo" in your class body will create
+        # methods collins_foo and collins_foo= which automatically get/set
+        # Collins attribute foo
+        def included(base)
+          base.class_eval do 
+            def self.collins_attr_accessor(*fields)
+              fields.each do |field|
+                define_method("collins_#{field}") do
+                  (collins_get(field) || '').downcase
+                end
+                define_method("collins_#{field}=") do |value|
+                  collins_set(field, value)
+                end
+              end
+            end
+            
+            # We make these 4 accessors available to ANY class including this mixin
+            collins_attr_accessor :primary_role, :secondary_role, :pool, :status
+          end
+        end
+        
+        # Returns the 'datacenter' config option for this plugin, or 'UNKNOWN-DC' if
+        # none has been configured. This only matters in multi-datacenter Collins
+        # topologies.
+        def datacenter
+          (Jetpants.plugins['jetpants_collins']['datacenter'] || 'UNKNOWN-DC').upcase
+        end
+        
+        # Ordinarily, in a multi-dacenter environment, jetpants_collins places a number
+        # of restrictions on interacting with assets that aren't in the local datacenter,
+        # for safety's sake and to simplify how hierarchical replication trees are represented:
+        #
+        #   * Won't change Collins attributes on remote server node assets.
+        #   * If a local node has a master in a remote datacenter, it is ignored/hidden.
+        #   * If a local node has a slave in a remote datacenter, it's treated as a backup_slave,
+        #     in order to prevent cross-datacenter master promotions. If any of these
+        #     remote-datacenter slaves have slaves of their own, they're ignored/hidden.
+        #   
+        # You may DISABLE these restrictions by calling enable_inter_dc_mode. Normally you
+        # do NOT want to do this, except in special sitautions like a migration between
+        # datacenters.
+        def enable_inter_dc_mode
+          Jetpants.plugins['jetpants_collins']['inter_dc_mode'] = true
+          Jetpants.plugins['jetpants_collins']['remote_lookup'] = true
+        end
+        
+        # Returns true if enable_inter_dc_mode has been called, false otherwise.
+        def inter_dc_mode?
+          Jetpants.plugins['jetpants_collins']['inter_dc_mode'] || false
+        end
+        
+        
+        private
+        
+        # Returns a Collins::Client object
+        def service
+          return @collins_service if @collins_service
+          
+          %w(url user password).each do |setting|
+            raise "No Collins #{setting} set in plugins -> jetpants_collins -> #{setting}" unless Jetpants.plugins['jetpants_collins'][setting]
+          end
+          
+          logger = Logger.new(STDOUT)
+          logger.level = Logger::INFO
+          config = {
+            :host     =>  Jetpants.plugins['jetpants_collins']['url'],
+            :timeout  =>  Jetpants.plugins['jetpants_collins']['timeout'] || 30,
+            :username =>  Jetpants.plugins['jetpants_collins']['user'],
+            :password =>  Jetpants.plugins['jetpants_collins']['password'],
+            :logger   =>  logger,
+          }
+          @collins_service = Collins::Client.new(config)
+        end
+      end
+      
+      
+      ##### INSTANCE (MIX-IN) METHODS ##########################################
+      
+      # The base class needs to implement this!
+      def collins_asset
+        raise "Any class including Plugin::JetCollins must also implement collins_asset instance method!"
+      end
+      
+      # Pass in a symbol, or array of symbols, to obtain from Collins for this
+      # asset. For example, :status, :pool, :primary_role, :secondary_role.
+      # If you pass in a single symbol, returns a single value.
+      # If you pass in an array, returns a hash mapping each of these fields to their values.
+      # Hash will also contain an extra field called :asset, storing the Collins::Asset object.
+      def collins_get(*field_names)
+        asset = collins_asset
+        if field_names.count > 1 || field_names[0].is_a?(Array)
+          field_names.flatten!
+          results = Hash[field_names.map {|field| [field, (asset ? asset.send(field) : '')]}]
+          results[:asset] = asset
+          results
+        elsif field_names.count == 1
+          return '' unless asset
+          asset.send field_names[0]
+        else
+          nil
+        end
+      end
+
+      # Pass in a hash mapping field name symbols to values to set
+      #   Symbol   => String         -- optionally set any Collins attribute
+      #   :status  => String         -- optionally set the status value for the asset
+      #   :asset   => Collins::Asset -- optionally pass this in to avoid an extra Collins API lookup, if asset already obtained
+      #
+      # Alternatively, pass in 2 strings (field_name, value) to set just a single Collins attribute (or status)
+      def collins_set(*args)
+        attrs = (args.count == 1 ? args[0] : {args[0] => args[1]})
+        asset = attrs[:asset] || collins_asset
+        
+        # refuse to set Collins values on machines in remote data center unless
+        # inter_dc_mode is enabled
+        if asset && asset.type.downcase == 'server_node' && asset.location && asset.location.upcase != Plugin::JetCollins.datacenter
+          asset = nil unless Jetpants::Plugin::JetCollins.inter_dc_mode?
+        end
+        
+        attrs.each do |key, val|
+          val ||= ''
+          case key
+          when :asset
+            next
+          when :status
+            unless asset
+              output "WARNING: unable to set Collins status to #{val}"
+              next
+            end
+            previous_value = asset.status
+            if previous_value != val.to_s
+              success = Jetpants::Plugin::JetCollins.set_status!(asset, val)
+              raise "#{self}: Unable to set Collins status to #{val}" unless success
+              output "Collins status changed from #{previous_value} to #{val}"
+            end
+          else
+            unless asset
+              output "WARNING: unable to set Collins attribute #{key} to #{val}"
+              next
+            end
+            previous_value = asset.send(key)
+            if previous_value != val.to_s.upcase
+              success = Jetpants::Plugin::JetCollins.set_attribute!(asset, key.to_s.upcase, val.to_s.upcase)
+              raise "#{self}: Unable to set Collins attribute #{key} to #{val}" unless success
+              if (val.to_s == '' || !val) && (previous_value == '' || !previous_value)
+                false
+              elsif val.to_s == ''
+                output "Collins attribute #{key.to_s.upcase} removed (was: #{previous_value})"
+              elsif !previous_value || previous_value == ''
+                output "Collins attribute #{key.to_s.upcase} set to #{val.to_s.upcase}"
+              else
+                output "Collins attribute #{key.to_s.upcase} changed from #{previous_value} to #{val.to_s.upcase}"
+              end
+            end
+          end
+        end
+        
+      end
+      
+    end # module JetCollins
+  end # module Plugin
+end
+
+
+# load all the monkeypatches for other Jetpants classes
+%w(asset host db pool shard topology).each {|mod| require "jetpants_collins/#{mod}"}
+

data/plugins/jetpants_collins/pool.rb

@@ -0,0 +1,145 @@
+# JetCollins monkeypatches to add Collins integration
+
+module Jetpants
+  class Pool
+    
+    ##### JETCOLLINS MIX-IN ####################################################
+    
+    include Plugin::JetCollins
+    
+    # Used at startup time, to keep track of parent/child shard relationships
+    attr_accessor :has_parent
+    
+    # Collins accessors for configuration asset metadata
+    collins_attr_accessor :slave_pool_name, :aliases, :master_read_weight, :config_sort_order
+    
+    # Returns a Collins::Asset for this pool. Can optionally create one if not found.
+    def collins_asset(create_if_missing=false)
+      selector = {
+        operation:    'and',
+        details:      true,
+        type:         'CONFIGURATION', 
+        primary_role: 'MYSQL_POOL',
+        pool:         "^#{@name.upcase}$",
+        status:       'Allocated',
+      }
+      selector[:remoteLookup] = true if Jetpants.plugins['jetpants_collins']['remote_lookup']
+
+      results = Plugin::JetCollins.find selector
+      
+      # If we got back multiple results, try ignoring the remote datacenter ones
+      if results.count > 1
+        filtered_results = results.select {|a| a.location.nil? || a.location.upcase == Plugin::JetCollins.datacenter}
+        results = filtered_results if filtered_results.count > 0
+      end
+      
+      if results.count > 1
+        raise "Multiple configuration assets found for pool #{name}"
+      elsif results.count == 0 && create_if_missing
+        output "Could not find configuration asset for pool; creating now"
+        new_tag = 'mysql-' + @name
+        asset = Collins::Asset.new type: 'CONFIGURATION', tag: new_tag, status: 'Allocated'
+        begin
+          Plugin::JetCollins.create!(asset)
+        rescue
+          collins_set asset:  asset,
+                      status: 'Allocated'
+        end
+        collins_set asset: asset, 
+                    primary_role: 'MYSQL_POOL', 
+                    pool: @name.upcase
+        Plugin::JetCollins.get new_tag
+      elsif results.count == 0 && !create_if_missing
+        raise "Could not find configuration asset for pool #{name}"
+      else
+        results.first
+      end
+    end
+    
+    
+    ##### METHOD OVERRIDES #####################################################
+    
+    # Examines the current state of the pool (as known to Jetpants) and updates
+    # Collins to reflect this, in terms of the pool's configuration asset as
+    # well as the individual hosts.
+    def sync_configuration
+      asset = collins_asset(true)
+      collins_set asset: asset,
+                  slave_pool_name: slave_name || '',
+                  aliases: aliases.join(',') || '',
+                  master_read_weight: master_read_weight
+      [@master, slaves].flatten.each do |db|
+        current_status = (db.collins_status || '').downcase
+        db.collins_status = 'Allocated' unless current_status == 'maintenance'
+        db.collins_pool = @name
+      end
+      @master.collins_secondary_role = 'MASTER'
+      slaves(:active).each do |db| 
+        db.collins_secondary_role = 'ACTIVE_SLAVE'
+        weight = @active_slave_weights[db]
+        db.collins_slave_weight = (weight == 100 ? '' : weight)
+      end
+
+      slaves(:standby).each {|db| db.collins_secondary_role = 'STANDBY_SLAVE'}
+      slaves(:backup).each {|db| db.collins_secondary_role = 'BACKUP_SLAVE'}
+      true
+    end
+    
+    # If the pool's master hasn't been probed yet, return active_slaves list
+    # based strictly on what we found in Collins. This is a major speed-up at
+    # start-up time, especially for tasks that need to iterate over all pools.
+    alias :active_slaves_from_probe :active_slaves
+    def active_slaves
+      if @master.probed?
+        active_slaves_from_probe
+      else
+        @active_slave_weights.keys
+      end
+    end
+    
+    
+    ##### CALLBACKS ############################################################
+    
+    # Pushes slave removal to Collins. (Normally this type of logic is handled by
+    # Pool#sync_configuration, but that won't handle this case, since
+    # sync_configuration only updates hosts still in the pool.)
+    def after_remove_slave!(slave_db)
+      slave_db.collins_pool = slave_db.collins_secondary_role = slave_db.collins_slave_weight = ''
+      current_status = (slave_db.collins_status || '').downcase
+      slave_db.collins_status = 'Unallocated' unless current_status == 'maintenance'
+    end
+    
+    # If the demoted master was offline, record some info in Collins, otherwise
+    # there will be 2 masters listed
+    def after_master_promotion!(promoted)
+      Jetpants.topology.clear_asset_cache
+      
+      # Find the master asset(s) for this pool, filtering down to only current datacenter
+      assets = Jetpants.topology.server_node_assets(@name, :master)
+      assets.reject! {|a| a.location && a.location.upcase != Plugin::JetCollins.datacenter}
+      assets.map(&:to_db).each do |db|
+        if db != @master || !db.running?
+          db.collins_status = 'Maintenance'
+          db.collins_pool = ''
+          db.collins_secondary_role = ''
+        end
+      end
+    end
+    
+    
+    ##### NEW METHODS ##########################################################
+    
+    # Called from DB#after_probe_master and DB#after_probe_slave for machines
+    # that are unreachable via SSH, or reachable but MySQL isn't running.
+    def slaves_according_to_collins
+      results = []
+      Jetpants.topology.server_node_assets(@name, :slave).each do |asset|
+        slave = asset.to_db
+        output "Collins found slave #{slave.ip} (#{slave.hostname})"
+        results << slave
+      end
+      results
+    end
+    
+  end
+end