jetpants 0.7.10 → 0.8.0

data/plugins/jetpants_collins/topology.rb

@@ -6,7 +6,7 @@ require 'json'
 module Jetpants
   class Topology
     
-    ##### METHOD OVERRIDES #####################################################
+    ##### METHODS THAT OTHER PLUGINS CAN OVERRIDE ##############################
     
     # IMPORTANT NOTE
     # This plugin does NOT implement write_config, since this format of
@@ -18,6 +18,26 @@ module Jetpants
     # on whatever your application uses.
     
     
+    # Handles extra options for querying spare nodes. Takes a Collins selector
+    # hash and an options hash, and returns a potentially-modified Collins
+    # selector hash.
+    # The default implementation here implements no special logic. Custom plugins
+    # (loaded AFTER jetpants_collins is loaded) can override this method to
+    # manipulate the selector; see commented-out example below. 
+    def process_spare_selector_options(selector, options)
+      # If you wanted to support an option of :role, and map this to the Collins
+      # SECONDARY_ROLE attribute, you could implement this via:
+      #    selector[:secondary_role] = options[:role].to_s.downcase if options[:role]
+      # This could be useful if, for example, you use a different hardware spec
+      # for masters vs slaves. (doing so isn't really recommended, which is why
+      # we omit this logic by default.)
+      
+      # return the selector
+      selector
+    end
+    
+    ##### METHOD OVERRIDES #####################################################
+    
     # Initializes list of pools + shards from Collins
     def load_pools
       # We keep a cache of Collins::Asset objects, organized as pool_name => role => [asset, asset, ...]
@@ -123,9 +143,9 @@ module Jetpants
         results = Plugin::JetCollins.find selector.dup # find apparently alters the selector object now, so we dup it
         done = results.count < per_page
         page += 1
-        assets.concat results
+        assets.concat(results.select {|a| a.pool}) # filter out any spare nodes, which will have no pool set
       end
-      
+
       # Next we need to update our @pool_role_assets cache. But first let's set it to [] for each pool/role
       # that we queried. This intentionally nukes any previous cached data, and also allows us to differentiate
       # between an empty result and a cache miss.
@@ -229,10 +249,28 @@ module Jetpants
         type:             'SERVER_NODE',
         status:           'Provisioned',
         primary_role:     'DATABASE',
-        size:             count,
+        size:             100,
       }
-      selector[:secondary_role] = options[:role].to_s.downcase if options[:role]
-      Plugin::JetCollins.find selector
+      selector = process_spare_selector_options(selector, options)
+      
+      nodes = Plugin::JetCollins.find(selector)
+      keep_nodes = []
+      
+      # Probe concurrently for speed reasons
+      nodes.map(&:to_db).concurrent_each(&:probe)
+      
+      # Now iterate in a single-threaded way for simplicity
+      nodes.each do |node|
+        db = node.to_db
+        db.probe
+        if node.pool || !db.available? || !db.running? || !db.usable_spare?
+          db.output "Removed from potential spare pool for failing checks"
+        else
+          keep_nodes << node
+          break if keep_nodes.size >= count
+        end
+      end
+      keep_nodes.slice(0,count)
     end
     
   end

data/plugins/simple_tracker/topology.rb

@@ -37,6 +37,7 @@ module Jetpants
       puts "Regenerated #{config_file_path}"
     end
     
+    # simple_tracker completely ignores any options like :role or :like
     def claim_spares(count, options={})
       raise "Not enough spare machines -- requested #{count}, only have #{@tracker.spares.count}" if @tracker.spares.count < count
       hashes = @tracker.spares.shift(count)

metadata

@@ -2,16 +2,18 @@
 name: jetpants
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.7.10
+  version: 0.8.0
 platform: ruby
 authors: 
 - Evan Elias
 - Dallas Marlow
+- Bob Patterson Jr.
+- Tom Christ
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2012-10-31 00:00:00 Z
+date: 2013-03-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mysql2
@@ -105,56 +107,58 @@ description: Jetpants is an automation toolkit for handling monstrously large My
 email: 
 - me@evanelias.com
 - dallasmarlow@gmail.com
+- bob@bobpattersonjr.com
+- tbchrist@gmail.com
 executables: 
 - jetpants
 extensions: []
 
 extra_rdoc_files: 
 - README.rdoc
-- doc/faq.rdoc
-- doc/plugins.rdoc
-- doc/jetpants_collins.rdoc
-- doc/requirements.rdoc
 - doc/configuration.rdoc
+- doc/faq.rdoc
 - doc/commands.rdoc
+- doc/requirements.rdoc
+- doc/jetpants_collins.rdoc
+- doc/plugins.rdoc
 files: 
 - Gemfile
 - README.rdoc
-- doc/faq.rdoc
-- doc/plugins.rdoc
-- doc/jetpants_collins.rdoc
-- doc/requirements.rdoc
 - doc/configuration.rdoc
+- doc/faq.rdoc
 - doc/commands.rdoc
-- lib/jetpants.rb
-- lib/jetpants/topology.rb
-- lib/jetpants/db.rb
-- lib/jetpants/host.rb
+- doc/requirements.rdoc
+- doc/jetpants_collins.rdoc
+- doc/plugins.rdoc
+- lib/jetpants/shard.rb
 - lib/jetpants/table.rb
-- lib/jetpants/callback.rb
-- lib/jetpants/db/import_export.rb
+- lib/jetpants/db.rb
+- lib/jetpants/pool.rb
+- lib/jetpants/db/privileges.rb
 - lib/jetpants/db/state.rb
+- lib/jetpants/db/import_export.rb
 - lib/jetpants/db/server.rb
 - lib/jetpants/db/client.rb
-- lib/jetpants/db/privileges.rb
 - lib/jetpants/db/replication.rb
-- lib/jetpants/shard.rb
-- lib/jetpants/pool.rb
+- lib/jetpants/callback.rb
 - lib/jetpants/monkeypatch.rb
+- lib/jetpants/topology.rb
+- lib/jetpants/host.rb
+- lib/jetpants.rb
 - bin/jetpants
-- plugins/simple_tracker/commandsuite.rb
-- plugins/simple_tracker/topology.rb
-- plugins/simple_tracker/db.rb
-- plugins/simple_tracker/shard.rb
-- plugins/simple_tracker/pool.rb
-- plugins/simple_tracker/simple_tracker.rb
 - plugins/jetpants_collins/jetpants_collins.rb
-- plugins/jetpants_collins/topology.rb
-- plugins/jetpants_collins/db.rb
-- plugins/jetpants_collins/host.rb
-- plugins/jetpants_collins/asset.rb
 - plugins/jetpants_collins/shard.rb
+- plugins/jetpants_collins/db.rb
 - plugins/jetpants_collins/pool.rb
+- plugins/jetpants_collins/asset.rb
+- plugins/jetpants_collins/topology.rb
+- plugins/jetpants_collins/host.rb
+- plugins/simple_tracker/simple_tracker.rb
+- plugins/simple_tracker/commandsuite.rb
+- plugins/simple_tracker/shard.rb
+- plugins/simple_tracker/db.rb
+- plugins/simple_tracker/pool.rb
+- plugins/simple_tracker/topology.rb
 - etc/jetpants.yaml.sample
 homepage: https://github.com/tumblr/jetpants/
 licenses: []