jetpants 0.7.10 → 0.8.0

data/lib/jetpants/host.rb

@@ -38,7 +38,8 @@ module Jetpants
     end
     
     # Returns a Host object for the machine Jetpants is running on.
-    def self.local(interface='bond0')
+    def self.local(interface=false)
+      interface ||= Jetpants.private_interface
       # This technique is adapted from Sergio Rubio Gracia's, described at
       # http://blog.frameos.org/2006/12/09/getting-network-interface-addresses-using-ioctl-pure-ruby-2/
       sock = Socket.new(Socket::AF_INET, Socket::SOCK_DGRAM,0)
@@ -156,38 +157,6 @@ module Jetpants
       @available
     end
     
-    ###### ini file manipulation ###############################################
-    
-    # Comments-out lines of an ini file beginning with any of the supplied prefixes
-    def comment_out_ini(file, *prefixes)
-      toggle_ini(file, prefixes, false)
-    end
-    
-    # Un-comments-out lines of an ini file beginning with any of the supplied prefixes
-    # The prefixes should NOT include the # comment-out character -- ie, pass them
-    # the same as you would to DB#comment_out_ini
-    def uncomment_out_ini(file, *prefixes)
-      toggle_ini(file, prefixes, true)
-    end
-    
-    # Comments-out (if enable is true) or un-comments-out (if enable is false) lines of an ini file.
-    def toggle_ini(file, prefixes, enable)
-      prefixes.flatten!
-      commands = []
-      prefixes.each do |setting|
-        if enable
-          search = '^#(\s*%s\s*(?:=.*)?)$' % setting
-          replace = '\1'
-        else
-          search = '^(\s*%s\s*(?:=.*)?)$' % setting
-          replace = '#\1'
-        end
-        commands << "ruby -i -pe 'sub(%r[#{search}], %q[#{replace}])' #{file}"
-      end
-      cmd_line = commands.join '; '
-      ssh_cmd cmd_line
-    end
-    
     
     ###### Directory Copying / Listing / Comparison methods ####################
     
@@ -346,6 +315,22 @@ module Jetpants
       end
       total_size
     end
+
+    def mount_stats(mount)
+      mount_stats = {}
+
+      output = ssh_cmd "df -k " + mount + "|tail -1| awk '{print $2\",\"$3\",\"$4}'" 
+      if output
+        output = output.split(',').map{|s| s.to_i}
+
+        mount_stats['total'] = output[0] * 1024
+        mount_stats['used'] = output[1] * 1024
+        mount_stats['available'] = output[2] * 1024
+        return mount_stats
+      else
+        false
+      end
+    end
     
     
     ###### Misc methods ########################################################
@@ -358,8 +343,8 @@ module Jetpants
     # methods that call Host#service with :status operation (such as 
     # DB#probe_running) in a custom plugin, to parse the output properly on 
     # your chosen Linux distro.
-    def service(operation, name)
-      ssh_cmd "service #{name} #{operation.to_s}"
+    def service(operation, name, options='')
+      ssh_cmd "service #{name} #{operation.to_s} #{options}".rstrip
     end
     
     # Changes the I/O scheduler to name (such as 'deadline', 'noop', 'cfq')
@@ -376,6 +361,19 @@ module Jetpants
       true
     end
     
+    # Checks if there's a process with the given process ID running on this host.
+    # Optionally also checks if matching_string is contained in the process name.
+    # Returns true if so, false if not.
+    # Warning: this implementation assumes Linux-style "ps" command; will not work
+    # on BSD hosts.
+    def pid_running?(pid, matching_string=false)
+      if matching_string
+        ssh_cmd("ps --no-headers -o command #{pid} | grep '#{matching_string}' | wc -l").chomp.to_i > 0
+      else
+        ssh_cmd("ps --no-headers #{pid} | wc -l").chomp.to_i > 0
+      end
+    end
+    
     # Returns number of cores on machine. (reflects virtual cores if hyperthreading
     # enabled, so might be 2x real value in that case.)
     # Not currently used by anything in Jetpants base, but might be useful for plugins

data/lib/jetpants/monkeypatch.rb

@@ -1,5 +1,14 @@
 # This file contains any methods we're adding to core Ruby modules
 
+# Add a deep_merge method to Hash in order to more effectively join configs
+class Hash
+  def deep_merge!(other_hash)
+    merge!(other_hash) do |key, oldval, newval|
+      (oldval.class == self.class && newval.class == oldval.class) ? oldval.deep_merge!(newval) : newval
+    end
+  end
+end
+
 # Reopen Enumerable to add some concurrent iterators
 module Enumerable
   # Works like each but runs the block in a separate thread per item.

data/lib/jetpants/pool.rb

@@ -125,16 +125,21 @@ module Jetpants
     
     # Remove a slave from a pool entirely. This is destructive, ie, it does a
     # RESET SLAVE on the db.
+    #
     # Note that a plugin may want to override this (or implement after_remove_slave!)
     # to actually sync the change to an asset tracker, depending on how the plugin
     # implements Pool#sync_configuration. (If the implementation makes sync_configuration
     # work by iterating over the pool's current slaves to update their status/role/pool, it 
     # won't see any slaves that have been removed, and therefore won't update them.)
+    #
+    # This method has no effect on slaves that are unavailable via SSH or have MySQL
+    # stopped, because these are only considered to be in the pool if your asset tracker
+    # plugin intentionally adds them. Such plugins could also handle this in the
+    # after_remove_slave! callback.
     def remove_slave!(slave_db)
       raise "Slave is not in this pool" unless slave_db.pool == self
+      return false unless (slave_db.running? && slave_db.available?)
       slave_db.disable_monitoring
-      slave_db.stop_replication
-      slave_db.repl_binlog_coordinates # displays how far we replicated, in case you need to roll back this change manually
       slave_db.disable_replication!
       sync_configuration # may or may not be sufficient -- see note above.
     end
@@ -177,21 +182,23 @@ module Jetpants
       end
       
       binlog_pos = extended_info ? details[@master][:coordinates].join(':') : ''
-      print "\tmaster          = %-15s %-30s %s\n" % [@master.ip, @master.hostname, binlog_pos]
+      print "\tmaster          = %-15s %-32s %s\n" % [@master.ip, @master.hostname, binlog_pos]
       
       [:active, :standby, :backup].each do |type|
         slave_list = slaves(type)
         slave_list.sort.each_with_index do |s, i|
           binlog_pos = extended_info ? details[s][:coordinates].join(':') : ''
           slave_lag = extended_info ? "lag=#{details[s][:lag]}" : ''
-          print "\t%-7s slave #{i + 1} = %-15s %-30s %-26s %s\n" % [type, s.ip, s.hostname, binlog_pos, slave_lag]
+          print "\t%-7s slave #{i + 1} = %-15s %-32s %-26s %s\n" % [type, s.ip, s.hostname, binlog_pos, slave_lag]
         end
       end
       true
     end
     
     # Demotes the pool's existing master, promoting a slave in its place.
-    def master_promotion!(promoted)
+    # The old master will become a slave of the new master if enslave_old_master is true,
+    # unless the old master is unavailable/crashed.
+    def master_promotion!(promoted, enslave_old_master=true)
       demoted = @master
       raise "Demoted node is already the master of this pool!" if demoted == promoted
       raise "Promoted host is not in the right pool!" unless demoted.slaves.include?(promoted)
@@ -201,7 +208,7 @@ module Jetpants
       # If demoted machine is available, confirm it is read-only and binlog isn't moving,
       # and then wait for slaves to catch up to this position
       if demoted.running?
-        demoted.enable_read_only! unless demoted.read_only?
+        demoted.enable_read_only!
         raise "Unable to enable global read-only mode on demoted machine" unless demoted.read_only?
         coordinates = demoted.binlog_coordinates
         raise "Demoted machine still taking writes (from superuser or replication?) despite being read-only" unless coordinates == demoted.binlog_coordinates
@@ -241,7 +248,7 @@ module Jetpants
       
       # gather our new replicas
       replicas.delete promoted
-      replicas << demoted if demoted.running?
+      replicas << demoted if demoted.running? && enslave_old_master
       
       # perform promotion
       replicas.each do |r|

data/lib/jetpants/shard.rb

@@ -34,8 +34,8 @@ module Jetpants
     #   :replicating    --  Child shard that is being cloned to new replicas. Shard not in production yet.
     #   :child          --  Child shard that is in production for reads, but still slaving from its parent for writes.
     #   :needs_cleanup  --  Child shard that is fully in production, but parent replication not torn down yet, and redundant data (from wrong range) not removed yet
-    #   :deprecated     --  Parent shard that has been split but children are still in :child or :needs_cleanup state. Shard may still be in production for writes.
-    #   :recycle        --  Parent shard that has been split and children are now in the :ready state. Shard no longer in production.
+    #   :deprecated     --  Parent shard that has been split but children are still in :child or :needs_cleanup state. Shard may still be in production for writes / replication not torn down yet.
+    #   :recycle        --  Parent shard that has been split and children are now in the :ready state. Shard no longer in production, replication to children has been torn down.
     attr_accessor :state
     
     # Constructor for Shard --
@@ -126,9 +126,9 @@ module Jetpants
     # If you omit id_ranges, the parent's ID range will be divided evenly amongst the
     # children automatically.
     def init_children(count, id_ranges=false)
-      # Make sure we have enough machines in spare pool
-      raise "Not enough master role machines in spare pool!" if count > Jetpants.topology.count_spares(role: 'master')
-      raise "Not enough standby_slave role machines in spare pool!" if count * Jetpants.standby_slaves_per_pool > Jetpants.topology.count_spares(role: 'standby_slave')
+      # Make sure we have enough machines (of correct hardware spec and role) in spare pool
+      raise "Not enough master role machines in spare pool!" if count > Jetpants.topology.count_spares(role: :master, like: master)
+      raise "Not enough standby_slave role machines in spare pool!" if count * Jetpants.standby_slaves_per_pool > Jetpants.topology.count_spares(role: :standby_slave, like: slaves.first)
       
       # Make sure enough slaves of shard being split
       raise "Must have at least #{Jetpants.standby_slaves_per_pool} slaves of shard being split" if master.slaves.count < Jetpants.standby_slaves_per_pool
@@ -149,7 +149,7 @@ module Jetpants
       end
       
       count.times do |i|
-        spare = Jetpants.topology.claim_spare(role: 'master')
+        spare = Jetpants.topology.claim_spare(role: :master, like: master)
         spare.disable_read_only! if (spare.running? && spare.read_only?)
         spare.output "Using ID range of #{id_ranges[i][0]} to #{id_ranges[i][1]} (inclusive)"
         s = Shard.new(id_ranges[i][0], id_ranges[i][1], spare, :initializing)
@@ -170,7 +170,6 @@ module Jetpants
       
       init_children(pieces) unless @children.count > 0
       
-      @children.concurrent_each {|c| c.disable_binary_logging}
       clone_to_children!
       @children.concurrent_each {|c| c.rebuild!}
       @children.each {|c| c.sync_configuration}
@@ -208,7 +207,6 @@ module Jetpants
           raise "Child shard master #{child_shard.master} is already a slave of another pool"
         elsif child_shard.master.is_slave?
           child_shard.output "Already slaving from parent shard master"
-          child_shard.restart_mysql # to make previous disable of binary logging take effect
         else
           targets << child_shard.master
         end
@@ -233,12 +231,12 @@ module Jetpants
       raise "Cannot rebuild a shard that isn't still slaving from another shard" unless @master.is_slave?
       raise "Cannot rebuild an active shard" if in_config?
       
-      stop_query_killer
       tables = Table.from_config 'sharded_tables'
       
       if [:initializing, :exporting].include? @state
         @state = :exporting
         sync_configuration
+        stop_query_killer
         export_schemata tables
         export_data tables, @min_id, @max_id
       end
@@ -248,17 +246,17 @@ module Jetpants
         sync_configuration
         import_schemata!
         alter_schemata if respond_to? :alter_schemata
+        restart_mysql '--skip-log-bin', '--skip-log-slave-updates', '--innodb-autoinc-lock-mode=2', '--skip-slave-start'
         import_data tables, @min_id, @max_id
+        restart_mysql # to clear out previous options '--skip-log-bin', '--skip-log-slave-updates', '--innodb-autoinc-lock-mode=2'
         start_query_killer
       end
       
       if [:importing, :replicating].include? @state
-        enable_binary_logging
-        restart_mysql
         @state = :replicating
         sync_configuration
         if Jetpants.standby_slaves_per_pool > 0
-          my_slaves = Jetpants.topology.claim_spares(Jetpants.standby_slaves_per_pool, role: 'standby_slave')
+          my_slaves = Jetpants.topology.claim_spares(Jetpants.standby_slaves_per_pool, role: :standby_slave, like: parent.slaves.first)
           enslave!(my_slaves)
           my_slaves.each {|slv| slv.resume_replication}
           [self, my_slaves].flatten.each {|db| db.catch_up_to_master}

data/lib/jetpants/table.rb

@@ -47,6 +47,10 @@ module Jetpants
     # 'sharding_key' (or equivalently 'primary_key'), 'chunks', and 'order_by'.
     def initialize(name, params={})
       @name = name
+      parse_params(params)
+    end
+
+    def parse_params(params = {})
       params['sharding_key'] ||= params['primary_keys'] || params['primary_key'] || 'user_id'
       @sharding_keys = (params['sharding_key'].is_a?(Array) ? params['sharding_key'] : [params['sharding_key']])
       @chunks = params['chunks'] || 1

data/lib/jetpants/topology.rb

@@ -71,18 +71,23 @@ module Jetpants
     synchronized
     # Plugin should override so that this returns an array of [count] Jetpants::DB
     # objects, or throws an exception if not enough left.
-    # Options hash is plugin-specific. The only assumed option used by the rest of
-    # Jetpants is :role of 'MASTER' or 'STANDBY_SLAVE', for grabbing hardware
-    # suited for a particular purpose. This can be ignored if your hardware is
-    # entirely uniform and/or a burn-in process is already performed on all new
-    # hardware intakes.
+    #
+    # Options hash is plugin-specific. Jetpants core will provide these two options,
+    # but it's up to a plugin to handle (or ignore) them:
+    #
+    # :role   =>  :master or :standby_slave, indicating what purpose the new node(s)
+    #             will be used for. Useful if your hardware spec varies by node role
+    #             (not recommended!) or if you vet your master candidates more carefully.
+    # :like   =>  a Jetpants::DB object, indicating that the spare node hardware spec
+    #             should be like the specified DB's spec.
     def claim_spares(count, options={})
       raise "Plugin must override Topology#claim_spares"
     end
     
     synchronized
     # Plugin should override so that this returns a count of spare machines
-    # matching the selected options.
+    # matching the selected options. options hash follows same format as for
+    # Topology#claim_spares.
     def count_spares(options={})
       raise "Plugin must override Topology#count_spares"
     end
@@ -109,12 +114,13 @@ module Jetpants
       @pools.reject {|p| p.is_a? Shard}
     end
     
-    # Finds and returns a single Jetpants::Pool. Target may be a name (string) or master (DB object).
+    # Finds and returns a single Jetpants::Pool. Target may be a name (string, case insensitive)
+    # or master (DB object).
     def pool(target)
       if target.is_a?(DB)
         @pools.select {|p| p.master == target}.first
       else
-        @pools.select {|p| p.name == target}.first
+        @pools.select {|p| p.name.downcase == target.downcase}.first
       end
     end
     
@@ -137,8 +143,21 @@ module Jetpants
     end
     
     # Returns the Jetpants::Shard that handles the given ID.
+    # During a shard split, if the child isn't "in production" yet (ie, it's
+    # still being built), this will always return the parent shard. Once the
+    # child is fully built / in production, this method will always return
+    # the child shard. However, Shard#db(:write) will correctly delegate writes
+    # to the parent shard when appropriate in this case. (see also: Topology#shard_db_for_id)
     def shard_for_id(id)
-      @shards.select {|s| s.min_id <= id && (s.max_id == 'INFINITY' || s.max_id >= id)}[0]
+      choices = shards.select {|s| s.min_id <= id && (s.max_id == 'INFINITY' || s.max_id >= id)}
+      choices.reject! {|s| s.parent && ! s.in_config?} # filter out child shards that are still being built
+      
+      # Preferentially return child shards at this point
+      if choices.any? {|s| s.parent}
+        choices.select {|s| s.parent}.first
+      else
+        choices.first
+      end
     end
     
     # Returns the Jetpants::DB that handles the given ID with the specified

data/plugins/jetpants_collins/db.rb

@@ -69,9 +69,21 @@ module Jetpants
     
     ##### NEW METHODS ##########################################################
     
+    # Returns true if this database is located in the same datacenter as jetpants_collins
+    # has been figured for, false otherwise.
     def in_remote_datacenter?
       @host.collins_location != Plugin::JetCollins.datacenter
     end
     
+    # Returns true if this database is a spare node and looks ready for use, false otherwise.
+    # The default implementation just ensures a collins status of Provisioned.
+    # Downstream plugins may override this to do additional checks to ensure the node is
+    # in a sane state. (The caller of this method already checks that the node is SSHable,
+    # and that MySQL is running, and the node isn't already in a pool -- so no need to
+    # check any of those here.)
+    def usable_spare?
+      collins_status.downcase == 'provisioned'
+    end
+    
   end
 end

data/plugins/jetpants_collins/jetpants_collins.rb

@@ -62,7 +62,7 @@ module Jetpants
             end
             
             # We make these 4 accessors available to ANY class including this mixin
-            collins_attr_accessor :primary_role, :secondary_role, :pool, :status
+            collins_attr_accessor :primary_role, :secondary_role, :pool, :status, :state
           end
         end
         
@@ -137,12 +137,18 @@ module Jetpants
         asset = collins_asset
         if field_names.count > 1 || field_names[0].is_a?(Array)
           field_names.flatten!
+          want_state = !! field_names.delete(:state)
           results = Hash[field_names.map {|field| [field, (asset ? asset.send(field) : '')]}]
+          results[:state] = asset.state.name if want_state
           results[:asset] = asset
           results
         elsif field_names.count == 1
           return '' unless asset
-          asset.send field_names[0]
+          if field_names[0] == :state
+            asset.state.name
+          else
+            asset.send field_names[0]
+          end
         else
           nil
         end
@@ -174,11 +180,32 @@ module Jetpants
               output "WARNING: unable to set Collins status to #{val}"
               next
             end
-            previous_value = asset.status
-            if previous_value != val.to_s
-              success = Jetpants::Plugin::JetCollins.set_status!(asset, val)
-              raise "#{self}: Unable to set Collins status to #{val}" unless success
-              output "Collins status changed from #{previous_value} to #{val}"
+            if attrs[:state]
+              previous_state = asset.state.name
+              previous_status = asset.status
+              if previous_state != attrs[:state].to_s || previous_status != attrs[:status].to_s
+                success = Jetpants::Plugin::JetCollins.set_status!(asset, attrs[:status], 'changed through jetpants', attrs[:state])
+                unless success
+                  Jetpants::Plugin::JetCollins.state_create!(attrs[:state], attrs[:state], attrs[:state], attrs[:status])
+                  success = Jetpants::Plugin::JetCollins.set_status!(asset, attrs[:status], 'changed through jetpants', attrs[:state])
+                end
+                raise "#{self}: Unable to set Collins state to #{attrs[:state]} and Unable to set Collins status to #{attrs[:status]}" unless success
+                output "Collins state changed from #{previous_state} to #{attrs[:state]}"
+                output "Collins status changed from #{previous_status} to #{attrs[:status]}"
+              end              
+            else
+              previous_value = asset.status
+              if previous_value != val.to_s
+                success = Jetpants::Plugin::JetCollins.set_status!(asset, val)
+                raise "#{self}: Unable to set Collins status to #{val}" unless success
+                output "Collins status changed from #{previous_value} to #{val}"
+              end
+            end
+          when :state
+            unless asset && asset.status && attrs[:status]
+              raise "#{self}: Unable to set state without settings a status" unless attrs[:status]
+              output "WARNING: unable to set Collins state to #{val}"
+              next
             end
           else
             unless asset

data/plugins/jetpants_collins/pool.rb

@@ -111,7 +111,7 @@ module Jetpants
     
     # If the demoted master was offline, record some info in Collins, otherwise
     # there will be 2 masters listed
-    def after_master_promotion!(promoted)
+    def after_master_promotion!(promoted, enslave_old_master=true)
       Jetpants.topology.clear_asset_cache
       
       # Find the master asset(s) for this pool, filtering down to only current datacenter
@@ -119,9 +119,13 @@ module Jetpants
       assets.reject! {|a| a.location && a.location.upcase != Plugin::JetCollins.datacenter}
       assets.map(&:to_db).each do |db|
         if db != @master || !db.running?
-          db.collins_status = 'Maintenance'
           db.collins_pool = ''
           db.collins_secondary_role = ''
+          if enslave_old_master
+            db.output 'REMINDER: you must manually put this host into Maintenance status in Collins' unless db.collins_status.downcase == 'maintenance'
+          else
+            db.collins_status = 'Unallocated'
+          end
         end
       end
     end