jetpants 0.7.10 → 0.8.0

data/lib/jetpants/db/privileges.rb

@@ -7,7 +7,8 @@ module Jetpants
   class DB
     # Create a MySQL user. If you omit parameters, the defaults from Jetpants'
     # configuration will be used instead.  Does not automatically grant any
-    # privileges; use DB#grant_privileges for that.
+    # privileges; use DB#grant_privileges for that.  Intentionally cannot
+    # create a passwordless user. 
     def create_user(username=false, password=false, skip_binlog=false)
       username ||= app_credentials[:user]
       password ||= app_credentials[:pass]
@@ -19,6 +20,11 @@ module Jetpants
       commands << "FLUSH PRIVILEGES"
       commands = commands.join '; '
       mysql_root_cmd commands, schema: true
+      Jetpants.mysql_grant_ips.each do |ip|
+        message = "Created user '#{username}'@'#{ip}'"
+        message += ' (only on this node - skipping binlog!)' if skip_binlog
+        output message
+      end
     end
     
     # Drops a user. Can optionally make this statement skip replication, if you
@@ -33,6 +39,11 @@ module Jetpants
       commands << "FLUSH PRIVILEGES"
       commands = commands.join '; '
       mysql_root_cmd commands, schema: true
+      Jetpants.mysql_grant_ips.each do |ip|
+        message = "Dropped user '#{username}'@'#{ip}'"
+        message += ' (only on this node - skipping binlog!)' if skip_binlog
+        output message
+      end
     end
     
     # Grants privileges to the given username for the specified database.
@@ -64,30 +75,68 @@ module Jetpants
       commands << "FLUSH PRIVILEGES"
       commands = commands.join '; '
       mysql_root_cmd commands, schema: true
+      Jetpants.mysql_grant_ips.each do |ip|
+        verb = (statement.downcase == 'revoke' ? 'Revoking' : 'Granting')
+        target_db = (database == '*' ? 'globally' : "on #{database}.*")
+        output "#{verb} privileges #{preposition.downcase} '#{username}'@'#{ip}' #{target_db}: #{privileges.downcase}"
+      end
     end
     
     # Disables access to a DB by the application user, and sets the DB to 
     # read-only. Useful when decommissioning instances from a shard that's
-    # been split.
+    # been split, or a former slave that's been permanently removed from the pool
     def revoke_all_access!
       user_name = app_credentials[:user]
       enable_read_only!
-      output "Revoking access for user #{user_name}."
-      output(drop_user(user_name, true)) # drop the user without replicating the drop statement to slaves
+      drop_user(user_name, true) # drop the user without replicating the drop statement to slaves
     end
     
     # Enables global read-only mode on the database.
     def enable_read_only!
-      output "Enabling global read_only mode"
-      mysql_root_cmd 'SET GLOBAL read_only = 1' unless read_only?
-      read_only?
+      if read_only?
+        output "Node already has read_only mode enabled"
+        true
+      else
+        output "Enabling read_only mode"
+        mysql_root_cmd 'SET GLOBAL read_only = 1'
+        read_only?
+      end
     end
     
     # Disables global read-only mode on the database.
     def disable_read_only!
-      output "Disabling global read_only mode"
-      mysql_root_cmd 'SET GLOBAL read_only = 0' if read_only?
-      not read_only?
+      if read_only?
+        output "Disabling read_only mode"
+        mysql_root_cmd 'SET GLOBAL read_only = 0'
+        not read_only?
+      else
+        output "Confirmed that read_only mode is already disabled"
+        true
+      end
+    end
+    
+    # Generate and return a random string consisting of uppercase
+    # letters, lowercase letters, and digits.
+    def self.random_password(length=50)
+      chars = [('a'..'z'), ('A'..'Z'), (0..9)].map(&:to_a).flatten
+      (1..length).map{ chars[rand(chars.length)] }.join
+    end
+
+    # override Jetpants.mysql_grant_ips temporarily before executing a block
+    # then set Jetpants.mysql_grant_ips back to the original values
+    #   eg. master.override_mysql_grant_ips(['10.10.10.10']) do
+    #         #something
+    #       end
+    def override_mysql_grant_ips(ips)
+      ip_holder = Jetpants.mysql_grant_ips
+      Jetpants.mysql_grant_ips = ips
+      begin
+        yield
+      rescue StandardError, Interrupt, IOError
+        Jetpants.mysql_grant_ips = ip_holder
+        raise
+      end
+      Jetpants.mysql_grant_ips = ip_holder
     end
     
   end

data/lib/jetpants/db/replication.rb

@@ -48,44 +48,67 @@ module Jetpants
     # Pauses replication
     def pause_replication
       raise "This DB object has no master" unless master
-      return if @repl_paused
       output "Pausing replication from #{@master}."
-      output mysql_root_cmd "STOP SLAVE"
-      @repl_paused = true
+      if @repl_paused
+        output "Replication was already paused."
+        repl_binlog_coordinates(true)
+      else
+        output mysql_root_cmd "STOP SLAVE"
+        repl_binlog_coordinates(true)
+        @repl_paused = true
+      end
     end
     alias stop_replication pause_replication
     
     # Starts replication, or restarts replication after a pause
     def resume_replication
       raise "This DB object has no master" unless master
+      repl_binlog_coordinates(true)
       output "Resuming replication from #{@master}."
       output mysql_root_cmd "START SLAVE"
       @repl_paused = false
     end
     alias start_replication resume_replication
     
+    # Stops replication at the same coordinates on two nodes
+    def pause_replication_with(sibling)
+      [self, sibling].each &:pause_replication
+      
+      # self and sibling at same coordinates: all done
+      return true if repl_binlog_coordinates == sibling.repl_binlog_coordinates
+      
+      # self ahead of sibling: handle via recursion with roles swapped
+      return sibling.pause_replication_with(self) if ahead_of? sibling
+      
+      # sibling ahead of self: catch up to sibling
+      sibling_coords = sibling.repl_binlog_coordinates
+      output "Resuming replication from #{@master} until (#{sibling_coords[0]}, #{sibling_coords[1]})."
+      output(mysql_root_cmd "START SLAVE UNTIL MASTER_LOG_FILE = '#{sibling_coords[0]}', MASTER_LOG_POS = #{sibling_coords[1]}")
+      sleep 1 while repl_binlog_coordinates != sibling_coords
+      true
+    end
+    
     # Permanently disables replication. Clears out the SHOW SLAVE STATUS output
     # entirely in MySQL versions that permit this.
     def disable_replication!
-      raise "This DB object has no master" unless master
+      stop_replication
       output "Disabling replication; this db is no longer a slave."
-
       ver = version_tuple
-
+      
       # MySQL < 5.5: allows master_host='', which clears out SHOW SLAVE STATUS
       if ver[0] == 5 && ver[1] < 5
-        output mysql_root_cmd "STOP SLAVE; CHANGE MASTER TO master_host=''; RESET SLAVE"
-
+        output mysql_root_cmd "CHANGE MASTER TO master_host=''; RESET SLAVE"
+      
       # MySQL 5.5.16+: allows RESET SLAVE ALL, which clears out SHOW SLAVE STATUS
       elsif ver[0] >= 5 && (ver[0] > 5 || ver[1] >= 5) && (ver[0] > 5 || ver[1] > 5 || ver[2] >= 16)
-        output mysql_root_cmd "STOP SLAVE; CHANGE MASTER TO master_user='test'; RESET SLAVE ALL"
-
+        output mysql_root_cmd "CHANGE MASTER TO master_user='test'; RESET SLAVE ALL"
+      
       # Other versions: no safe way to clear out SHOW SLAVE STATUS.  Still set master_user to 'test'
       # so that we know to ignore the slave status output.
       else
-        output mysql_root_cmd "STOP SLAVE; CHANGE MASTER TO master_user='test'; RESET SLAVE"
+        output mysql_root_cmd "CHANGE MASTER TO master_user='test'; RESET SLAVE"
       end
-
+      
       @master.slaves.delete(self) rescue nil
       @master = nil
       @repl_paused = nil
@@ -112,6 +135,7 @@ module Jetpants
                             log_pos:  pos, 
                             user:     repl_user, 
                             password: repl_pass  )
+        t.enable_read_only!
       end
       resume_replication if @master # should already have happened from the clone_to! restart anyway, but just to be explicit
       enable_monitoring
@@ -133,6 +157,7 @@ module Jetpants
                             log_pos:  pos,
                             user:     replication_credentials[:user],
                             password: replication_credentials[:pass]  )
+        t.enable_read_only!
       end
       resume_replication # should already have happened from the clone_to! restart anyway, but just to be explicit
       catch_up_to_master
@@ -240,18 +265,42 @@ module Jetpants
       user && pass ? {user: user, pass: pass} : Jetpants.replication_credentials
     end
     
-    # Disables binary logging in my.cnf.  Does not take effect until you restart
-    # mysql.
+    # This method is no longer supported. It used to manipulate /etc/my.cnf directly, which was too brittle.
+    # You can achieve the same effect by passing parameters to DB#restart_mysql.
     def disable_binary_logging
-      output "Disabling binary logging in MySQL configuration; will take effect at next restart"
-      comment_out_ini(mysql_config_file, 'log-bin', 'log-slave-updates')
+      raise "DB#disable_binary_logging is no longer supported, please use DB#restart_mysql('--skip-log-bin', '--skip-log-slave-updates') instead"
     end
     
-    # Re-enables binary logging in my.cnf after a prior call to disable_bin_log.
-    # Does not take effect until you restart mysql.
+    # This method is no longer supported. It used to manipulate /etc/my.cnf directly, which was too brittle.
+    # You can achieve the same effect by passing (or NOT passing) parameters to DB#restart_mysql.
     def enable_binary_logging
-      output "Re-enabling binary logging in MySQL configuration; will take effect at next restart"
-      uncomment_out_ini(mysql_config_file, 'log-bin', 'log-slave-updates')
+      raise "DB#enable_binary_logging is no longer supported, please use DB#restart_mysql() instead"
+    end
+    
+    # Return true if this node's replication progress is ahead of the provided
+    # node, or false otherwise. The nodes must be in the same pool for coordinates
+    # to be comparable. Does not work in hierarchical replication scenarios!
+    def ahead_of?(node)
+      my_pool = pool(true)
+      raise "Node #{node} is not in the same pool as #{self}" unless node.pool(true) == my_pool
+      
+      my_coords   = (my_pool.master == self ? binlog_coordinates      : repl_binlog_coordinates)
+      node_coords = (my_pool.master == node ? node.binlog_coordinates : node.repl_binlog_coordinates)
+      
+      # Same coordinates
+      if my_coords == node_coords
+        false
+      
+      # Same logfile: simply compare position
+      elsif my_coords[0] == node_coords[0]
+        my_coords[1] > node_coords[1]
+        
+      # Different logfile
+      else
+        my_logfile_num = my_coords[0].match(/^[a-zA-Z.0]+(\d+)$/)[1].to_i
+        node_logfile_num = node_coords[0].match(/^[a-zA-Z.0]+(\d+)$/)[1].to_i
+        my_logfile_num > node_logfile_num
+      end
     end
     
   end

data/lib/jetpants/db/server.rb

@@ -9,28 +9,44 @@ module Jetpants
     # OK to use this if MySQL is already stopped; it's a no-op then.
     def stop_mysql
       output "Attempting to shutdown MySQL"
+      disconnect if @db
       output service(:stop, 'mysql')
       running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
       raise "[#{@ip}] Failed to shut down MySQL: Something is still listening on port #{@port}" unless running.chomp == '0'
+      @options = []
       @running = false
     end
     
     # Starts MySQL, and confirms that something is now listening on the port.
     # Raises an exception if MySQL is already running or if something else is
     # already running on its port.
-    def start_mysql
-      @repl_paused = false if @master
+    # Options should be supplied as positional method args, for example:
+    #   start_mysql '--skip-networking', '--skip-grant-tables'
+    def start_mysql(*options)
+      if @master
+        @repl_paused = options.include?('--skip-slave-start')
+      end
       running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
       raise "[#{@ip}] Failed to start MySQL: Something is already listening on port #{@port}" unless running.chomp == '0'
-      output "Attempting to start MySQL"
-      output service(:start, 'mysql')
+      if options.size == 0
+        output "Attempting to start MySQL, no option overrides supplied"
+      else
+        output "Attempting to start MySQL with options #{options.join(' ')}"
+      end
+      output service(:start, 'mysql', options.join(' '))
+      @options = options
       confirm_listening
       @running = true
+      if role == :master && ! @options.include?('--skip-networking')
+        disable_read_only!
+      end
     end
     
     # Restarts MySQL.
-    def restart_mysql
-      @repl_paused = false if @master
+    def restart_mysql(*options)
+      if @master
+        @repl_paused = options.include?('--skip-slave-start')
+      end
       
       # Disconnect if we were previously connected
       user, schema = false, false
@@ -39,13 +55,21 @@ module Jetpants
         disconnect
       end
       
-      output "Attempting to restart MySQL"
-      output service(:restart, 'mysql')
+      if options.size == 0
+        output "Attempting to restart MySQL, no option overrides supplied"
+      else
+        output "Attempting to restart MySQL with options #{options.join(' ')}"
+      end
+      output service(:restart, 'mysql', options.join(' '))
+      @options = options
       confirm_listening
       @running = true
-      
-      # Reconnect if we were previously connected
-      connect(user: user, schema: schema) if user || schema
+      unless @options.include?('--skip-networking')
+        disable_read_only! if role == :master
+        
+        # Reconnect if we were previously connected
+        connect(user: user, schema: schema) if user || schema
+      end
     end
     
     # Has no built-in effect. Plugins can override it, and/or implement
@@ -60,7 +84,12 @@ module Jetpants
     
     # Confirms that a process is listening on the DB's port
     def confirm_listening(timeout=10)
-      confirm_listening_on_port(@port, timeout)
+      if @options.include? '--skip-networking'
+        output 'Unable to confirm mysqld listening because server started with --skip-networking'
+        false
+      else
+        confirm_listening_on_port(@port, timeout)
+      end
     end
     
     # Returns the MySQL data directory for this instance. A plugin can override this
@@ -69,13 +98,6 @@ module Jetpants
       '/var/lib/mysql'
     end
     
-    # Returns the MySQL server configuration file for this instance. A plugin can
-    # override this if needed, especially if running multiple MySQL instances on
-    # the same host.
-    def mysql_config_file
-      '/etc/my.cnf'
-    end
-    
     # Has no built-in effect. Plugins can override it, and/or implement
     # before_enable_monitoring and after_enable_monitoring callbacks.
     def enable_monitoring(*services)

data/lib/jetpants/db/state.rb

@@ -105,6 +105,28 @@ module Jetpants
       sleep(interval)
       global_status[:Connections].to_i - conn_counter > threshold
     end
+
+    # Gets the max theads connected over a time period
+    def max_threads_running(tries=8, interval=1.0)
+      poll_status_value(:Threads_running,:max, tries, interval)
+    end
+
+    # Gets the max or avg for a mysql value
+    def poll_status_value(field, type=:max, tries=8, interval=1.0)
+      max = 0
+      sum = 0
+      tries.times do
+        value = global_status[field].to_i
+        max = value unless max > value
+        sum += value
+        sleep(interval)
+      end
+      if type == :max
+        max
+      elsif type == :avg
+        sum.to_f/tries.to_f
+      end
+    end
     
     # Confirms the binlog of this node has not moved during a duration
     # of [interval] seconds.
@@ -134,6 +156,27 @@ module Jetpants
       @host.hostname.start_with? 'backup'
     end
     
+    # Returns true if the node can be promoted to be the master of its pool,
+    # false otherwise (also false if node is ALREADY the master)
+    # Don't use this in hierarchical replication scenarios, result may be
+    # unexpected.
+    def promotable_to_master?(detect_version_mismatches=true)
+      # backup_slaves are non-promotable
+      return false if for_backups?
+      
+      # already the master
+      p = pool(true)
+      return false if p.master == self
+      
+      # ordinarily, cannot promote a slave that's running a higher version of
+      # MySQL than any other node in the pool.
+      if detect_version_mismatches
+        p.nodes.all? {|db| db == self || !db.available? || db.version_cmp(self) >= 0}
+      else
+        true
+      end
+    end
+    
     # Returns a hash mapping global MySQL variables (as symbols)
     # to their values (as strings).
     def global_variables
@@ -155,15 +198,51 @@ module Jetpants
     # Returns an array of integers representing the version of the MySQL server.
     # For example, Percona Server 5.5.27-rel28.1-log would return [5, 5, 27]
     def version_tuple
-      raise "Cannot determine version of a stopped MySQL instance" unless running?
-      global_variables[:version].split('.', 3).map &:to_i
+      result = nil
+      if running?
+        # If the server is running, we can just query it
+        result = global_variables[:version].split('.', 3).map(&:to_i) rescue nil
+      end
+      if result.nil?
+        # Otherwise we need to parse the output of mysqld --version
+        output = ssh_cmd 'mysqld --version'
+        matches = output.downcase.match('ver\s*(\d+)\.(\d+)\.(\d+)')
+        raise "Unable to determine version for #{self}" unless matches
+        result = matches[1, 3].map(&:to_i)
+      end
+      result
+    end
+    
+    # Return a string representing the version. The precision indicates how
+    # many major/minor version numbers to return.
+    # ie, on 5.5.29, normalized_version(3) returns '5.5.29',
+    # normalized_version(2) returns '5.5', and normalized_version(1) returns '5'
+    def normalized_version(precision=2)
+      raise "Invalid precision #{precision}" if precision < 1 || precision > 3
+      version_tuple[0, precision].join('.')
+    end
+    
+    # Returns -1 if self is running a lower version than db; 1 if self is running
+    # a higher version; and 0 if running same version.
+    def version_cmp(db, precision=2)
+      raise "Invalid precision #{precision}" if precision < 1 || precision > 3
+      my_tuple = version_tuple[0, precision]
+      other_tuple = db.version_tuple[0, precision]
+      my_tuple.each_with_index do |subver, i|
+        return -1 if subver < other_tuple[i]
+        return 1 if subver > other_tuple[i]
+      end
+      0
     end
     
     # Returns the Jetpants::Pool that this instance belongs to, if any.
     # Can optionally create an anonymous pool if no pool was found. This anonymous
     # pool intentionally has a blank sync_configuration implementation.
     def pool(create_if_missing=false)
-      result = Jetpants.topology.pool(self) || Jetpants.topology.pool(master)
+      result = Jetpants.topology.pool(self)
+      if !result && master
+        result ||= Jetpants.topology.pool(master)
+      end
       if !result && create_if_missing
         pool_master = master || self
         result = Pool.new('anon_pool_' + pool_master.ip.tr('.', ''), pool_master)
@@ -178,10 +257,15 @@ module Jetpants
     # Note that we consider a node with no master and no slaves to be
     # a :master, since we can't determine if it had slaves but they're
     # just offline/dead, vs it being an orphaned machine.
+    #
+    # In hierarchical replication scenarios (such as the child shard
+    # masters in the middle of a shard split), we return :master if
+    # Jetpants.topology considers the node to be the master for a pool.
     def role
       p = pool
       case
-      when !@master then :master
+      when !@master then :master                                # nodes that aren't slaves (including orphans) 
+      when p.master == self then :master                        # nodes that the topology thinks are masters
       when for_backups? then :backup_slave
       when p && p.active_slave_weights[self] then :active_slave # if pool in topology, determine based on expected/ideal state
       when !p && !is_standby? then :active_slave                # if pool missing from topology, determine based on actual state
@@ -194,10 +278,15 @@ module Jetpants
     # a metric gigabyte.  This puts it on the same scale as the output to tools like 
     # "du -h" and "df -h".
     def data_set_size(in_gb=false)
-      bytes = dir_size("#{mysql_directory}/#{app_schema}")
+      bytes = dir_size("#{mysql_directory}/#{app_schema}") + dir_size("#{mysql_directory}/ibdata1")
       in_gb ? (bytes / 1073741824.0).round : bytes
     end
     
+    def mount_stats(mount=false)
+      mount ||= mysql_directory
+
+      host.mount_stats(mount)
+    end
     
     ###### Private methods #####################################################
     
@@ -230,9 +319,11 @@ module Jetpants
       else
         @master = self.class.new(status[:master_host], status[:master_port])
         if status[:slave_io_running] != status[:slave_sql_running]
-          message = "One replication thread is stopped and the other is not"
-          raise "#{self}: #{message}" if Jetpants.verify_replication
-          output message
+          output "One replication thread is stopped and the other is not."
+          if Jetpants.verify_replication
+            output "You must repair this node manually, OR remove it from its pool permanently if it is unrecoverable."
+            raise "Fatal replication problem on #{self}"
+          end
           pause_replication
         else
           @repl_paused = (status[:slave_io_running].downcase == 'no')