jetpants 0.7.0

data/lib/jetpants/db/privileges.rb

@@ -0,0 +1,89 @@
+module Jetpants
+  
+  #--
+  # User / Grant manipulation methods ##########################################
+  #++
+  
+  class DB
+    # Create a MySQL user. If you omit parameters, the defaults from Jetpants'
+    # configuration will be used instead.  Does not automatically grant any
+    # privileges; use DB#grant_privileges for that.
+    def create_user(username=false, database=false, password=false)
+      username ||= Jetpants.app_credentials[:user]
+      database ||= Jetpants.mysql_schema
+      password ||= Jetpants.app_credentials[:pass]
+      commands = []
+      Jetpants.mysql_grant_ips.each do |ip|
+        commands << "CREATE USER '#{username}'@'#{ip}' IDENTIFIED BY '#{password}'"
+      end
+      commands << "FLUSH PRIVILEGES"
+      mysql_root_cmd(commands.join '; ')
+    end
+    
+    # Drops a user. Can optionally make this statement skip replication, if you
+    # want to drop a user on master and not on its slaves.
+    def drop_user(username=false, skip_binlog=false)
+      username ||= Jetpants.app_credentials[:user]
+      commands = []
+      commands << 'SET sql_log_bin = 0' if skip_binlog
+      Jetpants.mysql_grant_ips.each do |ip|
+        commands << "DROP USER '#{username}'@'#{ip}'"
+      end
+      commands << "FLUSH PRIVILEGES"
+      mysql_root_cmd(commands.join '; ')
+    end
+    
+    # Grants privileges to the given username for the specified database.
+    # Pass in privileges as additional params, each as strings.
+    # You may omit parameters to use the defaults in the Jetpants config file.
+    def grant_privileges(username=false, database=false, *privileges)
+      grant_or_revoke_privileges('GRANT', username, database, privileges)
+    end
+    
+    # Revokes privileges from the given username for the specified database.
+    # Pass in privileges as additional params, each as strings.
+    # You may omit parameters to use the defaults in the Jetpants config file.
+    def revoke_privileges(username=false, database=false, *privileges)
+      grant_or_revoke_privileges('REVOKE', username, database, privileges)
+    end
+    
+    # Helper method that can do grants or revokes.
+    def grant_or_revoke_privileges(statement, username, database, privileges)
+      preposition = (statement.downcase == 'revoke' ? 'FROM' : 'TO')
+      username ||= Jetpants.app_credentials[:user]
+      database ||= Jetpants.mysql_schema
+      privileges = Jetpants.mysql_grant_privs if privileges.empty?
+      privileges = privileges.join(',')
+      commands = []
+      
+      Jetpants.mysql_grant_ips.each do |ip|
+        commands << "#{statement} #{privileges} ON #{database}.* #{preposition} '#{username}'@'#{ip}'"
+      end
+      commands << "FLUSH PRIVILEGES"
+      mysql_root_cmd(commands.join '; ')
+    end
+    
+    # Disables access to a DB by the application user, and sets the DB to 
+    # read-only. Useful when decommissioning instances from a shard that's
+    # been split.
+    def revoke_all_access!
+      user_name = Jetpants.app_credentials[:user]
+      output("Revoking access for user #{user_name} and setting global read-only.")
+      read_only!
+      output(drop_user(user_name, true)) # drop the user without replicating the drop statement to slaves
+    end
+    
+    # Enables global read-only mode on the database.
+    def read_only!
+      mysql_root_cmd 'SET GLOBAL read_only = 1' unless read_only?
+      read_only?
+    end
+    
+    # Disables global read-only mode on the database.
+    def disable_read_only!
+      mysql_root_cmd 'SET GLOBAL read_only = 0' if read_only?
+      not read_only?
+    end
+    
+  end
+end

data/lib/jetpants/db/replication.rb

@@ -0,0 +1,226 @@
+module Jetpants
+  
+  #--
+  # Replication and binlog-related methods #####################################
+  #++
+  
+  class DB
+    # Changes the master for this instance. Supply a Jetpants::DB indicating the new
+    # master, along with options :log_pos, :log_file, :user, :password.
+    # Does NOT automatically start replication afterwards on self!
+    #
+    # If you omit :log_pos or :log_file, uses the current position/file from new_master,
+    # though this is only safe if new_master is not receiving writes!
+    #
+    # If you omit :user or :password, tries obtaining replication credentials from global
+    # settings, and failing that from the current node (assuming it is already a slave)
+    def change_master_to(new_master, option_hash={})
+      return disable_replication! unless new_master   # change_master_to(nil) alias for disable_replication!
+      return if new_master == master                  # no change
+      
+      logfile = option_hash[:log_file]
+      pos     = option_hash[:log_pos]
+      if !(logfile && pos)
+        raise "Cannot use coordinates of a new master that is receiving updates" if new_master.master && ! new_master.repl_paused
+        logfile, pos = new_master.binlog_coordinates
+      end
+      
+      repl_user = option_hash[:user]     || Jetpants.replication_credentials[:user] || replication_credentials[:user]
+      repl_pass = option_hash[:password] || Jetpants.replication_credentials[:pass] || replication_credentials[:pass]
+
+      pause_replication if @master && !@repl_paused
+      result = mysql_root_cmd "CHANGE MASTER TO " +
+        "MASTER_HOST='#{new_master.ip}', " +
+        "MASTER_PORT=#{new_master.port}, " +
+        "MASTER_LOG_FILE='#{logfile}', " +
+        "MASTER_LOG_POS=#{pos}, " +
+        "MASTER_USER='#{repl_user}', " + 
+        "MASTER_PASSWORD='#{repl_pass}'"
+      
+      output "Changing master to #{new_master} with coordinates (#{logfile}, #{pos}): #{result}"
+      @master.slaves.delete(self) if @master rescue nil
+      @master = new_master
+      @repl_paused = true
+      new_master.slaves << self
+    end
+    
+    # Pauses replication
+    def pause_replication
+      raise "This DB object has no master" unless master
+      return if @repl_paused
+      output "Pausing replication from #{@master}."
+      output mysql_root_cmd "STOP SLAVE"
+      @repl_paused = true
+    end
+    alias stop_replication pause_replication
+    
+    # Starts replication, or restarts replication after a pause
+    def resume_replication
+      raise "This DB object has no master" unless master
+      output "Resuming replication from #{@master}."
+      output mysql_root_cmd "START SLAVE"
+      @repl_paused = false
+    end
+    alias start_replication resume_replication
+    
+    # Permanently disables replication
+    def disable_replication!
+      raise "This DB object has no master" unless master
+      output "Disabling replication; this db is no longer a slave."
+      output mysql_root_cmd "STOP SLAVE; RESET SLAVE"
+      @master.slaves.delete(self) rescue nil
+      @master = nil
+      @repl_paused = nil
+    end
+    alias reset_replication! disable_replication!
+    
+    # Wipes out the target instances and turns them into slaves of self.
+    # Resumes replication on self afterwards, but does NOT automatically start
+    # replication on the targets.
+    # You can omit passing in the replication user/pass if this machine is itself
+    # a slave OR already has at least one slave.
+    # Warning: takes self offline during the process, so don't use on a master that
+    # is actively in use by your application!
+    def enslave!(targets, repl_user=false, repl_pass=false)
+      repl_user ||= (Jetpants.replication_credentials[:user] || replication_credentials[:user])
+      repl_pass ||= (Jetpants.replication_credentials[:pass] || replication_credentials[:pass])
+      pause_replication if master && ! @repl_paused
+      file, pos = binlog_coordinates
+      clone_to!(targets)
+      targets.each do |t| 
+        t.change_master_to( self, 
+                            log_file: file, 
+                            log_pos:  pos, 
+                            user:     repl_user, 
+                            password: repl_pass  )
+      end
+      resume_replication if @master # should already have happened from the clone_to! restart anyway, but just to be explicit
+    end
+    
+    # Wipes out the target instances and turns them into slaves of self's master.
+    # Resumes replication on self afterwards, but does NOT automatically start
+    # replication on the targets.
+    # Warning: takes self offline during the process, so don't use on an active slave!
+    def enslave_siblings!(targets)
+      raise "Can only call enslave_siblings! on a slave instance" unless master
+      disable_monitoring
+      pause_replication unless @repl_paused
+      file, pos = repl_binlog_coordinates
+      clone_to!(targets)
+      targets.each do |t| 
+        t.change_master_to( master, 
+                            log_file: file,
+                            log_pos:  pos,
+                            user:     (Jetpants.replication_credentials[:user] || replication_credentials[:user]),
+                            password: (Jetpants.replication_credentials[:pass] || replication_credentials[:pass])  )
+      end
+      resume_replication # should already have happened from the clone_to! restart anyway, but just to be explicit
+      catch_up_to_master
+      enable_monitoring
+    end
+    
+    # Shortcut to call DB#enslave_siblings! on a single target
+    def enslave_sibling!(target)
+      enslave_siblings!([target])
+    end
+    
+    # Use this on a slave to return [master log file name, position] for how far
+    # this slave has executed (in terms of its master's binlogs) in its SQL replication thread.
+    def repl_binlog_coordinates(display_info=true)
+      raise "This instance is not a slave" unless master
+      status = slave_status
+      file, pos = status[:relay_master_log_file], status[:exec_master_log_pos].to_i
+      output "Has executed through master's binlog coordinates of (#{file}, #{pos})." if display_info
+      [file, pos]
+    end
+    
+    # Returns a two-element array containing [log file name, position] for this
+    # database. Only useful when called on a master. This is the current
+    # instance's own binlog coordinates, NOT the coordinates of replication
+    # progress on a slave!
+    def binlog_coordinates
+      hash = mysql_root_cmd('SHOW MASTER STATUS', :parse=>true)
+      raise "Cannot obtain binlog coordinates of this master becaues binary logging is not enabled" unless hash[:file]
+      output "Own binlog coordinates are (#{hash[:file]}, #{hash[:position].to_i})."
+      [hash[:file], hash[:position].to_i]
+    end
+    
+    # Returns the number of seconds beind the master the replication execution is,
+    # as reported by SHOW SLAVE STATUS.
+    def seconds_behind_master
+      raise "This instance is not a slave" unless master
+      slave_status[:seconds_behind_master].to_i
+    end
+    
+    # Waits for this instance's SECONDS_BEHIND_MASTER to reach 0 and stay at
+    # 0 after repeated polls (based on threshold and poll_frequency).  Will raise
+    # an exception if this has not happened within the timeout period, in seconds.
+    # In other words, with default settings: checks slave lag every 5+ sec, and
+    # returns true if slave lag is zero 3 times in a row. Gives up if this does
+    # not occur within a one-hour period. If a large amount of slave lag is
+    # reported, this method will automatically reduce its polling frequency.
+    def catch_up_to_master(timeout=3600, threshold=3, poll_frequency=5)
+      raise "This instance is not a slave" unless master
+      resume_replication if @repl_paused
+      
+      times_at_zero = 0
+      start = Time.now.to_i
+      output "Waiting to catch up to master"
+      while (Time.now.to_i - start) < timeout
+        lag = seconds_behind_master
+        if lag == 0
+          times_at_zero += 1
+          if times_at_zero >= threshold
+            output "Caught up to master."
+            return true
+          end
+          sleep poll_frequency
+        else
+          output "Currently #{lag} seconds behind master."
+          times_at_zero = 0
+          extra_sleep_time = (lag > 30000 ? 300 : (seconds_behind_master / 100).ceil)
+          sleep poll_frequency + extra_sleep_time
+        end
+      end
+      raise "This instance did not catch up to its master within #{timeout} seconds"
+    end
+    
+    # Returns a hash containing the information from SHOW SLAVE STATUS
+    def slave_status
+      hash = mysql_root_cmd('SHOW SLAVE STATUS', :parse=>true)
+      hash = {} if hash[:master_user] == 'test'
+      if @master && hash.count < 1
+        message = "should be a slave of #{@master}, but SHOW SLAVE STATUS indicates otherwise"
+        raise "#{self}: #{message}" if Jetpants.verify_replication
+        output message
+      end
+      hash
+    end
+    
+    # Reads an existing master.info file on this db or one of its slaves,
+    # propagates the info back to the Jetpants singleton, and returns it
+    def replication_credentials
+      unless @master || @slaves.count > 0
+        raise "Cannot obtain replication credentials from orphaned instance -- this instance is not a slave, and has no slaves"
+      end
+      target = (@master ? self : @slaves[0])
+      user, pass = target.ssh_cmd("cat #{mysql_directory}/master.info | head -6 | tail -2").split
+      {user: user, pass: pass}
+    end
+    
+    # Disables binary logging in my.cnf.  Does not take effect until you restart
+    # mysql.
+    def disable_binary_logging
+      output "Disabling binary logging in MySQL configuration; will take effect at next restart"
+      comment_out_ini(mysql_config_file, 'log-bin', 'log-slave-updates')
+    end
+    
+    # Re-enables binary logging in my.cnf after a prior call to disable_bin_log.
+    # Does not take effect until you restart mysql.
+    def enable_binary_logging
+      output "Re-enabling binary logging in MySQL configuration; will take effect at next restart"
+      uncomment_out_ini(mysql_config_file, 'log-bin', 'log-slave-updates')
+    end
+    
+  end
+end

data/lib/jetpants/db/server.rb

@@ -0,0 +1,79 @@
+module Jetpants
+  
+  #--
+  # MySQL server manipulation methods ##########################################
+  #++
+  
+  class DB
+    # Shuts down MySQL, and confirms that it is no longer listening.
+    # OK to use this if MySQL is already stopped; it's a no-op then.
+    def stop_mysql
+      output "Attempting to shutdown MySQL"
+      output service(:stop, 'mysql')
+      running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
+      raise "[#{@ip}] Failed to shut down MySQL: Something is still listening on port #{@port}" unless running.chomp == '0'
+      @running = false
+    end
+    
+    # Starts MySQL, and confirms that something is now listening on the port.
+    # Raises an exception if MySQL is already running or if something else is
+    # already running on its port.
+    def start_mysql
+      @repl_paused = false if @master
+      running = ssh_cmd "netstat -ln | grep #{@port} | wc -l"
+      raise "[#{@ip}] Failed to start MySQL: Something is already listening on port #{@port}" unless running.chomp == '0'
+      output "Attempting to start MySQL"
+      output service(:start, 'mysql')
+      confirm_listening
+      @running = true
+    end
+    
+    # Restarts MySQL.
+    def restart_mysql
+      @repl_paused = false if @master
+      output "Attempting to restart MySQL"
+      output service(:restart, 'mysql')
+      confirm_listening
+      @running = true
+    end
+    
+    # Has no built-in effect. Plugins can override it, and/or implement
+    # before_stop_query_killer and after_stop_query_killer callbacks.
+    def stop_query_killer
+    end
+    
+    # Has no built-in effect. Plugins can override it, and/or implement
+    # before_start_query_killer and after_start_query_killer callbacks.
+    def start_query_killer
+    end
+    
+    # Confirms that a process is listening on the DB's port
+    def confirm_listening(timeout=10)
+      confirm_listening_on_port(@port, timeout)
+    end
+    
+    # Returns the MySQL data directory for this instance. A plugin can override this
+    # if needed, especially if running multiple MySQL instances on the same host.
+    def mysql_directory
+      '/var/lib/mysql'
+    end
+    
+    # Returns the MySQL server configuration file for this instance. A plugin can
+    # override this if needed, especially if running multiple MySQL instances on
+    # the same host.
+    def mysql_config_file
+      '/etc/my.cnf'
+    end
+    
+    # Has no built-in effect. Plugins can override it, and/or implement
+    # before_enable_monitoring and after_enable_monitoring callbacks.
+    def enable_monitoring(*services)
+    end
+    
+    # Has no built-in effect. Plugins can override it, and/or implement
+    # before_disable_monitoring and after_disable_monitoring callbacks.
+    def disable_monitoring(*services)
+    end
+    
+  end
+end

data/lib/jetpants/db/state.rb

@@ -0,0 +1,212 @@
+module Jetpants
+  
+  #--
+  # State accessors ############################################################
+  #++
+  
+  class DB
+    # Returns the Jetpants::DB instance that is the master of this instance, or false if
+    # there isn't one, or nil if we can't tell because this instance isn't running.
+    def master
+      return nil unless running? || @master
+      probe if @master.nil?
+      @master
+    end
+    
+    # Returns an Array of Jetpants::DB instances that are slaving from this instance,
+    # or nil if we can't tell because this instance isn't running.
+    def slaves
+      return nil unless running? || @slaves
+      probe if @slaves.nil?
+      @slaves
+    end
+    
+    # Returns true if replication is paused on this instance, false if it isn't, or
+    # nil if this instance isn't a slave (or if we can't tell because the instance
+    # isn't running)
+    def repl_paused?
+      return nil unless master
+      probe if @repl_paused.nil?
+      @repl_paused
+    end
+
+    # Returns true if MySQL is running for this instance, false otherwise.
+    # Note that if the host isn't available/online/reachable, we consider
+    # MySQL to not be running.
+    def running?
+      probe if @running.nil?
+      @running
+    end
+    
+    # Returns true if we've probed this MySQL instance already.  Several
+    # methods trigger a probe, including master, slaves, repl_paused?, and
+    # running?.
+    def probed?
+      [@master, @slaves, @running].compact.count >= 3
+    end
+
+    # Probes this instance to discover its status, master, and slaves. Several
+    # other methods trigger a probe automatically, including master, slaves,
+    # repl_paused?, and running?.
+    # Ordinarily this method won't re-probe an instance that has already been
+    # probed, unless you pass force=true.  This can be useful if something
+    # external to Jetpants has changed a DB's state while Jetpants is running.
+    # For example, if you're using jetpants console and, for whatever reason,
+    # you stop replication on a slave manually outside of Jetpants.  In this
+    # case you will need to force a probe so that Jetpants learns about the
+    # change.
+    def probe(force=false)
+      return if probed? && !force
+      output "Probing MySQL installation"
+      probe_running
+      probe_master
+      probe_slaves
+    end
+    
+    # Alias for probe(true)
+    def probe!() probe(true) end
+    
+    # Returns true if the MySQL slave I/O thread and slave SQL thread are
+    # both running, false otherwise.  Note that this always checks the current
+    # actual state of the instance, as opposed to DB#repl_paused? which just
+    # remembers the state from the previous probe and any actions since then.
+    def replicating?
+      status = slave_status
+      [status[:slave_io_running], status[:slave_sql_running]].all? {|s| s.downcase == 'yes'}
+    end
+
+    # Returns true if this instance has a master, false otherwise.
+    def is_slave?
+      !!master
+    end
+
+    # Returns true if this instance had at least one slave when it was last
+    # probed, false otherwise. (This method will indirectly force a probe if
+    # the instance hasn't been probed before.)
+    def has_slaves?
+      slaves.count > 0
+    end
+
+    # Returns true if the global READ_ONLY variable is set, false otherwise.
+    def read_only?
+      global_variables[:read_only].downcase == 'on'
+    end
+
+    # Confirms instance has no more than [max] connections currently
+    # (AS VISIBLE TO THE APP USER), and in [interval] seconds hasn't
+    # received more than [threshold] additional connections.
+    # You may need to adjust max if running multiple query killers,
+    # monitoring agents, etc.
+    def taking_connections?(max=4, interval=2.0, threshold=1)
+      current_conns = query_return_array('show processlist').count
+      return true if current_conns > max
+      conn_counter = global_status[:Connections].to_i
+      sleep(interval)
+      global_status[:Connections].to_i - conn_counter > threshold
+    end
+
+    # Returns true if this instance appears to be a standby slave,
+    # false otherwise. Note that "standby" in this case is based
+    # on whether the slave is actively receiving connections, not
+    # based on any Pool's understanding of the slave's state. An asset-
+    # tracker plugin may want to override this to determine standby
+    # status differently.
+    def is_standby?
+      !(running?) || (is_slave? && !taking_connections?)
+    end
+    
+    # Jetpants supports a notion of dedicated backup machines, containing one
+    # or more MySQL instances that are considered "backup slaves", which will
+    # never be promoted to serve production queries.  The default
+    # implementation identifies these by a hostname beginning with "backup".
+    # You may want to override this with a plugin to use a different scheme
+    # if your architecture contains a similar type of node.
+    def for_backups?
+      @host.hostname.start_with? 'backup'
+    end
+    
+    # Returns a hash mapping global MySQL variables (as symbols)
+    # to their values (as strings).
+    def global_variables
+      query_return_array('show global variables').reduce do |variables, variable|
+        variables[variable[:Variable_name].to_sym] = variable[:Value]
+        variables
+      end
+    end
+    
+    # Returns a hash mapping global MySQL status fields (as symbols)
+    # to their values (as strings).
+    def global_status
+      query_return_array('show global status').reduce do |variables, variable|
+        variables[variable[:Variable_name].to_sym] = variable[:Value]
+        variables      
+      end
+    end
+
+    # Returns the Jetpants::Pool that this instance belongs to, if any.
+    def pool
+      Jetpants.topology.pool(self) || Jetpants.topology.pool(master)
+    end
+    
+    
+    ###### Private methods #####################################################
+    
+    private
+    
+    # Check if mysqld is running
+    def probe_running
+      if @host.available?
+        status = service(:status, 'mysql')
+        @running = !(status.downcase.include?('not running'))
+      else
+        @running = false
+      end
+    end
+    
+    # Checks slave status to determine master and whether replication is paused
+    # An asset-tracker plugin may want to implement DB#after_probe_master to
+    # populate @master even if @running is false.
+    def probe_master
+      return unless @running # leaves @master as nil to indicate unknown state
+      status = slave_status
+      if !status || status.count < 1
+        @master = false
+      else
+        @master = self.class.new(status[:master_host], status[:master_port])
+        if status[:slave_io_running] != status[:slave_sql_running]
+          message = "One replication thread is stopped and the other is not"
+          raise "#{self}: #{message}" if Jetpants.verify_replication
+          output message
+          pause_replication
+        end
+        @repl_paused = (status[:slave_io_running].downcase == 'no')
+      end
+    end
+    
+    # Check processlist as root to determine replication clients.  This assumes
+    # you're running only one MySQL instance per machine, and all MySQL instances
+    # use the standard port 3306.  This is a limitation of SHOW PROCESSLIST not
+    # containing the slave's listening port.
+    #
+    # An asset-tracker plugin may want to implement DB#after_probe_slaves to
+    # populate @slaves even if @running is false.
+    #
+    # Plugins may want to override DB#probe_slaves itself too, if running multiple
+    # MySQL instances per physical machine. In this case you'll want to use 
+    # SHOW SLAVE HOSTS, and all slaves must be using the --report-host option.
+    def probe_slaves
+      return unless @running # leaves @slaves as nil to indicate unknown state
+      @slaves = []
+      slaves_mutex = Mutex.new
+      processes = mysql_root_cmd("SHOW PROCESSLIST", :terminator => ';').split("\n")
+      processes.grep(/Binlog Dump/).concurrent_each do |p|
+        tokens = p.split
+        ip, dummy = tokens[2].split ':'
+        db = self.class.new(ip)
+        db.probe
+        slaves_mutex.synchronize {@slaves << db if db.master == self}
+      end
+    end
+    
+  end
+end