jetpants 0.7.0

data/plugins/simple_tracker/simple_tracker.rb

@@ -0,0 +1,74 @@
+# Entrypoint for simple_tracker example asset tracker plugin.
+# config options:
+#   tracker_data_file_path  --  path and filename of where to save the asset data JSON file
+#   app_config_file_path    --  path and filename of where to save the database configuration YAML file for a fictional web app
+
+require 'json'
+
+module Jetpants
+  module Plugin
+    
+    # The SimpleTracker class just handles the manipulations of the asset JSON file and the application
+    # YAML file. The Jetpants::Topology class is monkeypatched to maintain a single SimpleTracker object,
+    # which it uses to interact with these files.
+    class SimpleTracker
+      # Array of hashes, each containing info from Pool#to_hash
+      attr_accessor :global_pools
+      
+      # Array of hashes, each containing info from Shard#to_hash
+      attr_accessor :shards
+      
+      # Array of any of the following:
+      # * hashes each containing key 'node'. could expand to include 'role' or other metadata as well,
+      #   but currently not supported.
+      # * objects responding to to_db, such as String or Jetpants::DB
+      attr_accessor :spares
+      
+      attr_reader :app_config_file_path
+      
+      def initialize
+        @tracker_data_file_path = Jetpants.plugins['simple_tracker']['tracker_data_file_path'] || '/etc/jetpants_tracker.json'
+        @app_config_file_path   = Jetpants.plugins['simple_tracker']['app_config_file_path']   || '/var/lib/mysite/config/databases.yaml'
+        data = JSON.parse(File.read(@tracker_data_file_path)) rescue {'pools' => {}, 'shards' => [], 'spares' => []}
+        @global_pools = data['pools']
+        @shards = data['shards']
+        @spares = data['spares']
+      end
+      
+      def save
+        File.open(@tracker_data_file_path, 'w') do |f|
+          data = {'pools' => @global_pools, 'shards' => @shards, 'spares' => @spares}
+          f.puts JSON.pretty_generate(data)
+        end
+      end
+      
+      def determine_pool_and_role(ip, port=3306)
+        ip += ":#{port}" if port.to_i != 3306
+        
+        [@global_pools + @shards].each do |h|
+          pool = (h['name'] ? Jetpants.topology.pool(h['name']) : Jetpants.topology.shard(h['min_id'], h['max_id']))
+          return [pool, 'MASTER'] if h['master'] == ip
+          h['slaves'].each do |s|
+            return [pool, s['role']] if s['host'] == ip
+          end
+        end
+
+        raise "Unable to find #{ip} among tracked assets"
+      end
+      
+      def determine_slaves(ip, port=3306)
+        ip += ":#{port}" if port.to_i != 3306
+        
+        [@global_pools + @shards].each do |h|
+          next unless h['master'] == ip
+          return h['slaves'].map {|s| s['host'].to_db}
+        end
+        [] # return empty array if not a master
+      end
+      
+    end
+  end
+end
+
+# load all the monkeypatches for other Jetpants classes
+%w(pool shard topology).each {|mod| require "simple_tracker/#{mod}"}

data/plugins/simple_tracker/topology.rb

@@ -0,0 +1,66 @@
+module Jetpants
+  class Topology
+    
+    attr_accessor :tracker
+    
+    ##### METHOD OVERRIDES #####################################################
+    
+    # Populates @pools by reading asset tracker data
+    def load_pools
+      @tracker = Jetpants::Plugin::SimpleTracker.new
+      
+      # Create Pool and Shard objects
+      @pools.concat(@tracker.global_pools.map {|h| Pool.from_hash(h)}.compact)
+      all_shards = @tracker.shards.map {|h| Shard.from_hash(h)}.reject {|s| s.state == :recycle}
+      @pools.concat all_shards
+      
+      # Now that all shards exist, we can safely assign parent/child relationships
+      @tracker.shards.each {|h| Shard.assign_relationships(h, all_shards)}
+    end
+    
+    # Generates a database configuration file for a hypothetical web application
+    def write_config
+      config_file_path = @tracker.app_config_file_path
+      
+      # Convert the pool list into a hash
+      db_data = {
+        'database' => {
+          'pools' => functional_partitions.map {|p| p.to_hash(true)},
+          'shards' => shards.select {|s| s.in_config?}.map {|s| s.to_hash(true)},
+        }
+      }
+      
+      # Convert that hash to YAML and write it to a file
+      File.open(config_file_path, 'w') do |f| 
+        f.write db_data.to_yaml
+      end
+      puts "Regenerated #{config_file_path}"
+    end
+    
+    def claim_spares(count, options={})
+      raise "Not enough spare machines -- requested #{count}, only have #{@tracker.spares.count}" if @tracker.spares.count < count
+      hashes = @tracker.spares.shift(count)
+      hashes.map {|h| h['node'] ? h['node'].to_db : h.to_db}
+    end
+    
+    def count_spares(options={})
+      @tracker.spares.count
+    end
+    
+    
+    ##### NEW METHODS ##########################################################
+    
+    # Called by Pool#sync_configuration to update our asset tracker json.
+    # This actually re-writes all the json. With a more dynamic asset tracker
+    # (something backed by a database, for example) this wouldn't be necessary -
+    # instead Pool#sync_configuration could just update the info for that pool
+    # only.
+    def update_tracker_data
+      @tracker.global_pools = functional_partitions.map &:to_hash
+      @tracker.shards = shards.map &:to_hash
+      @tracker.save
+    end
+    
+    
+  end
+end

data/tasks/promotion.rb

@@ -0,0 +1,260 @@
+module Jetpants
+  module Tasks
+    class Promotion
+
+      def initialize nodes = {}
+        @demoted  = nodes['demote']
+        @promoted = nodes['promote']
+        super
+        Jetpants.verify_replication = false # since master may be offline
+        advise
+        establish_roles
+        prepare
+      end
+
+      def error message
+        abort ['ERROR:'.red, message].join ' '
+      end
+
+      def inform message
+        puts message.blue
+      end
+    
+      def is_ip? address
+        address =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+      end
+
+      def establish_roles
+        establish_demoted
+        establish_replicas
+        establish_promoted
+      end
+
+      def establish_demoted
+        # derive demoted from promoted if possible
+        if @promoted and not @demoted
+          error "invalid ip address #{@promoted}" unless is_ip? @promoted
+          @promoted = Jetpants::DB.new @promoted
+
+          # bail the promoted node isn't a slave or we can't connect
+          unless @promoted.is_slave?
+            error "node (#{@promoted}) does not appear to be a replica of another node"
+          end rescue error("unable to connect to node #{@promoted} to promote")
+
+          # recommend a node to demote
+          agreed = agree [
+            "Would you like to demote the following node?",
+            "address: #{@promoted.master}",
+            "slaves : #{@promoted.master.slaves.join(', ')}",
+            "- yes/no -"
+          ].join "\n"
+          error "unable to promote #{@promoted} unless you demote #{@promoted.master}" unless agreed
+
+          @demoted = @promoted.master.ip
+        end
+      
+        # unable to derive demoted, so ask and convert to a DB object
+        unless @demoted.kind_of? Jetpants::DB
+          @demoted = ask 'Please enter the node to demote:' unless @demoted
+          error "Invalid IP address #{@demoted}" unless is_ip? @demoted
+          @demoted = @demoted.to_db
+        end
+          
+        # connect and ensure node is a master; handle offline nodes appropriately
+        if @demoted.available?
+          error 'Cannot demote a node that has no slaves!' unless @demoted.has_slaves?
+        else
+          inform "unable to connect to node #{@demoted} to demote"
+          error  "unable to perform promotion" unless agree "please confirm that #{@demoted} is offline: yes/no "
+          @replicas = @demoted.slaves # An asset-tracker plugin may have been populated the slave list anyway
+          if !@replicas || @replicas.count < 1
+            replicas = ask "please provide a comma seperated list of current replicas of #{@demoted}: ", lambda {|replicas| replicas.split /,\s*/}
+            error "user supplied list of replicas appears to be invalid - #{replicas}" unless replicas.all? {|replica| is_ip? replica}
+            @replicas = replicas.collect {|replica| replica.to_db}
+            
+            # ensure they were replicas of @demoted
+            @replicas.each do |replica|
+              error "#{replica} does not appear to be a valid replica of #{@demoted}" unless replica.master == @demoted
+            end
+          end
+        end
+          
+        error 'unable to establish demoteable node' unless @demoted.kind_of? Jetpants::DB
+      end
+
+      def establish_replicas
+        @replicas ||= @demoted.slaves
+        error 'no replicas to promote' if @replicas.empty?
+        error 'replicas appear to be invalid' unless @replicas.all? {|replica| replica.kind_of? Jetpants::DB}
+        inform "#{@demoted} has the following replicas: #{@replicas.join(', ')}"
+      end
+
+      def establish_promoted
+        # user supplied node to promote
+        if @promoted and not @promoted.kind_of? Jetpants::DB
+          error "invalid ip address #{@promoted}" unless is_ip? @promoted
+          @promoted = Jetpants::DB.new @promoted
+        end
+
+        # user hasn't supplied a valid node to promote
+        unless @replicas.include? @promoted
+          inform "unable to promote node (#{@promoted}) that is not a replica of #{@demoted}" if @promoted
+
+          # recommend a node
+          puts "\nREPLICA LIST:"
+          @replicas.sort_by {|replica| replica.seconds_behind_master}.each do |node|
+            file, pos = node.repl_binlog_coordinates(false)
+            puts " * %-13s %-30s  lag: %2ds   coordinates: (%-13s, %d)" % [node.ip, node.hostname, node.seconds_behind_master, file, pos]
+          end
+          puts
+          recommended = @replicas.sort_by {|replica| replica.seconds_behind_master}.reject {|r| r.for_backups?}.first
+          agreed = agree [
+            "Would you like to promote the following replica?",
+            "#{recommended.ip} (#{recommended.hostname})",
+            "- yes/no -"
+          ].join "\n"
+          @promoted = recommended if agreed
+        
+          # choose a new node if they disagreed with our recommendation
+          unless agreed
+            choose do |promote|
+              promote.prompt = 'Please choose a replica to promote:'
+              @replicas.each do |replica|
+                promote.choice "#{replica} - replication lag: #{replica.seconds_behind_master} seconds" do
+                  @promoted = replica
+                end
+              end
+            end
+            raise "You chose a backup slave. These are not suitable for promotion. Please try again." if @promoted.for_backups?
+          end
+        end
+
+        error "unable to establish node to promote" unless @promoted.kind_of? Jetpants::DB
+      end
+ 
+      def advise
+        @states = {
+          preparing:  "processing promotion requirements",
+          prepared:   "preparing to disable writes on #{@demoted}",
+          read_only:  "writes have been disabled on #{@demoted}, preparing to demote #{@demoted} and promote #{@promoted}",
+          promoted:   "#{@promoted} has been promoted, please prepare database config for deploy.",
+          deployable: "promotion is complete, please commit and deploy.",
+        }
+        inform @states[@state.to_sym]
+      end
+  
+      state_machine :initial => :preparing do
+        after_transition any => any, :do => :advise
+      
+        event :prepare do
+          transition :preparing => :prepared, :if => :roles_populated?
+        end
+        after_transition :preparing => :prepared, :do => :disable_writes
+
+        event :disable_writes do
+          transition :prepared  => :read_only, :if => :read_only!
+        end
+        after_transition :prepared => :read_only, :do => :promote
+      
+        event :promote do
+          transition :read_only => :promoted, :if => :execute_promotion
+        end
+        after_transition :read_only => :promoted, :do => :prepare_config
+      
+        event :prepare_config do
+          transition :promoted => :deployable, :if => :nodes_consistent? 
+        end
+        after_transition :promoted => :deployable, :do => :summarize_promotion
+        
+        state :preparing, :prepared do
+          def is_db? node
+            node.kind_of? Jetpants::DB
+          end
+          
+          def roles_populated?
+            # ensure our roles are populated with dbs
+            [@demoted, @promoted, @replicas].all? do |role|
+              is_db? role or role.all? do |node|
+                is_db? node
+              end
+            end
+          end
+    
+          def read_only!
+            unless @demoted.available?
+              status = @promoted.slave_status
+              @log, @position = status[:master_log_file], status[:exec_master_log_pos].to_i
+              return true
+            end
+
+            # set read_only if needed
+            @demoted.read_only! unless @demoted.read_only?
+            # bail if we're unable to set read_only
+            error "unable to set 'read_only' on #{@demoted}" unless @demoted.read_only?
+            # record the current log possition to ensure writes are not taking place later.
+            @log, @position = @demoted.binlog_coordinates
+            error "#{@demoted} is still taking writes, unable to promote #{@promoted}" unless writes_disabled?
+            @demoted.read_only?            
+          end
+
+          def writes_disabled?
+            return true unless @demoted.available?
+
+            # ensure no writes have been logged since read_only!
+            [@log, @position] == @demoted.binlog_coordinates
+          end
+          
+        end
+
+        state :read_only, :promoted, :promoted, :deployable do
+          def nodes_consistent?
+            return true unless @demoted.available?
+            @replicas.all? {|replica| replica.slave_status[:exec_master_log_pos].to_i == @position}
+          end
+
+          def ensure_nodes_consistent?
+            inform "ensuring replicas are in a consistent state"
+            until nodes_consistent? do
+              print '.'
+              sleep 0.5
+            end
+            nodes_consistent?
+          end
+
+          def promotable?
+            disable_replication if ensure_nodes_consistent? and @promoted.disable_read_only! 
+          end
+
+          def execute_promotion
+            error 'nodes are not in a promotable state.' unless promotable?
+            error 'replicas are not in a consistent state' unless nodes_consistent? 
+            
+            @demoted.pool.master_promotion! @promoted
+          end
+          
+          def replicas_replicating? replicas = @replicas
+            replicas.all? {|replica| replica.replicating?}
+          end
+
+          def disable_replication replicas = @replicas
+            replicas.each do |replica|
+              replica.pause_replication if replica.replicating?
+            end
+            not replicas_replicating? replicas
+          end
+
+          def summarize_promotion transition
+            summary = Terminal::Table.new :title => 'Promotion Summary:' do |rows|
+              rows << ['demoted',  @demoted]
+              rows << ['promoted', @promoted]
+              rows << ["replicas of #{@promoted}", @promoted.slaves.join(', ')]
+            end
+            puts summary
+            exit
+          end
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification 
+name: jetpants
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.7.0
+platform: ruby
+authors: 
+- Evan Elias
+- Dallas Marlow
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-06-07 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mysql2
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: net-ssh
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: state_machine
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: pry
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: thor
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: highline
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: terminal-table
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: colored
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id009
+description: Jetpants is an automation toolkit for handling monstrously large MySQL database topologies. It is geared towards common operational tasks like cloning slaves, rebalancing shards, and performing master promotions. It features a command suite for easy use by operations staff, though it's also a full Ruby library for use in developing custom migration scripts and database automation.
+email: 
+- me@evanelias.com
+- dallasmarlow@gmail.com
+executables: 
+- jetpants
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- doc/plugins.rdoc
+- doc/configuration.rdoc
+- doc/commands.rdoc
+- doc/requirements.rdoc
+files: 
+- Gemfile
+- README.rdoc
+- doc/plugins.rdoc
+- doc/configuration.rdoc
+- doc/commands.rdoc
+- doc/requirements.rdoc
+- lib/jetpants/callback.rb
+- lib/jetpants/topology.rb
+- lib/jetpants/db/server.rb
+- lib/jetpants/db/state.rb
+- lib/jetpants/db/import_export.rb
+- lib/jetpants/db/privileges.rb
+- lib/jetpants/db/client.rb
+- lib/jetpants/db/replication.rb
+- lib/jetpants/shard.rb
+- lib/jetpants/db.rb
+- lib/jetpants/host.rb
+- lib/jetpants/pool.rb
+- lib/jetpants/monkeypatch.rb
+- lib/jetpants/table.rb
+- lib/jetpants.rb
+- bin/jetpants
+- plugins/simple_tracker/topology.rb
+- plugins/simple_tracker/shard.rb
+- plugins/simple_tracker/simple_tracker.rb
+- plugins/simple_tracker/db.rb
+- plugins/simple_tracker/pool.rb
+- tasks/promotion.rb
+- etc/jetpants.yaml.sample
+homepage: https://github.com/tumblr/jetpants/
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --title
+- "Jetpants: a MySQL automation toolkit by Tumblr"
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: "Jetpants: a MySQL automation toolkit by Tumblr"
+test_files: []
+