mysql_cache_manager 0.2.1

data/bin/mysql_cache_manager

@@ -0,0 +1,200 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'pp'
+require 'getoptlong'
+require 'ostruct'
+require 'mysql_cache_manager'
+
+def usage(exit_code, message = nil)
+  print "Error: #{message}\n\n" unless message.nil?
+
+  print <<'END_OF_USAGE'
+
+Usage:
+
+mysql_cache_manager (--save|--restore) [options] <save file>
+
+Arguments:
+
+--save
+  Perform a dump and save of the buffer pool cache to the specified file.  
+  Cannot be used in conjunction with --restore.
+
+--restore
+  Perform a reload of the buffer pool cache from the specified file.
+  Cannot be used in conjunction with --save
+
+Options:
+
+  --help | -?
+    Print this message.
+
+  --verbose
+    Print verbose messages.
+
+  --image-format
+    Specify the storage backend to use for the cache image.  Currently, only sqlite3 is supported.
+
+  --mysql-host
+    Remote host for mysql connection.
+    
+  --mysql-user
+    User for mysql connection.
+
+  --mysql-password
+    Password for mysql connection.
+
+  --batch-size
+    Specifies the size of each "chunk" of pages to be dumped.
+
+END_OF_USAGE
+
+  exit exit_code
+end
+
+@options = OpenStruct.new
+@options.verbose            = false
+@options.mode               = nil
+@options.mode_options       = 0
+@options.image_format       = "sqlite3"
+@options.mysql_host         = "localhost"
+@options.mysql_user         = nil
+@options.mysql_password     = nil
+@options.batch_size         = 1000
+
+getopt = GetoptLong.new(
+  [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--verbose",          "-v",     GetoptLong::NO_ARGUMENT ],
+  [ "--save",             "-s",     GetoptLong::NO_ARGUMENT ],
+  [ "--restore",          "-r",     GetoptLong::NO_ARGUMENT ],
+  [ "--image-format",     "-i",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--mysql-host",       "-h",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--mysql-user",       "-u",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--mysql-password",   "-p",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--batch-size",       "-b",     GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+getopt.each do |opt, arg|
+  case opt
+    when "--help"
+      usage 0
+    when "--verbose"
+      @options.verbose = true
+    when "--save"
+      @options.mode_options += 1
+      @options.mode = :save
+    when "--restore"
+      @options.mode_options += 1
+      @options.mode = :restore
+    when "--image-format"
+      @options.image_format = arg
+    when "--mysql-host"
+      @options.mysql_host = arg
+    when "--mysql-user"
+      @options.mysql_user = arg
+    when "--mysql-password"
+      @options.mysql_password = arg
+    when "--batch-size"
+      @options.batch_size = arg.to_i
+  end
+end
+
+if @options.mode.nil? or @options.mode_options != 1
+  usage 1, "Exactly one mode (--save, --restore) must be specified"
+end
+
+if ARGV.size != 1
+  usage 1, "A save file name must be provided"
+end
+
+def log(message)
+  if @options.verbose
+    time = Time.now.strftime("%F %T ")
+    puts time + message
+  end
+end
+
+def save(cache_manager)
+  cache_manager.save_cache(@options.save_file)
+end
+
+def restore(cache_manager)
+  status_start = cache_manager.innodb_buffer_pool.status
+  time_start = Time.now.to_f
+  log "Starting restore from %s" % [@options.save_file]
+  total_pages_fetched = cache_manager.restore_cache(@options.save_file, @options.batch_size) do
+    |mysql, pages_fetched, pages_attempted|
+    log "Fetched %i pages\n" % [
+      pages_fetched,
+    ]
+  end
+  status_finish = cache_manager.innodb_buffer_pool.status
+  time_finish = Time.now.to_f
+  time_delta = (time_finish - time_start)
+
+  log "Finished restore from %s in %0.2f seconds" % [
+    @options.save_file,
+    time_delta,
+  ]
+
+  puts
+  puts "Statistics:"
+  puts "  Run time (s):             %10.2f" % [time_delta]
+  puts "  Fetch time (s):           %10.2f" % [cache_manager.timing["fetch"]]
+  puts "  Fetch speed (pages/s):    %10.2f" % [
+    total_pages_fetched / time_delta
+  ]
+  data_read_delta = (status_finish["data_read"] - status_start["data_read"])
+  puts "  Data read (GB):           %10.2f" % [
+    data_read_delta / (1024**3)
+  ]
+  puts "  Data read (MB/s):         %10.2f" % [
+    data_read_delta / (1024**2) / time_delta
+  ]
+  puts "  Buffer pool data start:   %10.2f%%" % [
+    100.0 * (status_start["buffer_pool_pages_data"].to_f /
+              status_start["buffer_pool_pages_total"].to_f)
+  ]
+  puts "  Buffer pool data finish:  %10.2f%%" % [
+    100.0 * (status_finish["buffer_pool_pages_data"].to_f /
+              status_finish["buffer_pool_pages_total"].to_f)
+  ]
+  puts "  Buffer pool restored:     %10.2f%%" % [
+    100.0 * (status_finish["buffer_pool_pages_data"].to_f /
+              cache_manager.image.metadata["buffer_pool_pages_data"].to_f)
+  ]
+  puts "  Timings (s):"
+  cache_manager.timing.each do |metric, time|
+    puts "    %-10s: %10.2f" % [metric, time]
+  end
+end
+
+@options.save_file = ARGV.shift
+
+image_format_names = MysqlCacheManager::CacheImage.constants.select do |c|
+  c.downcase == @options.image_format.downcase
+end
+
+if image_format_names.empty?
+  usage 1, "Unable to find cache image class '%s'; valid classes are: %s" % [
+    @options.image_format,
+    MysqlCacheManager::CacheImage.constants.join(", "),
+  ]
+end
+
+cache_image_class = MysqlCacheManager::CacheImage.const_get(image_format_names.first)
+
+cache_manager = MysqlCacheManager::CacheManager.new(
+  cache_image_class,
+  @options.mysql_host,
+  @options.mysql_user,
+  @options.mysql_password
+)
+
+case @options.mode
+  when :save
+    save(cache_manager)
+  when :restore
+    restore(cache_manager)
+end

data/lib/mysql_cache_manager/cache_image/sqlite3.rb

@@ -0,0 +1,128 @@
+require 'sqlite3'
+
+module MysqlCacheManager
+  module CacheImage
+    class SQLite3
+      TABLE_SCHEMA = {
+        "metadata" => "
+          CREATE TABLE metadata (
+            k STRING NOT NULL,
+            v STRING NOT NULL,
+            PRIMARY KEY (k)
+          )
+        ",
+        "pages" => "
+          CREATE TABLE pages (
+            space         INTEGER NOT NULL,
+            page_number   INTEGER NOT NULL
+          )
+        ",
+      }
+
+      def initialize(filename, create_if_needed)
+        @filename = filename
+
+        if File.exists?(filename)
+          @db = ::SQLite3::Database.new(@filename)
+        else
+          if create_if_needed
+            @db = ::SQLite3::Database.new(@filename)
+            TABLE_SCHEMA.each do |name, schema|
+              @db.query(schema)
+            end
+          else
+            raise "File not found: #{filename}"
+          end
+        end
+
+        @db.cache_size = 200000
+        @db.synchronous = "off"
+        @db.temp_store = "memory"
+      end
+
+      def empty!
+        @db.query("DELETE FROM metadata")
+        @db.query("DELETE FROM pages")
+      end
+
+      def add_metadata(k, v)
+        @insert_metadata ||= @db.prepare("INSERT INTO metadata (k, v) VALUES (?, ?)")
+        @insert_metadata.execute(k, v)
+      end
+
+      def metadata
+        image_metadata = {}
+        @db.execute("SELECT k, v FROM metadata ORDER BY k") do |row|
+          image_metadata[row[0]] = row[1]
+        end
+
+        image_metadata
+      end
+
+      def save_pages(page_iterator)
+        @db.transaction
+        pages = page_iterator.each do |page|
+          save_page(*page)
+        end
+        @db.commit
+
+        pages
+      end
+
+      def save_page(space, page_number)
+        @insert_page ||= @db.prepare("INSERT OR IGNORE INTO pages (space, page_number) VALUES (?, ?)")
+
+        @insert_page.execute(space, page_number)
+      end
+
+      def each_space
+        unless block_given?
+          return Enumerable::Enumerator.new(self, :each_space)
+        end
+
+        spaces = 0
+        @db.execute("SELECT DISTINCT space FROM pages ORDER BY space") do |row|
+          spaces += 1
+          yield row[0]
+        end
+
+        spaces
+      end
+
+      def each_page(space)
+        unless block_given?
+          return Enumerable::Enumerator.new(self, :each_page, space)
+        end
+
+        pages = 0
+        @db.execute("SELECT page_number FROM pages WHERE space = #{space} ORDER BY page_number") do |row|
+          pages += 1
+          yield row[0]
+        end
+
+        pages
+      end
+
+      def each_page_batch(space, batch_size)
+        unless block_given?
+          return Enumerable::Enumerator.new(self, :each_page_batch, space, batch_size)
+        end
+
+        batch_pages = Array.new
+        pages = each_page(space) do |page_number|
+          batch_pages << page_number
+          if batch_pages.size >= batch_size
+            yield batch_pages
+            batch_pages.clear
+          end
+        end
+
+        unless batch_pages.empty?
+          yield batch_pages
+        end
+
+        pages
+      end
+    end
+  end
+end

data/lib/mysql_cache_manager/cache_image.rb

@@ -0,0 +1,4 @@
+Dir.glob(File.dirname(__FILE__) + "/cache_image/*.rb").each do |file|
+  require file
+end
+

data/lib/mysql_cache_manager/cache_manager.rb

@@ -0,0 +1,82 @@
+require 'mysql'
+
+module MysqlCacheManager
+  class CacheManager
+    USEFUL_METADATA = [
+      "buffer_pool_pages_total",
+      "buffer_pool_pages_data",
+      "buffer_pool_pages_misc",
+      "buffer_pool_pages_free"
+    ]
+
+    attr_accessor :mysql, :innodb_buffer_pool, :image, :timing
+
+    def initialize(image_class, host, user, password)
+      @cache_image_class = image_class
+      @mysql_host = host
+      @mysql_user = user
+      @mysql_password = password
+      @image = nil
+      @timing = Hash.new(0.0)
+
+      connect
+    end
+
+    def track_timing(name)
+      raise "No block given" unless block_given?
+      start_time = Time.now.to_f
+      yield
+      end_time = Time.now.to_f
+
+      @timing[name] += (end_time - start_time)
+    end
+
+    def connect
+      @mysql = Mysql.new(@mysql_host, @mysql_user, @mysql_password)
+      @innodb_buffer_pool = InnodbBufferPool.new(@mysql)
+    end
+
+    def save_cache(filename)
+      track_timing("open") do
+        @image = @cache_image_class.new(filename, true)
+        @image.empty!
+      end
+
+      track_timing("stats") do
+        @innodb_buffer_pool.status.each do |k, v|
+          if USEFUL_METADATA.include? k
+            @image.add_metadata(k, v)
+          end
+        end
+      end
+
+      track_timing("save") do
+        @image.save_pages(@innodb_buffer_pool.each_page)
+      end
+    end
+
+    def restore_cache(filename, batch_size=100)
+      track_timing("open") do
+        @image = @cache_image_class.new(filename, false)
+      end
+
+      pages_attempted = 0
+      pages_fetched = 0
+      @image.each_space do |space|
+        @image.each_page_batch(space, batch_size) do |page_batch|
+          track_timing("fetch") do
+            pages_attempted += page_batch.size
+            pages_fetched += @innodb_buffer_pool.fetch_page(space, page_batch)
+          end
+
+          if block_given?
+            track_timing("yield") do
+              yield @mysql, pages_fetched, pages_attempted
+            end
+          end
+        end
+      end
+      pages_fetched
+    end
+  end
+end

data/lib/mysql_cache_manager/innodb_buffer_pool.rb

@@ -0,0 +1,59 @@
+module MysqlCacheManager
+  class InnodbBufferPool
+    QUERY_STATUS = <<-EOQ
+      SELECT * FROM information_schema.global_status
+      WHERE variable_name LIKE 'INNODB\\_%'
+    EOQ
+
+    QUERY_PAGES = <<-EOQ
+      SELECT space, page_number
+      FROM information_schema.innodb_buffer_page
+      WHERE page_type IN ("INDEX")
+    EOQ
+
+    def initialize(mysql)
+      @mysql = mysql
+    end
+
+    def status
+      status_result = @mysql.query(QUERY_STATUS)
+      status_vars = {}
+      status_result.each_hash do |row|
+        var = row['VARIABLE_NAME'].sub("INNODB_", "").downcase
+        status_vars[var] = row['VARIABLE_VALUE'].to_i
+      end
+      status_vars
+    end
+
+    def each_page
+      unless block_given?
+        return Enumerable::Enumerator.new(self, :each_page)
+      end
+
+      pages = 0
+      pages_result = @mysql.query(QUERY_PAGES)
+      pages_result.each_hash do |row|
+        pages += 1
+        if 0 == (pages % 1000)
+          puts "Fetched #{pages} pages so far..."
+        end
+        yield row["space"].to_i, row["page_number"].to_i
+      end
+
+      pages
+    end
+
+    def fetch_page(space, pages)
+      unless pages.is_a? Array
+        pages = [pages]
+      end
+      fetch_query = "SELECT engine_control(innodb, prefetch_pages, #{space}, #{pages.join(',')}) AS pages_fetched"
+      if result = @mysql.query(fetch_query)
+        if status_row = result.fetch_hash
+          return status_row["pages_fetched"].to_i
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/mysql_cache_manager.rb

@@ -0,0 +1,4 @@
+require 'mysql_cache_manager/version'
+require 'mysql_cache_manager/cache_manager'
+require 'mysql_cache_manager/cache_image'
+require 'mysql_cache_manager/innodb_buffer_pool'

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification 
+name: mysql_cache_manager
+version: !ruby/object:Gem::Version 
+  hash: 21
+  prerelease: 
+  segments: 
+  - 0
+  - 2
+  - 1
+  version: 0.2.1
+platform: ruby
+authors: 
+- Jeremy Cole
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-12-04 00:00:00 Z
+dependencies: []
+
+description: A tool for saving and restoring the InnoDB buffer pool using the information_schema.buffer_page table and engine_control(InnoDB, prefetch_pages, ...) function.
+email: jeremy@jcole.us
+executables: 
+- mysql_cache_manager
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/mysql_cache_manager.rb
+- lib/mysql_cache_manager/cache_manager.rb
+- lib/mysql_cache_manager/cache_image.rb
+- lib/mysql_cache_manager/cache_image/sqlite3.rb
+- lib/mysql_cache_manager/innodb_buffer_pool.rb
+- bin/mysql_cache_manager
+homepage: http://jcole.us/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: MySQL Cache Manager
+test_files: []
+
+has_rdoc: