sqlup 0.0.1

data/lib/mysql_backup/entity/identifier.rb

@@ -0,0 +1,179 @@
+require 'rubygems'
+require 'named_arguments'
+require 'mysql_backup/utilities/factory_create_method'
+
+module MysqlBackup; end
+class MysqlBackup::Entity; end;
+
+# There are four different kinds of things you store for MySQL backups.
+# 
+# - Completed logs.
+# - The current log (the log file that MySQL is writing to)
+# - mysqldump files (output from the mysqldump command)
+# - binary files (tar | gzip | split of the files in the mysql data directory)
+#
+# They're stored using the following name schemes:
+#
+#   :log:type_complete:thelog.000005
+#     A completed log file.
+#
+#   :log:type_current:log_file_thelog.0000000006:log_position_0000000311:n_parts_0000000001:part_number_0000000000
+#     A current log file.  The position is 311 (that's where mysql will write the next statement).
+#     There's only one part, and this is it.  (Part numbers start with 0.)
+#     In this release, log files can only have one part - no attempt is made to split them
+#     into chunks small enough to fit in S3.  Don't let your log files grow larger than 5G.
+#   :full:type_mysqldump:log_file_thelog.0000000006:log_position_0000000382:n_parts_0000000001:part_number_0000000000
+#     A full mysqldump file.  The current log file is thelog.0000000006, and the position in
+#     that log is 382.  Since we don't flush logs, anything written after 382 might not be
+#     complete.
+#   :full:type_binary:log_file_thelog.0000000006:log_position_0000000311:n_parts_0000000001:part_number_0000000000
+#     A full copy of the MySQL data files, created by tarring up all the data files,
+#     then passing them through gzip and split to make sure they'll fit in S3 objects that
+#     can only hold 5G.
+class MysqlBackup::Entity::Identifier
+  include NamedArguments
+  include FactoryCreateMethod
+  
+  # category is one of
+  #   :full => a full backup of the mysql files
+  #   :log_current => the mysql binary log file that's being written to
+  #   :log_complete => mysql binary log files that are complete
+  attr_accessor :category
+  
+  # type depends on the category.
+  # 
+  # for category +full+, type is one of 
+  #   :binary, :mysqldump
+  attr_accessor :type
+  
+  # The name of the log file
+  attr_accessor :log_file
+  
+  # The numeric position in the log file
+  attr_accessor :log_position
+  
+  # For multipart storage units, the part number
+  attr_accessor :part_number
+  
+  # For multipart storage units, the total number of parts
+  attr_accessor :n_parts
+  
+  # Time the object was created
+  attr_accessor :timestamp
+  
+  def initialize args
+    super
+    
+    part_number ||= 0
+    part_number = sprintf "%06d", part_number.to_i
+    
+    n_parts ||= 1
+    n_parts = sprintf "%06d", n_parts.to_i
+    
+    throw :bad if log_file == 'foo'
+  end
+  
+  def merge new_values_hash
+    to_hash.merge new_values_hash
+  end
+  
+  def to_hash
+    result = {}
+    [:category, :type, :log_file, :log_position, :n_parts, :part_number].each do |i|
+      result[i] = send i
+    end
+    result
+  end
+  
+  def to_s
+    pos = n_digits log_position
+    np = n_digits n_parts
+    pn = n_digits part_number
+    return "#{category}:type_#{@type}:log_file_#{log_file}:log_position_#{pos}:n_parts_#{np}:part_number_#{pn}"
+  end
+  
+  def n_digits x
+    sprintf "%010d", x
+  end
+  
+  # Get the sequence number from the log file name.
+  #
+  #   mylogfile.000034 => 34
+  def log_file_number
+    log_file[/\.(\d+)$/, 1].to_i
+  end
+  
+  # Examples of the four identifiers:
+  #
+  # log/type_complete/thelog.000001
+  # log/type_current/log_file_thelog.0000000006/log_position_0000000311/n_parts_0000000001/part_number_0000000000
+  # full/type_mysqldump/log_file_thelog.0000000006/log_position_0000000382/n_parts_0000000001/part_number_0000000000
+  # full/type_binary/log_file_thelog.0000000006/log_position_0000000311/n_parts_0000000001/part_number_0000000000
+  def self.string_to_args s
+    parts = s.split(':')
+    args = {}
+    args[:category] = parts.shift.to_sym
+    args[:type] = (parts.shift)[/^type_(.*)/, 1].to_sym
+    args[:log_file] = (parts.shift)[/log_file_(.*)/, 1]
+    unless parts.empty?
+      args[:log_position] = (parts.shift)[/log_position_(.*)/, 1].to_i
+    end
+    unless parts.empty?
+      args[:n_parts] = (parts.shift)[/n_parts_(.*)/, 1].to_i
+      args[:part_number] = (parts.shift)[/part_number_(.*)/, 1].to_i
+    end
+    args
+  end
+  
+  def [] x
+    send x  
+  end
+  
+  def <=> rhs
+    category <=> rhs.category || self[:type] <=> rhs[:type] || log_file <=> rhs.log_file || log_position <=> rhs.log_position
+  end
+  
+  append_factory_method do |args|
+    result = args[:string] && create_object(string_to_args(args[:string]))
+    if result
+      result.timestamp = args[:timestamp]
+    end
+    result
+  end
+end
+
+class MysqlBackup::Entity::Identifier::Full < MysqlBackup::Entity::Identifier
+end
+
+class MysqlBackup::Entity::Identifier::Full::Binary < MysqlBackup::Entity::Identifier::Full
+  append_factory_method {|args| args[:category] == :full && args[:type] == :binary && new(args)}
+end
+
+class MysqlBackup::Entity::Identifier::Full::Mysqldump < MysqlBackup::Entity::Identifier::Full
+  append_factory_method {|args| args[:category] == :full && args[:type] == :mysqldump && new(args)}
+end
+
+class MysqlBackup::Entity::Identifier::Log < MysqlBackup::Entity::Identifier
+end
+
+class MysqlBackup::Entity::Identifier::Log::Current < MysqlBackup::Entity::Identifier::Log
+  def to_s
+    pos = n_digits log_position
+    return "#{category}:type_#{@type}:log_file_#{log_file}:log_position_#{pos}"
+  end
+  
+  append_factory_method {|args| args[:category] == :log && args[:type] == :current && new(args)}
+end
+
+class MysqlBackup::Entity::Identifier::Log::Complete < MysqlBackup::Entity::Identifier::Log
+  def to_s
+    return "#{category}:type_#{@type}:log_file_#{log_file}"
+  end
+  append_factory_method {|args| args[:category] == :log && args[:type] == :complete && new(args)}
+end
+
+class Pathname
+  def path
+    to_s
+  end
+end

data/lib/mysql_backup/entity/logs.rb

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'named_arguments'
+require 'mysql_backup/entity/files'
+
+class MysqlBackup::Entity::Logs < MysqlBackup::Entity::Files
+  include NamedArguments
+  
+  attr_accessor :log_bin_dir
+  attr_accessor :completed_logs
+  attr_accessor :log_position
+  attr_accessor :log_file
+  
+  # Takes the following arguments:
+  #
+  #   :log_bin_dir => The prefix of the log files
+  #   :completed_logs => All the logs before the one being written
+  #   :log_file => The current log
+  #   :log_position => The position in the log file
+  def initialize args = {}
+    super
+    @log_bin_dir = Pathname.new(args[:log_bin_dir])
+  end
+  
+  def completed_logs_paths
+    completed_logs.map {|l| Pathname.new(log_bin_dir) + l}
+  end
+  
+  def log_file_path
+    Pathname.new(log_bin_dir) + log_file
+  end
+  
+  def save args = {}
+    # Do the complete logs
+    completed_logs_paths.each do |p|
+      i = MysqlBackup::Entity::Identifier.create_object :category => :log, :type => :complete, :log_file => p.basename.to_s, :n_parts => 1, :part_number => 0
+      files = self.class.tar_files [p.to_s]
+      yield :identifier => i, :file => files.first
+    end
+    
+    # Do the current log
+    i = MysqlBackup::Entity::Identifier.create_object :category => :log, :type => :current, :log_file => log_file, :log_position => log_position, :n_parts => 1, :part_number => 0
+    yield :identifier => i, :file => log_file_path
+  end
+end

data/lib/mysql_backup/entity/mysqldump.rb

@@ -0,0 +1,57 @@
+require 'mysql_backup/entity'
+
+# Mysqldump files are stored gzipped and split
+class MysqlBackup::Entity::Mysqldump
+  include NamedArguments
+  
+  attr_accessor :log
+  attr_accessor :log_file
+  attr_accessor :log_position
+  
+  # Create a mysqldump file and yield 
+  # a hash with a MysqlBackup::Entity::Identifier.
+  def create 
+    Tempfile.open 'mysqldump' do |f|
+      cmd = "mysqldump --opt --all-databases --single-transaction --master-data=2 > #{f.path}"
+      log && log.info("running #{cmd}")
+      system cmd or raise RuntimeError, "failed to run command: #{cmd}"
+      f.flush
+      get_log_position f or raise RuntimeError, "could not get log position"
+      f.seek 0
+      compressed_files = compress_and_split f
+      identifier = MysqlBackup::Entity::Identifier.create_object :category => :full, :type => :mysqldump, :n_parts => compressed_files.length, :log_position => log_position, :log_file => log_file
+      compressed_files.each_with_index do |cf, n|
+        i = identifier.dup
+        i.part_number = n
+        yield :identifier => i, :file => cf  
+      end
+    end
+  end
+  
+  # Search through a mysqldump file looking for a line like
+  #   CHANGE MASTER TO MASTER_LOG_FILE='thelog.000005', MASTER_LOG_POS=1163;
+  #
+  # Set log_position and log_file to the corresponding values.
+  def get_log_position file_obj
+    result = {}
+    n = 0
+    file_obj.each_line do |l|
+      if l =~ /CHANGE MASTER TO MASTER_LOG_FILE='(.*)', MASTER_LOG_POS=(\d+)/i
+        @log_file = $1
+        @log_position = $2.to_i
+        return true
+      end
+      n += 1
+      break if n > 100
+    end
+    false
+  end
+  
+  def compress_and_split file_obj
+    destination_name = "#{file_obj.path}xxx"
+    cmd = "cat #{file_obj.path} | gzip | split --suffix-length=4 --bytes=10240000 --numeric-suffixes - #{destination_name}"
+    log && log.info("running " + cmd)
+    system cmd
+    Pathname.glob destination_name + "*"
+  end
+end

data/lib/mysql_backup/entity.rb

@@ -0,0 +1,38 @@
+require 'rubygems'
+require 'pathname'
+require 'tempfile'
+
+require 'mysql_backup/entity/identifier'
+
+module MysqlBackup; end
+
+class MysqlBackup::Entity
+  attr_accessor :log
+
+  # Create a set of tarred, gziped, and split files
+  # for all the files given by required_path_strings.
+  #
+  # Return an array of Pathname objects for the 
+  # split files.
+  #
+  # The caller is responsible for removing the files returned
+  # from tar_files.
+  def self.tar_files path_strings #:nodoc:
+    result = []
+    # Creating a tempfile to get a guaranteed unique filename.
+    # We don't actually write to this tempfile, we just
+    # use its name as the base name for the split.
+    Tempfile.open 'mysqltarball' do |f|
+      # We should never see existing files, but just in
+      # case we'll need to remove any.
+      Pathname.glob(f.path.to_s + 'xxx*').each {|e| e.unlink}
+      
+      paths = path_strings.map {|ps| ps.slice(1..-1)}.join(' ')
+      five_gig = 1024 * 1024 * 1024 * 5
+      tar_cmd = "( cd / ; tar cf - #{paths} | gzip | split --suffix-length=4 --bytes=#{five_gig} --numeric-suffixes - #{f.path}xxx )"
+      do_tar :cmd => tar_cmd
+      result = Pathname.glob(f.path.to_s + 'xxx*')
+    end
+    result
+  end
+end

data/lib/mysql_backup/librarian/backup.rb

@@ -0,0 +1,134 @@
+require 'rubygems'
+require 'named_arguments'
+require 'mysql_backup/entity/identifier'
+
+module MysqlBackup; end
+class MysqlBackup::Librarian; end
+
+# A backup is a complete set of identifiers for each run of backup type (full or log).
+# Each backup has multiple identifiers because a single backup can have multiple files.
+# The mysql binary logs are tarred up and then split into smaller parts to fit on S3.
+# The same for mysqldump files; they can be greater than the size you can store in one 
+# S3 bucket.
+class MysqlBackup::Librarian::Backup
+  include Comparable
+  include FactoryCreateMethod
+  include NamedArguments
+  
+  # Returns an array of MysqlBackup::Entity::Identifier objects
+  # that are the members of this group
+  attr_accessor :identifiers
+  attribute_defaults :identifiers => []
+  
+  create_class_settings_method :identifier_class
+  
+  def initialize args = {}
+    super
+    args[:identifier] or raise RuntimeError, "Must provide :identifier"
+    identifiers << args[:identifier]
+  end
+  
+  # True if this backup is the most recent of its type
+  def most_recent?
+  end
+  
+  
+  # Write the files for this backup, untarring and unzipping
+  # as required.
+  def write_files directory
+  end
+  
+  # Write the files for this backup without untarring and unzipping.
+  def write_raw_files directory
+  end
+  
+  # Returns true if the identifier was added to this group
+  def add_identifier_to_group_if_the_identifier_should_be_in_this_group identifier
+    is_part = is_part_of_this_group? identifier
+    identifiers << identifier if is_part
+    is_part
+  end
+  
+  def is_part_of_this_group? identifier
+    return false unless identifier_class?.first === identifier
+    log_file == identifier.log_file && log_position == identifier.log_position
+  end
+  
+  def log_file
+    identifiers.first.log_file
+  end
+  
+  def log_position
+    identifiers.first.log_position
+  end
+  
+  def each_identifier 
+    identifiers.each {|i| yield i}  
+  end
+  
+  def <=> rhs
+    identifiers.first.log_file_number <=> rhs.log_file_number || identifiers.first.log_position <=> rhs.log_position
+  end
+  
+  # The id string for a backup doesn't include the number of parts and the
+  # part itself.  
+  #
+  # (Split on :, ignore the last two elements)
+  def to_s
+    identifiers.first.to_s.split(':').slice(0..-3).join(':')
+  end
+  
+  def name_match rhs
+    to_s == rhs
+  end
+  
+  def self.new_if_class_match klass
+    new_if_class klass, :identifier
+  end
+end
+
+module MysqlBackup
+  class Librarian::Backup::Full < Librarian::Backup
+  end
+  
+  class Librarian::Backup::Full::Binary < Librarian::Backup::Full
+    new_if_class_match Entity::Identifier::Full::Binary
+    identifier_class Entity::Identifier::Full::Binary
+  end
+  
+  class Librarian::Backup::Full::Mysqldump < Librarian::Backup::Full
+    new_if_class_match Entity::Identifier::Full::Mysqldump
+    identifier_class Entity::Identifier::Full::Mysqldump
+  end
+  
+  class Librarian::Backup::Log < Librarian::Backup
+    def to_s
+      identifiers.first.to_s
+    end
+  end
+  
+  class Librarian::Backup::Log::Current < Librarian::Backup::Log
+    new_if_class_match Entity::Identifier::Log::Current
+    identifier_class Entity::Identifier::Log::Current
+  end
+  
+  class Librarian::Backup::Log::Complete < Librarian::Backup::Log
+    new_if_class_match Entity::Identifier::Log::Complete
+    identifier_class Entity::Identifier::Log::Complete
+    
+    def is_part_of_this_group? identifier
+      return false unless identifier_class?.first === identifier
+      log_file == identifier.log_file
+    end
+    
+    def log_position
+      0
+    end
+  end
+end
+
+class Symbol
+  def to_proc
+    Proc.new { |obj, *args| obj.send(self, *args) }
+  end
+end

data/lib/mysql_backup/librarian/backup_collection.rb

@@ -0,0 +1,63 @@
+require 'named_arguments'
+require 'mysql_backup/librarian/backup'
+
+# A backup group collection is a list of all the backup groups held by 
+# a MysqlBackup::Storage object.
+#
+# Backup group collections::  A collection of backup groups.  Usually contains all of the backup groups in an S3 bucket.
+# Backup group::  A collection of identifiers for S3 buckets.  These identifiers point to a a single backup.
+module MysqlBackup; end
+class MysqlBackup::Librarian; end
+
+class MysqlBackup::Librarian::BackupCollection
+  include NamedArguments
+  
+  attr_accessor :groups
+  attribute_defaults :groups => []
+  
+  def add_backup_group g
+    groups << g
+  end
+  
+  def add_identifier identifier
+    result = nil
+    result = groups.find do |g|
+      g.add_identifier_to_group_if_the_identifier_should_be_in_this_group identifier
+    end
+    unless result
+      result = MysqlBackup::Librarian::Backup.create_object :identifier => identifier
+      groups << result
+    end
+    result
+  end
+  
+  def groups_matching_type t
+    each_group t    
+  end
+  
+  def each_log
+    each_group MysqlBackup::Librarian::Backup::Log do |g|
+      yield g
+    end
+  end
+  
+  def each_group matching_type = Object
+    result = []
+    groups.select {|g| matching_type === g}.each do |gg|
+      yield gg if block_given?
+      result << gg
+    end
+    result
+  end
+  
+  def find_group group_name
+    each_group do |g|
+      return g if group_name == g.to_s
+    end
+    nil
+  end
+  
+  def types
+    groups.map(&:class).uniq
+  end
+end

data/lib/mysql_backup/librarian.rb

@@ -0,0 +1,176 @@
+require 'rubygems'
+require 'named_arguments'
+
+require 'mysql_backup/server'
+require 'mysql_backup/storage/s3'
+require 'mysql_backup/entity/files'
+require 'mysql_backup/entity/files/innodb'
+require 'mysql_backup/entity/files/myisam'
+require 'mysql_backup/entity/logs'
+require 'mysql_backup/entity/mysqldump'
+require 'mysql_backup/librarian/backup_collection'
+
+# backup_data_files:: Save the binary backups
+# backup_mysqldump:: Send the mysqldump to s3
+# backup_binary_logs:: Send the logs to S3
+class MysqlBackup::Librarian
+  include NamedArguments 
+  
+  # MySQL connection parameters
+  attr_accessor :host, :user, :password, :db, :port, :sock, :flag
+  
+  # The logger.  Default is to log via STDERR.
+  attr_accessor :log
+  
+  # A MysqlBackup::Storage object.
+  attr_accessor :storage
+  
+  # The connection to MySQL
+  attr_accessor :connection 
+  
+  # The MysqlBackup::Server object used to talk to MySQL.
+  attr_writer :mysql_server
+  
+  # S3 parameters
+  attr_accessor :access_key_id, :secret_access_key, :bucket
+  
+  # Takes a required argument to specify the location of the log files:
+  #
+  #   :log_bin_dir => '/var/lib/mysql'
+  #
+  # Takes required arguments for S3:
+  #
+  #   :access_key_id     => 'abc',
+  #   :secret_access_key => '123'
+  #   :bucket => 'name_of_the_backup_bucket'
+  #
+  # Takes these arguments for the connection to MySQL:
+  #
+  #   :host
+  #   :user
+  #   :password
+  #   :db
+  #   :port
+  #   :sock
+  #   :flag
+  # 
+  # Many installations just need to specify <tt>:host</tt> and <tt>:user</tt>.
+  def initialize args = {}
+    super
+  end
+  
+  def backup_data_files
+    MysqlBackup::Entity::Files.create_tar_files :log => log, :mysql_server => mysql_server, :mysql_files => [innodb_files, myisam_files] do |args|
+      storage.conditional_save args
+    end
+  end
+  
+  def backup_mysqldump
+    m = MysqlBackup::Entity::Mysqldump.new :log => log
+    m.create do |args|
+      storage.conditional_save args
+    end
+  end
+  
+  def backup_binary_logs
+    logs_obj = mysql_server.create_logs_obj
+    logs_obj.save do |args|
+      storage.conditional_save args
+    end
+  end
+  
+  def innodb_files
+    @innodb_files ||= mysql_server.create_innodb_files_obj
+  end
+  
+  def myisam_files
+    @myisam_files ||= mysql_server.create_myisam_files_obj
+  end
+  
+  def log_files
+    @log_files ||= mysql_server.create_logs_obj
+  end
+  
+  def mysql_server
+    @mysql_server ||= MysqlBackup::Server.new :connection => create_connection, :log => log
+  end
+  
+  def create_backup_group_collection_from_storage
+    unless @backups
+      c = MysqlBackup::Librarian::BackupCollection.new
+      storage.yield_identifiers do |i|
+        c.add_identifier i
+      end
+      @backups = c
+    end
+    @backups
+  end
+  
+  def ls klass = Object
+    result = []
+    create_backup_group_collection_from_storage.each_group(klass) do |g|
+      result << g.to_s
+    end
+    result
+  end
+  
+  def rm backup_name
+    g = find_group backup_name
+    g.each_identifier do |i|
+      storage.rm i
+    end
+  end
+  
+  def get f, directory
+    case f
+    when /^full:type_binary/
+      get_full_binary f, directory
+    when /^full:type_mysqldump/
+      get_mysqldump f, directory
+    end
+  end
+  
+  def get_full_binary f, destination_dir = '/tmp'
+    get_backup f do |tempfile|
+      run_cmd "( cd #{destination_dir}; zcat #{tempfile.path} | tar xf - )"
+    end
+  end
+  
+  def get_mysqldump f, destination_dir = '/tmp'
+    get_backup f do |tempfile|
+      run_cmd "( cd #{destination_dir}; zcat #{tempfile.path} > #{f})"
+    end
+  end
+  
+  def get_logs destination_dir = '/tmp'
+    create_backup_group_collection_from_storage.each_log do |g|
+      log && log.info("Looking at backup #{g.to_s}")
+      storage.retrieve_backup_and_then_yield_file g do |tempfile|
+        run_cmd "( cd #{destination_dir}; zcat #{tempfile.path} | tar xf - )"
+      end
+    end
+  end
+  
+  def run_cmd cmd
+    log && log.debug("Run command: #{cmd}")
+    system cmd
+  end
+  
+  # Always call this with a block that will do something
+  # with the raw data retrieved from S3.
+  def get_backup f, &block
+    b = find_group f
+    raise RuntimeError, "Failed to find backup for #{f}" unless b
+    if b
+      storage.retrieve_backup_and_then_yield_file(b, &block)
+    end
+  end
+  
+  def find_group f
+    create_backup_group_collection_from_storage.find_group f
+  end
+  
+  def create_connection 
+    @connection ||= Mysql.connect(@host, @user, @pass, @db, @port, @sock, @flag)
+  end
+end