mysql_s3_backup 0.0.1

data/README.md

@@ -0,0 +1,57 @@
+# Send your Mysql backups to S3
+A simple backup script for Mysql and S3 with incremental backups.
+
+It's all based on Paul Dowman's blog post: http://pauldowman.com/2009/02/08/mysql-s3-backup/, so read this first.
+
+## Configuration
+
+To use incremental backups you need to enable binary logging by making sure that the MySQL config file (my.cnf) has the following line in it:
+
+    log_bin = /var/db/mysql/binlog/mysql-bin
+
+The MySQL user needs to have the RELOAD and the SUPER privileges, these can be granted with the following SQL commands (which need to be executed as the MySQL root user):
+
+    GRANT RELOAD ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password';
+    GRANT SUPER ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password';
+
+## Usage
+
+Create a YAML config file:
+
+    mysql:
+      # Database name to backup
+      database: muffins_development
+      # Mysql user and password to execute commands
+      user: root
+      password: secret
+      # Path to mysql binaries, like mysql, mysqldump (optional)
+      bin_path: /usr/bin/
+      # Path to the binary logs, should match the bin_log option in your my.cnf
+      bin_log: /var/lib/mysql/binlog/mysql-bin
+
+    s3:
+      # S3 bucket name to backup to
+      bucket: db_backups
+      # S3 credentials
+      access_key_id: XXXXXXXXXXXXXXX
+      secret_access_key: XXXXXXXXXXXXXXXXXXXXXX
+
+Create a full backup:
+
+    mysql_s3_backup -c=your_config.yml full
+
+Create an incremental backup:
+
+    mysql_s3_backup -c=your_config.yml inc
+
+Restore the latest backup (applying incremental backups):
+
+    mysql_s3_backup -c=your_config.yml restore
+
+Restore a specific backup (NOT applying incremental backups):
+
+    mysql_s3_backup -c=your_config.yml restore 20091126112233
+
+## Running the specs
+
+Create a config file in config/test.yml

data/Rakefile

@@ -0,0 +1,17 @@
+require "spec/rake/spectask"
+
+task :default => :spec
+
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = %w(-fs -c)
+  t.spec_files = FileList["spec/**_spec.rb"]
+end
+
+task :gem do
+  sh "gem build mysql_s3_backup.gemspec"
+end
+
+task :push => :gem do
+  file = Dir["*.gem"].sort.last
+  puts "gem push #{file}"
+end

data/bin/mysql_s3_backup

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby -s
+require "mysql_s3_backup"
+
+config_file = $c
+
+abort <<-USAGE unless config_file.is_a?(String) && ARGV.size > 0
+usage:
+  
+  Backing up:
+  
+    mysql_s3_backup -c=<config.yml> full
+  
+  Incremental backups:
+  
+    mysql_s3_backup -c=<config.yml> inc
+  
+  Restore a backup:
+  
+    mysql_s3_backup -c=<config.yml> restore <name>
+
+USAGE
+
+MysqlS3Backup::Config.from_yaml_file(config_file).backup.send(*ARGV)

data/config/sample.yml

@@ -0,0 +1,18 @@
+mysql:
+  # Database name to backup
+  database: muffins_development
+  # Mysql user and password to execute commands
+  user: root
+  password: secret
+  # Path to mysql binaries, like mysql, mysqldump (optional)
+  bin_path: /usr/bin/
+  # Path to the binary logs, should match the bin_log option in your my.cnf
+  bin_log: /var/lib/mysql/binlog/mysql-bin
+
+s3:
+  # S3 bucket name to backup to
+  bucket: db_backups
+  # S3 credentials
+  access_key_id: XXXXXXXXXXXXXXX
+  secret_access_key: XXXXXXXXXXXXXXXXXXXXXX
+  

data/config/test.yml

@@ -0,0 +1,9 @@
+mysql:
+  database: mysql_s3_backup
+  user: root
+  bin_log: /usr/local/var/mysql/binlogs/mysql-bin
+
+s3:
+  bucket: mysql_backups_test
+  access_key_id: AKIAJVTWYOMR7ZXKDJEA
+  secret_access_key: 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

data/lib/mysql_s3_backup.rb

@@ -0,0 +1,6 @@
+require "mysql_s3_backup/hash_ext"
+require "mysql_s3_backup/shell"
+require "mysql_s3_backup/bucket"
+require "mysql_s3_backup/mysql"
+require "mysql_s3_backup/backup"
+require "mysql_s3_backup/config"

data/lib/mysql_s3_backup/backup.rb

@@ -0,0 +1,84 @@
+require 'tempfile'
+require 'lockfile'
+
+module MysqlS3Backup
+  class Backup
+    attr_reader :mysql, :bucket
+    
+    def initialize(mysql, bucket)
+      @mysql = mysql
+      @bucket = bucket
+      @bin_log_prefix = "#{@mysql.database}/bin_logs"
+    end
+    
+    def full(name=make_new_name)
+      lock do
+        # When the full backup runs it delete any binary log files that might already exist
+        # in the bucket. Otherwise the restore will try to restore them even though they’re
+        # older than the full backup.
+        @bucket.delete_all @bin_log_prefix
+      
+        with_temp_file do |file|
+          @mysql.dump(file)
+          @bucket.store(dump_file_name(name), file)
+          @bucket.copy(dump_file_name(name), dump_file_name("latest"))
+        end
+      end
+    end
+    
+    def incremental
+      lock do
+        @mysql.each_bin_log do |log|
+          @bucket.store "#{@bin_log_prefix}/#{File.basename(log)}", log
+        end
+      end
+    end
+    alias :inc :incremental
+    
+    def restore(name="latest")
+      lock do
+        # restore from the dump file
+        with_temp_file do |file|
+          @bucket.fetch(dump_file_name(name), file)
+          @mysql.restore(file)
+        end
+      
+        if name == "latest"
+          # Restoring binary log files
+          @bucket.find("#{@bin_log_prefix}/").sort.each do |log|
+            with_temp_file do |file|
+              @bucket.fetch log, file
+              @mysql.apply_bin_log file
+            end
+          end
+        end
+      end
+    end
+    
+    private
+      def lock
+        result = nil
+        Lockfile("mysql_s3_backup_lock", :retries => 0) do
+          result = yield
+        end
+        result
+      end
+      
+      def dump_file_name(name)
+        raise ArgumentError, "Need a backup name" unless name.is_a?(String)
+        "#{@mysql.database}/dumps/#{name}.sql.gz"
+      end
+      
+      def make_new_name
+        Time.now.utc.strftime("%Y%m%d%H%M")
+      end
+      
+      def with_temp_file
+        dump_file = Tempfile.new("mysql-dump")
+        yield dump_file.path
+        nil
+      ensure
+        dump_file.close!
+      end
+  end
+end

data/lib/mysql_s3_backup/bucket.rb

@@ -0,0 +1,45 @@
+require "aws/s3"
+
+module MysqlS3Backup
+  class Bucket
+    def initialize(name, options)
+      @name = name
+      @s3_options = options.symbolize_keys.merge(:use_ssl => true)
+      connect
+      create
+    end
+    
+    def connect
+      AWS::S3::Base.establish_connection!(@s3_options)
+    end
+    
+    def create
+      # It doesn't hurt to try to create a bucket that already exists
+      AWS::S3::Bucket.create(@name)
+    end
+    
+    def store(file_name, file)
+      AWS::S3::S3Object.store(file_name, open(file), @name)
+    end
+    
+    def copy(file_name, new_file_name)
+      AWS::S3::S3Object.copy(file_name, new_file_name, @name)
+    end
+    
+    def fetch(file_name, file)
+      open(file, 'w') do |f|
+        AWS::S3::S3Object.stream(file_name, @name) do |chunk|
+          f.write chunk
+        end
+      end
+    end
+    
+    def find(prefix)
+      AWS::S3::Bucket.objects(@name, :prefix => prefix).map { |obj| obj.key }
+    end
+    
+    def delete_all(prefix)
+      AWS::S3::Bucket.objects(@name, :prefix => prefix).each { |obj| obj.delete }
+    end
+  end
+end

data/lib/mysql_s3_backup/config.rb

@@ -0,0 +1,30 @@
+require "yaml"
+
+module MysqlS3Backup
+  class Config
+    attr_reader :mysql_config, :s3_config, :bucket
+    
+    def initialize(config)
+      config = config.symbolize_keys
+      @mysql_config = config[:mysql].symbolize_keys
+      @s3_config = config[:s3].symbolize_keys
+      @bucket = @s3_config.delete(:bucket)
+    end
+    
+    def mysql
+      MysqlS3Backup::Mysql.new(@mysql_config)
+    end
+    
+    def bucket
+      MysqlS3Backup::Bucket.new(@bucket, @s3_config)
+    end
+    
+    def backup
+      MysqlS3Backup::Backup.new(mysql, bucket)
+    end
+    
+    def self.from_yaml_file(file)
+      new YAML.load_file(file)
+    end
+  end
+end

data/lib/mysql_s3_backup/hash_ext.rb

@@ -0,0 +1,10 @@
+class Hash
+  # Return a new hash with all keys converted to symbols.
+  # Stolen from ActiveSupport
+  def symbolize_keys()
+    inject({}) do |options, (key, value)|
+      options[(key.to_sym rescue key) || key] = value
+      options
+    end
+  end
+end

data/lib/mysql_s3_backup/mysql.rb

@@ -0,0 +1,59 @@
+module MysqlS3Backup
+  class Mysql
+    include Shell
+    
+    attr_reader :database, :bin_log_path
+    
+    def initialize(options)
+      options = options.symbolize_keys
+      @user = options[:user] || raise(ArgumentError, "user required")
+      @password = options[:password]
+      @database = options[:database] || raise(ArgumentError, "database required")
+      @bin_log_path = options[:bin_log]
+      @bin_path = options[:bin_path]
+    end
+    
+    def cli_options
+      cmd = "-u'#{@user}'"
+      cmd += " -p'#{@password}'" if @password
+      cmd += " #{@database}"
+      cmd
+    end
+    
+    def execute(sql)
+      run %{#{@bin_path}mysql -e "#{sql}" #{cli_options}}
+    end
+    
+    def execute_file(file)
+      run "cat '#{file}' | #{@bin_path}mysql #{cli_options}"
+    end
+    
+    def dump(file)
+      cmd = "#{@bin_path}mysqldump --quick --single-transaction --create-options -u'#{@user}'"
+      cmd += " --flush-logs --master-data=2 --delete-master-logs" if @bin_log_path
+      cmd += " -p'#{@password}'" if @password
+      cmd += " #{@database} | gzip > #{file}"
+      run cmd
+    end
+    
+    def restore(file)
+      run "gunzip -c #{file} | #{@bin_path}mysql #{cli_options}"
+    end
+    
+    def each_bin_log
+      execute "flush logs"
+      logs = Dir.glob("#{@bin_log_path}.[0-9]*").sort
+      logs_to_archive = logs[0..-2] # all logs except the last
+      logs_to_archive.each do |log|
+        yield log
+      end
+      execute "purge master logs to '#{File.basename(logs[-1])}'"
+    end
+    
+    def apply_bin_log(file)
+      cmd = "#{@bin_path}mysqlbinlog --database=#{@database} #{file} | mysql -u#{@user} "
+      cmd += " -p'#{@password}' " if @password
+      run cmd
+    end
+  end
+end

data/lib/mysql_s3_backup/shell.rb

@@ -0,0 +1,13 @@
+module MysqlS3Backup
+  class ShellCommandError < RuntimeError ; end
+  
+  module Shell
+    def run(command)
+      puts command if $VERBOSE
+      result = `#{command}`.chomp
+      puts result if $VERBOSE
+      raise ShellCommandError, "error, process exited with status #{$?.exitstatus}: #{result}" unless $?.success?
+      result
+    end
+  end
+end

data/mysql_s3_backup.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |s|
+  s.name            = "mysql_s3_backup"
+  s.version         = "0.0.1"
+ 
+  s.authors         = ["Marc-Andre Cournoyer"]
+  s.email           = "macournoyer@gmail.com"
+  s.files           = Dir["**/*"]
+  s.homepage        = "http://github.com/macournoyer/mysql_s3_backup"
+  s.require_paths   = ["lib"]
+  s.bindir          = "bin"
+  s.executables     = Dir["bin/*"].map { |f| File.basename(f) }
+  s.summary         = "A simple backup script for Mysql and S3 with incremental backups."
+  s.test_files      = Dir["spec/**"]
+  
+  s.add_dependency  "aws-s3"
+  s.add_dependency  "lockfile"
+end

data/spec/backup_spec.rb

@@ -0,0 +1,51 @@
+require File.dirname(__FILE__) + "/spec_helper"
+
+describe MysqlS3Backup::Backup do
+  after do
+    CONFIG.bucket.delete_all(CONFIG.mysql_config[:database])
+  end
+  
+  it "should restore from full backup" do
+    backup = CONFIG.backup
+    
+    execute_sql_file "load.sql"
+    execute_sql_file "insert.sql"
+    
+    backup.full
+    
+    execute_sql_file "drop.sql"
+    backup.restore
+    
+    backup.mysql.execute("select count(*) as n from users;").should == "n\n1"
+  end
+  
+  it "should restore from incremental backup" do
+    backup = CONFIG.backup
+    
+    execute_sql_file "load.sql"
+    backup.full
+    
+    execute_sql_file "insert.sql"
+    backup.inc
+    
+    execute_sql_file "drop.sql"
+    backup.restore
+    
+    backup.mysql.execute("select count(*) as n from users;").should == "n\n1"
+  end
+  
+  it "should restore named backup and ignore binary logs" do
+    backup = CONFIG.backup
+    
+    execute_sql_file "load.sql"
+    backup.full("named")
+    
+    execute_sql_file "insert.sql"
+    backup.inc
+    
+    execute_sql_file "drop.sql"
+    backup.restore("named")
+    
+    backup.mysql.execute("select count(*) as n from users;").should == "n\n0"
+  end
+end

data/spec/drop.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS users;

data/spec/insert.sql

@@ -0,0 +1 @@
+INSERT INTO users (name) VALUES ("ma");

data/spec/load.sql

@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS users;
+CREATE TABLE users (
+  name VARCHAR(255) NULL
+);

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+require "rubygems"
+require "spec"
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require "mysql_s3_backup"
+
+CONFIG = MysqlS3Backup::Config.from_yaml_file(File.dirname(__FILE__) + "/../config/test.yml")
+
+module Helpers
+  def execute_sql_file(file)
+    CONFIG.mysql.execute_file File.dirname(__FILE__) + "/#{file}"
+  end
+end
+
+Spec::Runner.configure do |config|
+  config.include Helpers
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: mysql_s3_backup
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Marc-Andre Cournoyer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-26 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: lockfile
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: macournoyer@gmail.com
+executables: 
+- mysql_s3_backup
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/mysql_s3_backup
+- config/sample.yml
+- config/test.yml
+- lib/mysql_s3_backup/backup.rb
+- lib/mysql_s3_backup/bucket.rb
+- lib/mysql_s3_backup/config.rb
+- lib/mysql_s3_backup/hash_ext.rb
+- lib/mysql_s3_backup/mysql.rb
+- lib/mysql_s3_backup/shell.rb
+- lib/mysql_s3_backup.rb
+- mysql_s3_backup-0.0.1.gem
+- mysql_s3_backup.gemspec
+- Rakefile
+- README.md
+- spec/backup_spec.rb
+- spec/drop.sql
+- spec/insert.sql
+- spec/load.sql
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/macournoyer/mysql_s3_backup
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A simple backup script for Mysql and S3 with incremental backups.
+test_files: 
+- spec/backup_spec.rb
+- spec/drop.sql
+- spec/insert.sql
+- spec/load.sql
+- spec/spec_helper.rb