mys3ql 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in mys3ql.gemspec
+gemspec

data/README.md

@@ -0,0 +1,70 @@
+# mys3ql = mysql + s3
+
+Simple backup of your MySql database onto Amazon S3.
+
+
+## Quick start
+
+Install and configure as below.
+
+To perform a full backup:
+
+    $ mys3ql full
+
+If you are using MySql's binary logging (see below), back up the binary logs like this:
+
+    $ mys3ql incremental
+
+
+## Installation
+
+First install the gem:
+
+    $ gem install mys3ql
+    
+Second, create your `~/.mys3ql` config file:
+
+    mysql:
+      # Database to back up
+      database: aircms_production
+      # MySql credentials
+      user: root
+      password:
+      # Path (with trailing slash) to mysql commands e.g. mysqldump
+      bin_path: /usr/local/mysql/bin/
+      # If you are using MySql binary logging:
+      # Path to the binary logs, should match the bin_log option in your my.cnf.
+      # Comment out if you are not using mysql binary logging
+      bin_log: /Users/andy/Desktop/mysql-bin
+
+    s3:
+      # S3 credentials
+      access_key_id: XXXXXXXXXXXXXXXXXXXX
+      secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+      # Bucket in which to store your backups
+      bucket: db_backups
+
+
+## Binary logging
+
+To use incremental backups you need to enable binary logging by making sure that the MySQL config file (my.cnf) has the following line in it:
+
+    log_bin = /var/db/mysql/binlog/mysql-bin
+
+The MySQL user needs to have the RELOAD and the SUPER privileges, these can be granted with the following SQL commands (which need to be executed as the MySQL root user):
+
+    GRANT RELOAD ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password';
+    GRANT SUPER ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password';
+
+
+## Inspiration
+
+Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s3_backup).
+
+
+## To Do
+
+- tests ;)
+- restore (pull latest dump, pull bin files, pipe dump into mysql, pipe binfiles into mysql)
+- remove old dump files (s3)
+- fix verbosity/debugging flag

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/mys3ql

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby-local-exec
+
+lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
+$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
+
+require 'mys3ql'
+
+Mys3ql::Conductor.run ARGV

data/lib/mys3ql.rb

@@ -0,0 +1,2 @@
+require 'mys3ql/conductor'
+require 'mys3ql/version'

data/lib/mys3ql/conductor.rb

@@ -0,0 +1,46 @@
+require 'mys3ql/config'
+require 'mys3ql/mysql'
+require 'mys3ql/s3'
+
+module Mys3ql
+  class Conductor
+
+    def self.run(args)
+      abort usage unless args.length == 1 && %w[ full inc ].include?(args.first)
+      conductor = Conductor.new
+
+      command = args.first
+      if command == 'full'
+        conductor.full
+      else
+        conductor.incremental
+      end
+    end
+
+    def initialize
+      @config = Config.new
+      @mysql = Mysql.new @config
+      @s3 = S3.new @config
+    end
+
+    def full
+      @mysql.dump 
+      @s3.push_dump_to_s3 @mysql.dump_file
+      @mysql.clean_up_dump
+      @s3.delete_bin_logs_on_s3
+    end
+
+    def incremental
+      @s3.push_bin_logs_to_s3
+    end
+
+    def self.usage
+      <<END
+usage:
+  mys3ql full            - full backup, push to S3
+  mys3ql incremental     - push bin logs to S3
+END
+    end
+
+  end
+end

data/lib/mys3ql/config.rb

@@ -0,0 +1,67 @@
+require 'yaml'
+
+module Mys3ql
+  class Config
+
+    def initialize
+      @config = YAML.load_file config_file
+    rescue Errno::ENOENT
+      $stderr.puts "missing ~/.mys3ql config file"
+      exit 1
+    end
+
+    #
+    # MySQL
+    #
+
+    def user
+      mysql['user']
+    end
+
+    def password
+      mysql['password']
+    end
+
+    def database
+      mysql['database']
+    end
+
+    def bin_path
+     mysql['bin_path']
+    end
+
+    def bin_log
+      mysql['bin_log']
+    end
+
+    #
+    # S3
+    #
+
+    def access_key_id
+      s3['access_key_id']
+    end
+
+    def secret_access_key
+      s3['secret_access_key']
+    end
+
+    def bucket
+      s3['bucket']
+    end
+
+    private
+
+    def mysql
+      @config['mysql']
+    end
+
+    def s3
+      @config['s3']
+    end
+
+    def config_file
+      File.join "#{ENV['HOME']}", '.mys3ql'
+    end
+  end
+end

data/lib/mys3ql/mysql.rb

@@ -0,0 +1,40 @@
+require 'mys3ql/shell'
+
+module Mys3ql
+  class Mysql
+    include Shell
+
+    def initialize(config)
+      @config = config
+    end
+
+    def dump
+      cmd  = "#{@config.bin_path}mysqldump -u'#{@config.user}'"
+      cmd += " -p'#{@config.password}'" if @config.password
+      cmd += " --quick --single-transaction --create-options"
+      cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging?
+      cmd += " #{@config.database} | gzip > #{dump_file}"
+      execute cmd
+    end
+
+    def clean_up_dump
+      File.delete dump_file
+      debug "deleted #{dump_file}"
+    end
+
+    def dump_file
+      @dump_file ||= "#{timestamp}.sql.gz"
+    end
+
+    private
+
+    def timestamp
+      Time.now.utc.strftime "%Y%m%d%H%M"
+    end
+
+    def binary_logging?
+      @config.bin_log && @config.bin_log.length > 0
+    end
+
+  end
+end

data/lib/mys3ql/s3.rb

@@ -0,0 +1,87 @@
+require 'mys3ql/shell'
+require 'fog'
+
+module Mys3ql
+  class S3
+    include Shell
+
+    def initialize(config)
+      @config = config
+    end
+
+    def push_dump_to_s3(dump_file)
+      key      = "#{dumps_prefix}/#{dump_file}"
+      copy_key = "#{dumps_prefix}/latest.sql.gz"
+      s3_file  = push_to_s3 dump_file, key
+      if s3_file
+        s3_file.copy @config.bucket, copy_key
+        debug "copied #{key} to #{copy_key}"
+      end
+    end
+
+    def push_bin_logs_to_s3
+      if bin_logs_exist?
+        Dir["#{@config.bin_log}/*"].each do |file|
+          name = File.basename file
+          key = "#{bin_logs_prefix}/#{name}"
+          push_to_s3 file, key
+        end
+      end
+    end
+
+    def delete_bin_logs_on_s3
+      bucket.files.all(:prefix => "#{bin_logs_prefix}").each do |file|
+        file.destroy
+        debug "destroyed #{file.key}"
+      end
+    end
+
+    private
+
+    def s3
+      @s3 ||= begin
+        s = Fog::Storage.new(
+          :provider              => 'AWS',
+          :aws_secret_access_key => @config.secret_access_key,
+          :aws_access_key_id     => @config.access_key_id
+        )
+        debug 'connected to s3'
+        s
+      end
+    end
+
+    def bucket
+      @directory ||= begin
+        d = s3.directories.get @config.bucket  # assume bucket exists
+        debug "opened bucket #{@config.bucket}"
+        d
+      end
+    end
+
+    # returns Fog::Storage::AWS::File if we pushed, nil otherwise.
+    def push_to_s3(local_file_name, s3_key)
+      unless bucket.files.head(s3_key)
+        s3_file = bucket.files.create(
+          :key    => s3_key,
+          :body   => File.open(local_file_name),
+          :public => false
+        )
+        debug "pushed #{local_file_name} to #{s3_key}"
+        s3_file
+      end
+    end
+
+    def dumps_prefix
+      "#{@config.database}/dumps"
+    end
+
+    def bin_logs_prefix
+      "#{@config.database}/bin_logs"
+    end
+
+    def bin_logs_exist?
+      @config.bin_log && @config.bin_log.length > 0 && File.exist?(@config.bin_log)
+    end
+
+  end
+end

data/lib/mys3ql/shell.rb

@@ -0,0 +1,22 @@
+module Mys3ql
+  class ShellCommandError < RuntimeError ; end
+
+  module Shell
+    def execute(command)
+      log command
+      result = `#{command}`
+      log "==> #{result}"
+      raise ShellCommandError, "error (exit status #{$?.exitstatus}): #{command} ==> #{result}: #{$?}" unless $?.success?
+      result
+    end
+
+    def log(message)
+      puts message if debugging?
+    end
+
+    def debugging?
+      true
+    end
+
+  end
+end

data/lib/mys3ql/version.rb

@@ -0,0 +1,3 @@
+module Mys3ql
+  VERSION = "0.0.1"
+end

data/mys3ql.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "mys3ql/version"
+
+Gem::Specification.new do |s|
+  s.name        = "mys3ql"
+  s.version     = Mys3ql::VERSION
+  s.authors     = ["Andy Stewart"]
+  s.email       = ["boss@airbladesoftware.com"]
+  s.homepage    = 'https://github.com/airblade/mys3ql'
+  s.summary     = 'Simple backup of your MySql database onto Amazon S3.'
+  s.description = s.summary
+
+  s.rubyforge_project = "mys3ql"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'fog', '~> 1.0.0'
+  s.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification 
+name: mys3ql
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Andy Stewart
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-10-26 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fog
+  prerelease: false
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :runtime
+  requirement: *id001
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  requirement: *id002
+description: Simple backup of your MySql database onto Amazon S3.
+email: 
+- boss@airbladesoftware.com
+executables: 
+- mys3ql
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/mys3ql
+- lib/mys3ql.rb
+- lib/mys3ql/conductor.rb
+- lib/mys3ql/config.rb
+- lib/mys3ql/mysql.rb
+- lib/mys3ql/s3.rb
+- lib/mys3ql/shell.rb
+- lib/mys3ql/version.rb
+- mys3ql.gemspec
+has_rdoc: true
+homepage: https://github.com/airblade/mys3ql
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: mys3ql
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Simple backup of your MySql database onto Amazon S3.
+test_files: []
+