ariossw-db2s3 0.2.5

data/README

@@ -0,0 +1,37 @@
+DB2S3 - A rails plugin to backup Mysql to Amazon S3
+---------------------------------------------------
+You're looking at a monthly spend of four cents
+So pony up you cheap bastard, and store your backups on S3
+
+Usage:
+  # In config/environment.rb
+  config.gem "xaviershay-db2s3", :lib => "db2s3", :source => "http://gems.github.com"
+
+  # In Rakefile
+  require 'db2s3/tasks'
+
+  # In config/initializers/db2s3.rb
+  DB2S3::Config.instance_eval do
+    S3 = {
+      :access_key_id     => 'yourkey',
+      :secret_access_key => 'yoursecretkey',
+      :bucket            => 'yourapp-db-backup'
+    }
+  end
+  # DB credentials are read from your rails environment
+
+  rake gems:install
+
+  # Add to your crontab or whatever
+  rake db2s3:backup:full
+  rake db2s3:backup:incremental # Unimplemented
+
+  # Handy tasks
+  rake db2s3:metrics         # Estimated costs
+  rake db2s3:backup:restore  # You should be testing this regularly
+
+Caveats:
+  Currently does not clean up old back ups
+
+Kudos:
+  http://github.com/pauldowman/blog_code_examples/tree/master/mysql_s3_backup

data/Rakefile

@@ -0,0 +1,4 @@
+desc "Create test database"
+task :create_test_db do
+  `mysqladmin -u root create db2s3_unittest`
+end

data/db2s3.gemspec

@@ -0,0 +1,47 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{db2s3}
+  s.version = "0.2.5"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Xavier Shay"]
+  s.date = %q{2009-03-08}
+  s.description = %q{db2s3 provides rake tasks for backing up and restoring your DB to S3}
+  s.email = %q{contact@rhnh.net}
+  s.files = %w(
+    README
+    Rakefile
+    db2s3.gemspec
+    init.rb
+    lib
+    lib/db2s3.rb
+    lib/db2s3/tasks.rb
+    rails
+    rails/init.rb
+    spec
+    spec/db2s3_spec.rb
+    spec/mysql_drop_schema.sql
+    spec/mysql_schema.sql
+    spec/s3_config.example.rb
+    spec/s3_config.rb
+    spec/spec_helper.rb
+    tasks
+    tasks/tasks.rake
+  )  
+  s.has_rdoc = false
+  s.homepage = %q{http://github.com/ariossw/db2s3}
+  #s.rdoc_options = ["--inline-source", "--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  #s.rubyforge_project = %q{grit}
+  s.rubygems_version = %q{1.3.0}
+  s.summary = %q{db2s3 provides rake tasks for backing up and restoring your DB to S3}
+
+  # TODO: WTF does this do
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+  end
+
+  s.add_dependency(%q<aws-s3>, [">= 0.5.1"])
+end

data/init.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + "/rails/init"

data/lib/db2s3.rb

@@ -0,0 +1,117 @@
+require 'aws/s3'
+require 'tempfile'
+
+class DB2S3
+  class Config
+  end
+
+  def initialize
+  end
+
+  def full_backup
+    file_name = "dump-#{db_credentials[:database]}-#{Time.now.utc.strftime("%Y%m%d%H%M")}.sql.gz"
+    store.store(file_name, open(dump_db.path))
+    store.store(most_recent_dump_file_name, file_name)
+  end
+
+  def restore
+    dump_file_name = store.fetch(most_recent_dump_file_name).read
+    file = store.fetch(dump_file_name)
+    run "gunzip -c #{file.path} | mysql #{mysql_options}"
+  end
+
+  def metrics
+    dump_file = dump_db
+
+    storage_dollars_per_byte_per_month  = 0.15 / 1024.0 / 1024.0 / 1024.0
+    transfer_dollars_per_byte_per_month = 0.10 / 1024.0 / 1024.0 / 1024.0
+    full_dumps_per_month = 30
+
+    storage_cost = (dump_file.size * storage_dollars_per_byte_per_month * 100).ceil / 100.0
+    transfer_cost = (dump_file.size * full_dumps_per_month * transfer_dollars_per_byte_per_month * 100).ceil / 100.0
+    requests_cost = 0.02 # TODO: Actually calculate this, with incremental backups could be more
+
+    {
+      :db_size       => dump_file.size,
+      :storage_cost  => storage_cost,
+      :transfer_cost => transfer_cost,
+      :total_cost    => storage_cost + transfer_cost + requests_cost,
+      :requests_cost => requests_cost,
+      :full_backups_per_month => full_dumps_per_month
+    }
+  end
+
+  private
+
+  def dump_db
+    dump_file = Tempfile.new("dump")
+    
+    #cmd = "mysqldump --quick --single-transaction --create-options -u#{db_credentials[:user]} --flush-logs --master-data=2 --delete-master-logs"
+    cmd = "mysqldump --quick --single-transaction --create-options #{mysql_options}"
+    cmd += " | gzip > #{dump_file.path}"
+    run(cmd)
+
+    dump_file
+  end
+
+  def mysql_options
+    cmd = " -u#{db_credentials[:username]} "
+    cmd += " -p'#{db_credentials[:password]}'" unless db_credentials[:password].nil?
+    cmd += " #{db_credentials[:database]}"
+  end
+
+  def store
+    @store ||= S3Store.new
+  end
+
+  def most_recent_dump_file_name
+    "most-recent-dump-#{db_credentials[:database]}.txt"
+  end
+
+  def run(command)
+    result = system(command)
+    raise("error, process exited with status #{$?.exitstatus}") unless result
+  end
+
+  def db_credentials 
+    ActiveRecord::Base.connection.instance_eval { @config } # Dodgy!
+  end
+
+  class S3Store
+    def initialize
+      @connected = false
+    end
+
+    def ensure_connected
+      return if @connected
+      AWS::S3::Base.establish_connection!(DB2S3::Config::S3.slice(:access_key_id, :secret_access_key).merge(:use_ssl => true))
+      AWS::S3::Bucket.create(bucket)
+      @connected = true
+    end
+
+    def store(file_name, file)
+      ensure_connected
+      AWS::S3::S3Object.store(file_name, file, bucket)
+    end
+
+    def fetch(file_name)
+      ensure_connected
+      AWS::S3::S3Object.find(file_name, bucket)
+      
+      file = Tempfile.new("dump")
+      open(file.path, 'w') do |f|
+        AWS::S3::S3Object.stream(file_name, bucket) do |chunk|
+          f.write chunk
+        end
+      end
+      file
+    end
+
+    private
+
+    def bucket
+      DB2S3::Config::S3[:bucket]
+    end
+  end
+
+end

data/lib/db2s3/tasks.rb

@@ -0,0 +1 @@
+Dir["#{File.dirname(__FILE__)}/../../tasks/*.rake"].each { |ext| load ext }

data/rails/init.rb

@@ -0,0 +1 @@
+require File.expand_path(File.dirname(__FILE__) + '/../lib/db2s3')

data/spec/db2s3_spec.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe 'db2s3' do
+  def load_schema
+    `cat '#{File.dirname(__FILE__) + '/mysql_schema.sql'}' | mysql -u #{DBConfig[:user]} #{DBConfig[:database]}`
+  end
+
+  def drop_schema
+    `cat '#{File.dirname(__FILE__) + '/mysql_drop_schema.sql'}' | mysql -u #{DBConfig[:user]} #{DBConfig[:database]}`
+  end
+
+  class Person < ActiveRecord::Base
+  end
+
+  if DB2S3::Config.const_defined?('S3')
+    it 'can save and restore a backup to S3' do
+      db2s3 = DB2S3.new
+      load_schema
+      Person.create!(:name => "Baxter")
+      db2s3.full_backup
+      drop_schema
+      db2s3.restore
+      Person.find_by_name("Baxter").should_not be_nil
+    end
+  end
+
+  it 'provides estimated metrics' do
+    db2s3 = DB2S3.new
+    # 1 GB DB
+    db2s3.stub!(:dump_db).and_return(stub("dump file", :size => 1024 * 1024 * 1024))
+    metrics = db2s3.metrics
+    metrics.should == {
+      :storage_cost => 0.15, # 15c/GB-Month rounded up to nearest cent, we're only storing one backup
+      :transfer_cost => 3.0, # 10c/GB-Month * 30 backups
+      :db_size       => 1024 * 1024 * 1024, # 1 GB
+      :total_cost    => 3.17,
+      :requests_cost => 0.02,
+      :full_backups_per_month => 30 # Default 1 backup/day
+    }
+  end
+
+  it 'rounds transfer cost metric up to nearest cent' do
+    db2s3 = DB2S3.new
+    # 1 KB DB
+    db2s3.stub!(:dump_db).and_return(stub("dump file", :size => 1024))
+    metrics = db2s3.metrics
+    metrics[:storage_cost].should == 0.01
+    metrics[:transfer_cost].should == 0.01
+  end
+end

data/spec/mysql_drop_schema.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS people;

data/spec/mysql_schema.sql

@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS people;
+CREATE TABLE people (
+name VARCHAR(255) NULL
+);

data/spec/s3_config.example.rb

@@ -0,0 +1,5 @@
+DB2S3::Config:: S3 = {
+  :access_key_id     => 'yourkey',
+  :secret_access_key => 'yoursecretkey',
+  :bucket            => 'db2s3_test'
+}

data/spec/s3_config.rb

@@ -0,0 +1,5 @@
+DB2S3::Config:: S3 = {
+  :access_key_id     => 'yourkey',
+  :secret_access_key => 'yoursecretkey',
+  :bucket            => 'db2s3_test'
+}

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+require 'spec'
+require 'activerecord'
+require File.dirname(__FILE__) + '/../lib/db2s3'
+if File.exists?(File.dirname(__FILE__) + '/s3_config.rb')
+  require File.dirname(__FILE__) + '/s3_config.rb'
+else
+  puts "s3_config.rb does not exist - not running live tests"
+end
+
+DBConfig = {
+  :adapter  => "mysql",
+  :encoding => "utf8",
+  :database => 'db2s3_unittest',
+  :user     => "root"
+}
+
+ActiveRecord::Base.configurations = { 'production' => DBConfig }
+ActiveRecord::Base.establish_connection(:production)

data/tasks/tasks.rake

@@ -0,0 +1,40 @@
+namespace :db2s3 do
+  namespace :backup do
+    desc "Save a full back to S3"
+    task :full => :environment do
+      DB2S3.new.full_backup
+    end
+
+    desc "Restore your DB from S3"
+    task :restore => :environment do
+      DB2S3.new.restore
+    end
+  end
+
+  desc "Provide estimated costs for backing up your DB to S3"
+  task :metrics => :environment do
+    def format_size(size)
+      units = %w{B KB MB GB TB}
+      e = (Math.log(size)/Math.log(1024)).floor
+      s = "%.3f" % (size.to_f / 1024**e)
+      s.sub(/\.?0*$/, units[e])
+    end
+
+    def format_cost(cost)
+      "%.2f" % [cost]
+    end
+
+    metrics = DB2S3.new.metrics
+    puts <<-EOS
+Estimates only, does not take into account metadata overhead 
+Code has recently been added that keeps old backups around - this is not taken into account in these estimates
+
+DB Size:            #{format_size(metrics[:db_size])}
+Full backups/month: #{metrics[:full_backups_per_month]}
+Storage Cost $US:   #{format_cost(metrics[:storage_cost])}
+Transfer Cost $US:  #{format_cost(metrics[:transfer_cost])}
+Requests Cost $US:  #{format_cost(metrics[:requests_cost])}
+Total Cost $US:     #{format_cost(metrics[:total_cost])}
+    EOS
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+name: ariossw-db2s3
+version: !ruby/object:Gem::Version 
+  version: 0.2.5
+platform: ruby
+authors: 
+- Xavier Shay
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-08 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.5.1
+    version: 
+description: db2s3 provides rake tasks for backing up and restoring your DB to S3
+email: contact@rhnh.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- Rakefile
+- db2s3.gemspec
+- init.rb
+- lib
+- lib/db2s3.rb
+- lib/db2s3/tasks.rb
+- rails
+- rails/init.rb
+- spec
+- spec/db2s3_spec.rb
+- spec/mysql_drop_schema.sql
+- spec/mysql_schema.sql
+- spec/s3_config.example.rb
+- spec/s3_config.rb
+- spec/spec_helper.rb
+- tasks
+- tasks/tasks.rake
+has_rdoc: false
+homepage: http://github.com/ariossw/db2s3
+licenses: 
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: db2s3 provides rake tasks for backing up and restoring your DB to S3
+test_files: []
+