db2fog 0.4

data/HISTORY

@@ -0,0 +1,17 @@
+v0.4.0 (22nd July 2011)
+  - forked db2s3 into db2fog
+0.3.1 (7 Dec 2009)
+  - Fixed hardcoded DB name in statistics task
+0.3.0 (6 Dec 2009)
+  - Added db2s3:backup:clean task to delete old backups
+  - Added dependency on activesupport for the clean task
+0.2.6 (6 Dec 2009)
+  - Remove metrics task, since it was far too inaccurate
+  - Add statistics task to show you the size of your tables
+  - Only add username to mysql command line if provided in database.yml
+0.2.5
+  - Use host provided in database.yml
+0.2.4 (2 Sep 2009)
+  - Fix credentials bug
+0.2.3
+  - Keep old backups around

data/README.rdoc

@@ -0,0 +1,87 @@
+= DB2Fog
+
+A rails plugin to backup Mysql to a cloud storage provider. You're looking at
+a monthly spend of four cents. So pony up you cheap bastard, and store your
+backups offsite.
+
+A grandfather style system to decide what backups to keep copies of:
+
+* all backups from the past 24 hours
+* one backup per day for the past week
+* one backup per week forever
+
+Depending on your tolerance for data loss you should be running a backup at
+least once a day, probably more.
+
+== Installation
+
+Add the following to your project Gemfile
+
+  gem "db2fog"
+
+== Configuration
+
+Add the following to config/initializers/db2fog.rb
+
+In general, you can use any configuration options supported by
+Fog::Storage.new, plus the :directory option. If fog adds support for extra
+providers they should work with just a config change to Db2Fog.
+
+=== Amazon S3
+
+  DB2Fog.config = {
+    :aws_access_key_id     => 'yourkey',
+    :aws_secret_access_key => 'yoursecretkey',
+    :directory             => 'bucket-name',
+    :provider              => 'AWS'
+  }
+
+=== Rackspace Cloudfiles
+
+  DB2Fog.config = {
+    :rackspace_username => 'username',
+    :rackspace_api_key  => 'api key',
+    :directory          => 'bucket-name',
+    :provider           => 'Rackspace'
+  }
+
+=== Local Storage
+
+  DB2Fog.config = {
+    :directory  => 'bucket-name',
+    :local_root => Rails.root.to_s + '/db/backups',
+    :provider   => 'Local'
+  }
+
+== Usage
+
+  # Add to your crontab or whatever
+  rake db2fog:backup:full
+
+  # Handy tasks
+  rake db2fog:statistics      # Shows you the size of your DB
+  rake db2fog:backup:restore  # You should be testing this regularly
+  rake db2fog:backup:clean    # Clean up old backups - cron this
+
+== Compatibility
+
+This is pure so ruby should run on most ruby VMs. I develop on MRI 1.9.2.
+
+This will only work work with rails 3. Supporting earlier versions is more
+complication than I can handle at the moment.
+
+== Development
+
+Specs are really weak. This code is bit hackish but is being used by quite a few people.
+
+== Kudos
+
+This is a fork of Xavier Shay's db2s3 gem. It worked perfectly, but only
+supported Amazon S3. By switching the dependency to using fog this now
+supports multiple storage providers.
+
+Xavier's original gem is available at https://github.com/xaviershay/db2s3
+
+Xavier quotes the following example as inspiration:
+
+http://github.com/pauldowman/blog_code_examples/tree/master/mysql_s3_backup

data/lib/db2fog.rb

@@ -0,0 +1,162 @@
+require 'active_support'
+require 'active_support/core_ext/class/attribute_accessors'
+require 'active_support/core_ext/hash/except'
+require 'fog'
+require 'tempfile'
+require 'db2fog/railtie'
+
+class DB2Fog
+  cattr_accessor :config
+
+  def full_backup
+    file_name = "dump-#{db_credentials[:database]}-#{Time.now.utc.strftime("%Y%m%d%H%M")}.sql.gz"
+    store.store(file_name, open(dump_db.path))
+    store.store(most_recent_dump_file_name, file_name)
+  end
+
+  def restore
+    dump_file_name = store.fetch(most_recent_dump_file_name).read
+    file = store.fetch(dump_file_name)
+    run "gunzip -c #{file.path} | mysql #{mysql_options}"
+  end
+
+  # TODO: This method really needs specs
+  def clean
+    to_keep = []
+    filelist = store.list
+    files = filelist.reject {|file| file.ends_with?(most_recent_dump_file_name) }.collect do |file|
+      {
+        :path => file,
+        :date => Time.parse(file.split('-').last.split('.').first)
+      }
+    end
+    # Keep all backups from the past day
+    files.select {|x| x[:date] >= 1.day.ago }.each do |backup_for_day|
+      to_keep << backup_for_day
+    end
+
+    # Keep one backup per day from the last week
+    files.select {|x| x[:date] >= 1.week.ago }.group_by {|x| x[:date].strftime("%Y%m%d") }.values.each do |backups_for_last_week|
+      to_keep << backups_for_last_week.sort_by{|x| x[:date].strftime("%Y%m%d") }.first
+    end
+
+    # Keep one backup per week since forever
+    files.group_by {|x| x[:date].strftime("%Y%W") }.values.each do |backups_for_week|
+      to_keep << backups_for_week.sort_by{|x| x[:date].strftime("%Y%m%d") }.first
+    end
+
+    to_destroy = filelist - to_keep.uniq.collect {|x| x[:path] }
+    to_destroy.delete_if {|x| x.ends_with?(most_recent_dump_file_name) }
+    to_destroy.each do |file|
+      store.delete(file.split('/').last)
+    end
+  end
+
+  def statistics
+      # From http://mysqlpreacher.com/wordpress/tag/table-size/
+    results = ActiveRecord::Base.connection.execute(<<-EOS)
+    SELECT
+      engine,
+      ROUND(data_length/1024/1024,2) total_size_mb,
+      ROUND(index_length/1024/1024,2) total_index_size_mb,
+      table_rows,
+      table_name article_attachment
+      FROM information_schema.tables
+      WHERE table_schema = '#{db_credentials[:database]}'
+      ORDER BY total_size_mb + total_index_size_mb desc;
+    EOS
+    rows = []
+    results.each {|x| rows << x.to_a }
+    rows
+  end
+
+  private
+
+  def dump_db
+    dump_file = Tempfile.new("dump")
+
+    cmd = "mysqldump --quick --single-transaction --create-options #{mysql_options}"
+    cmd += " | gzip > #{dump_file.path}"
+    run(cmd)
+
+    dump_file
+  end
+
+  def mysql_options
+    cmd = ''
+    cmd += " -u #{db_credentials[:username]} " unless db_credentials[:username].nil?
+    cmd += " -p'#{db_credentials[:password]}'" unless db_credentials[:password].nil?
+    cmd += " -h '#{db_credentials[:host]}'"    unless db_credentials[:host].nil?
+    cmd += " #{db_credentials[:database]}"
+  end
+
+  def store
+    @store ||= FogStore.new
+  end
+
+  def most_recent_dump_file_name
+    "most-recent-dump-#{db_credentials[:database]}.txt"
+  end
+
+  def run(command)
+    result = system(command)
+    raise("error, process exited with status #{$?.exitstatus}") unless result
+  end
+
+  def db_credentials
+    ActiveRecord::Base.connection.instance_eval { @config } # Dodgy!
+  end
+
+  class FogStore
+
+    def store(remote_filename, io)
+      unless directory.files.head(remote_filename)
+        directory.files.create(:key => remote_filename, :body => io, :public => false)
+      end
+    end
+
+    def fetch(remote_filename)
+      remote_file = directory.files.get(remote_filename)
+
+      file = Tempfile.new("dump")
+      open(file.path, 'w') { |f| f.write(remote_file.body) }
+      file
+    end
+
+    def list
+      directory.files.map { |f| f.key }
+    end
+
+    def delete(remote_filename)
+      remote_file = remote_file.head(remote_filename)
+      remote_file.destroy if remote_file
+    end
+
+    private
+
+    def fog_options
+      if DB2Fog.config.respond_to?(:[])
+        DB2Fog.config.except(:directory)
+      else
+        raise "DB2Fog not configured"
+      end
+    end
+
+    def directory_name
+      if DB2Fog.config.respond_to?(:[])
+      DB2Fog.config[:directory]
+      else
+        raise "DB2Fog not configured"
+      end
+    end
+
+    def directory
+      @directory ||= storage.directories.get(directory_name)
+    end
+
+    def storage
+      @storage = Fog::Storage.new(fog_options)
+    end
+  end
+
+end

data/lib/db2fog/railtie.rb

@@ -0,0 +1,11 @@
+require 'rails'
+require 'db2fog'
+
+class DB2Fog
+  class Railtie < Rails::Railtie
+
+    rake_tasks do
+      load File.expand_path('tasks.rb', File.dirname(__FILE__))
+    end
+  end
+end

data/lib/db2fog/tasks.rb

@@ -0,0 +1,28 @@
+# coding: utf-8
+
+namespace :db2fog do
+  namespace :backup do
+    desc "Save a full back to S3"
+    task :full => :environment do
+      DB2Fog.new.full_backup
+    end
+
+    desc "Restore your DB from S3"
+    task :restore => :environment do
+      DB2Fog.new.restore
+    end
+
+    desc "Keep all backups for the last day, one per day for the last week, and one per week before that. Delete the rest."
+    task :clean => :environment do
+      DB2Fog.new.clean
+    end
+  end
+
+  desc "Show table sizes for your database"
+  task :statistics => :environment do
+    rows = DB2Fog.new.statistics
+    rows.sort_by {|x| -x[3].to_i }
+    header = [["Type", "Data MB", "Index", "Rows", "Name"], []]
+    puts (header + rows).collect {|x| x.join("\t") }
+  end
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification 
+name: db2fog
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 4
+  version: "0.4"
+platform: ruby
+authors: 
+- James Healy
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-22 00:00:00 +10:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rails
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 3
+        - 0
+        version: "3.0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 3
+        - 0
+        version: "3.0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mysql2
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 2
+        - 0
+        version: 0.2.0
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: fog
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 9
+        - 0
+        version: 0.9.0
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 6
+        version: "2.6"
+  type: :development
+  version_requirements: *id006
+description: db2fig provides rake tasks for backing up and restoring your DB to cloud storage providers
+email: 
+- james@yob.id.au
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/db2fog.rb
+- lib/db2fog/railtie.rb
+- lib/db2fog/tasks.rb
+- README.rdoc
+- HISTORY
+has_rdoc: true
+homepage: http://github.com/yob/db2fog
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --title
+- DB2Fog
+- --line-numbers
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 3
+      - 2
+      version: 1.3.2
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: db2fog provides rake tasks for backing up and restoring your DB to cloud storage providers
+test_files: []
+