sauberia-db2s3 0.3.2.1

data/.gitignore

@@ -0,0 +1,2 @@
+spec/s3_config.rb
+pkg

data/HISTORY

@@ -0,0 +1,15 @@
+0.3.1 (7 Dec 2009)
+  - Fixed hardcoded DB name in statistics task
+0.3.0 (6 Dec 2009)
+  - Added db2s3:backup:clean task to delete old backups
+  - Added dependency on activesupport for the clean task
+0.2.6 (6 Dec 2009)
+  - Remove metrics task, since it was far too inaccurate
+  - Add statistics task to show you the size of your tables
+  - Only add username to mysql command line if provided in database.yml
+0.2.5
+  - Use host provided in database.yml
+0.2.4 (2 Sep 2009)
+  - Fix credentials bug
+0.2.3
+  - Keep old backups around

data/README.rdoc

@@ -0,0 +1,40 @@
+= DB2S3
+
+A rails plugin to backup Mysql to Amazon S3. You're looking at a monthly spend of four cents. So pony up you cheap bastard, and store your backups on S3
+
+== Usage
+
+  # In config/environment.rb
+  config.gem "db2s3", :source => "http://gemcutter.org"
+
+  # In Rakefile
+  require 'db2s3/tasks'
+
+  # In config/initializers/db2s3.rb
+  DB2S3::Config.instance_eval do
+    S3 = {
+      :access_key_id     => 'yourkey',
+      :secret_access_key => 'yoursecretkey',
+      :bucket            => 'yourapp-db-backup'
+    }
+  end
+  # DB credentials are read from your rails environment
+
+  rake gems:install
+
+  # Add to your crontab or whatever
+  rake db2s3:backup:full
+  rake db2s3:backup:incremental # Unimplemented
+
+  # Handy tasks
+  rake db2s3:statistics      # Shows you the size of your DB
+  rake db2s3:backup:restore  # You should be testing this regularly
+  rake db2s3:backup:clean    # Clean up old backups - cron this
+
+== Development
+
+Specs are really week. This code is bit hackish but is being used by quite a few people.
+
+== Kudos
+
+http://github.com/pauldowman/blog_code_examples/tree/master/mysql_s3_backup

data/Rakefile

@@ -0,0 +1,4 @@
+desc "Create test database"
+task :create_test_db do
+  `mysqladmin -u root create db2s3_unittest`
+end

data/VERSION

@@ -0,0 +1 @@
+0.3.2

data/init.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + "/rails/init"

data/lib/db2s3/tasks.rb

@@ -0,0 +1 @@
+Dir["#{File.dirname(__FILE__)}/../../tasks/*.rake"].each { |ext| load ext }

data/lib/db2s3.rb

@@ -0,0 +1,177 @@
+begin
+  require 'active_support' # The new one
+rescue LoadError
+  require 'activesupport' # The old one
+end
+require 's3'
+require 'tempfile'
+
+class DB2S3
+  class Config
+  end
+
+  def initialize
+  end
+
+  def full_backup
+    file_name = dump_file_name(Time.now)
+    store.store(file_name, open(dump_db.path))
+    store.store(most_recent_dump_file_name, file_name)
+  end
+
+  def restore
+    dump_file_name = store.fetch(most_recent_dump_file_name).read
+    file = store.fetch(dump_file_name)
+    run "gunzip -c #{file.path} | mysql #{mysql_options}"
+  end
+
+  # TODO: This method really needs specs
+  def clean
+    files = file_objects_from_paths(store.list("#{dump_file_name_prefix}-"))
+    determine_what_to_keep files
+    delete_surplus_backups files
+  end
+  
+  def file_objects_from_paths paths
+    paths.collect do |path| {
+        :path => path,
+        :date => Time.parse(path.split('-').last.split('.').first),
+        :keep => false
+      }
+    end
+  end
+  
+  def determine_what_to_keep files
+    # Keep all backups from the past day
+    files.select {|x| x[:date] >= 1.day.ago }.map! do |backup_for_day|
+      backup_for_day[:keep] = true
+    end
+    # Keep one backup per day from the last week
+    files.select {|x| x[:date] >= 1.week.ago }.group_by {|x| x[:date].strftime("%u") }.values.map! do |backups_for_last_week|
+      backups_for_last_week.sort_by{|x| x[:path] }.first[:keep] = true
+    end
+    # Keep one backup per week from the last 28 days
+    files.select {|x| x[:date] >= 28.days.ago }.group_by {|x| x[:date].strftime("%Y%W") }.values.map! do |backups_for_last_28_days|
+      backups_for_last_28_days.sort_by{|x| x[:path] }.first[:keep] = true
+    end
+    # Keep one backup per month since forever
+    files.group_by {|x| x[:date].strftime("%Y%m") }.values.map! do |backups_for_month|
+      backups_for_month.sort_by{|x| x[:path] }.first[:keep] = true
+    end
+  end
+  
+  def delete_surplus_backups files
+    files.each do |file|
+      store.delete(file[:path]) unless file[:keep]
+    end
+  end
+
+  def statistics
+      # From http://mysqlpreacher.com/wordpress/tag/table-size/
+    results = ActiveRecord::Base.connection.execute(<<-EOS)
+    SELECT
+      engine,
+      ROUND(data_length/1024/1024,2) total_size_mb,
+      ROUND(index_length/1024/1024,2) total_index_size_mb,
+      table_rows,
+      table_name article_attachment
+      FROM information_schema.tables
+      WHERE table_schema = '#{db_credentials[:database]}'
+      ORDER BY total_size_mb + total_index_size_mb desc;
+    EOS
+    rows = []
+    results.each {|x| rows << x.to_a }
+    rows
+  end
+
+  private
+
+  def dump_db
+    dump_file = Tempfile.new('dump')
+    cmd = "mysqldump --quick --single-transaction --create-options #{mysql_options}"
+    cmd += " | gzip > #{dump_file.path}"
+    run(cmd)
+
+    dump_file
+  end
+
+  def mysql_options
+    cmd = ''
+    cmd += " -u #{db_credentials[:username]} " unless db_credentials[:username].nil?
+    cmd += " -p'#{db_credentials[:password]}'" unless db_credentials[:password].nil?
+    cmd += " -h '#{db_credentials[:host]}'"    unless db_credentials[:host].nil?
+    cmd += " #{db_credentials[:database]}"
+  end
+
+  def store
+    @store ||= S3Store.new
+  end
+  
+  def dump_file_name_prefix
+    "dump-#{db_credentials[:database]}"
+  end
+  
+  def dump_file_name time
+    "#{dump_file_name_prefix}-#{time.utc.strftime("%Y%m%d%H%M%S")}.sql.gz"
+  end
+
+  def most_recent_dump_file_name
+    "most-recent-#{dump_file_name_prefix}.txt"
+  end
+
+  def run(command)
+    result = system(command)
+    raise("error, process exited with status #{$?.exitstatus}") unless result
+  end
+
+  def db_credentials
+    ActiveRecord::Base.connection.instance_eval { @config } # Dodgy!
+  end
+
+  class S3Store
+    def initialize
+      @connected = false
+    end
+
+    def ensure_connected
+      return if @connected
+      s3_service = S3::Service.new(DB2S3::Config::S3.slice(:access_key_id, :secret_access_key).merge(:use_ssl => true))
+      @bucket = s3_service.buckets.build(DB2S3::Config::S3[:bucket])
+      @connected = true
+    end
+
+    def store(file_name, file)
+      ensure_connected
+      object = bucket.objects.build(file_name)
+      object.content = file.class == String ? file : (file.rewind; file.read)
+      object.save
+    end
+
+    def fetch(file_name)
+      ensure_connected
+      file = Tempfile.new('dump')
+      file.binmode if file.respond_to?(:binmode)
+      file.write(bucket.objects.find(file_name).content)
+      file.rewind
+      file
+    end
+
+    def list(prefix)
+      ensure_connected
+      bucket.objects.find_all(:prefix => prefix).collect {|x| x.key }
+    end
+
+    def delete(file_name)
+      if object = bucket.objects.find(file_name)
+        object.delete
+      end
+    end
+
+    private
+
+    def bucket
+      @bucket
+    end
+  end
+
+end

data/rails/init.rb

@@ -0,0 +1 @@
+require File.expand_path(File.dirname(__FILE__) + '/../lib/db2s3')

data/spec/db2s3_spec.rb

@@ -0,0 +1,26 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe 'db2s3' do
+  def load_schema
+    `cat '#{File.dirname(__FILE__) + '/mysql_schema.sql'}' | mysql -u #{DBConfig[:username]} #{DBConfig[:database]}`
+  end
+
+  def drop_schema
+    `cat '#{File.dirname(__FILE__) + '/mysql_drop_schema.sql'}' | mysql -u #{DBConfig[:username]} #{DBConfig[:database]}`
+  end
+
+  class Person < ActiveRecord::Base
+  end
+
+  if DB2S3::Config.const_defined?('S3')
+    it 'can save and restore a backup to S3' do
+      db2s3 = DB2S3.new
+      load_schema
+      Person.create!(:name => "Baxter")
+      db2s3.full_backup
+      drop_schema
+      db2s3.restore
+      Person.find_by_name("Baxter").should_not be_nil
+    end
+  end
+end

data/spec/mysql_drop_schema.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS people;

data/spec/mysql_schema.sql

@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS people;
+CREATE TABLE people (
+name VARCHAR(255) NULL
+);

data/spec/s3_config.example.rb

@@ -0,0 +1,5 @@
+DB2S3::Config:: S3 = {
+  :access_key_id     => 'yourkey',
+  :secret_access_key => 'yoursecretkey',
+  :bucket            => 'db2s3_test'
+}

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'spec'
+
+begin
+  require 'active_record' # The new one
+rescue LoadError
+  require 'activerecord' # The old one
+end
+
+require File.dirname(__FILE__) + '/../lib/db2s3'
+if File.exists?(File.dirname(__FILE__) + '/s3_config.rb')
+  require File.dirname(__FILE__) + '/s3_config.rb'
+else
+  puts "s3_config.rb does not exist - not running live tests"
+end
+
+DBConfig = {
+  :adapter  => "mysql",
+  :encoding => "utf8",
+  :database => 'db2s3_unittest',
+  :username => "root"
+}
+
+ActiveRecord::Base.configurations = { 'production' => DBConfig }
+ActiveRecord::Base.establish_connection(:production)

data/tasks/tasks.rake

@@ -0,0 +1,26 @@
+namespace :db2s3 do
+  namespace :backup do
+    desc "Save a full back to S3"
+    task :full => :environment do
+      DB2S3.new.full_backup
+    end
+
+    desc "Restore your DB from S3"
+    task :restore => :environment do
+      DB2S3.new.restore
+    end
+
+    desc "Keep all backups for the last day, one per day for the last week, one per week for the last 28 days, and one per month before that. Delete the rest."
+    task :clean => :environment do
+      DB2S3.new.clean
+    end
+  end
+
+  desc "Show table sizes for your database"
+  task :statistics => :environment do
+    rows = DB2S3.new.statistics
+    rows.sort_by {|x| -x[3].to_i }
+    header = [["Type", "Data MB", "Index", "Rows", "Name"], []]
+    puts (header + rows).collect {|x| x.join("\t") }
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+name: sauberia-db2s3
+version: !ruby/object:Gem::Version 
+  hash: 93
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 2
+  - 1
+  version: 0.3.2.1
+platform: ruby
+authors: 
+- Xavier Shay
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-22 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: s3
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 0
+        - 3
+        - 7
+        version: 0.3.7
+  type: :runtime
+  version_requirements: *id001
+description: db2s3 provides rake tasks for backing up and restoring your DB to S3
+email: contact@rhnh.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .gitignore
+- HISTORY
+- README.rdoc
+- Rakefile
+- VERSION
+- init.rb
+- lib/db2s3.rb
+- lib/db2s3/tasks.rb
+- rails/init.rb
+- spec/db2s3_spec.rb
+- spec/mysql_drop_schema.sql
+- spec/mysql_schema.sql
+- spec/s3_config.example.rb
+- spec/spec_helper.rb
+- tasks/tasks.rake
+has_rdoc: true
+homepage: http://github.com/xaviershay/db2s3
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Summarize your gem
+test_files: 
+- spec/db2s3_spec.rb
+- spec/s3_config.example.rb
+- spec/spec_helper.rb