s3-backup 0.6.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Ben Koski
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,63 @@
+= s3-backup
+
+Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.
+
+== Install
+
+  sudo gem install s3-backup
+
+== Easy Example
+
+    b = S3Backup.new('vps-backups')  # Init with the name of bucket to push to; keys handled by AWSCredentials
+    
+    b.files << '/usr/local/important.data'
+    b.files << '/usr/local/more/important.data'
+    b.files << '/usr/local/secrets'
+    
+    b.run  
+
+This will:
+
+1. Create a .tar.gz file including everything in <tt>files</tt>
+2. Push tar to S3
+3. Delete old backups (keeps 5 by default, but you can change <tt>copies_to_keep</tt>)
+4. Remove scratch files
+
+== Hooks
+
+S3Backup also includes a <tt>before_backup</tt> and <tt>after_backup</tt> hook to take care of preparing dumpfiles
+and restarting services.  These are just blocks of code.
+
+Also useful, <tt>files_to_cleanup</tt> contains a list of files to cleanup post-backup.  You
+can add your scratch files to this list and they'll be deleted after each run, even if an error is encountered.
+
+<tt>tar_excludes</tt> is an array of patterns for tar to exclude -- for example, "*.log"
+
+For example:
+
+    b = S3Backup.new('vps-backups')
+
+    b.before_backup do
+      `mysqldump -h locahost important_data > /tmp/important_data.sql`
+      raise "mysqdump failed" if !$?.success?
+
+      b.files_to_cleanup << '/tmp/important_data.sql'
+      b.files << '/tmp/important_data.sql'
+    end
+
+    b.run
+
+You'll find additional opts in the S3Backup docs.
+
+== Suggested deployment
+
+Create a ruby script, and add to your crontab.  Don't forget to 2>&1 the output and set a <tt>MAILTO</tt> to get error notices.
+
+== Notes
+
+1. This uses AWSCredentials[http://github.com/bkoski/aws_credentials] to manage AWS keys.
+2. This doesn't work on Windows.
+
+== Copyright
+
+Copyright (c) 2010 Ben Koski. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,61 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "s3-backup"
+    gem.summary = %Q{Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.}
+    gem.description = %Q{Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.}
+    gem.email = "gems@benkoski.com"
+    gem.homepage = "http://github.com/bkoski/s3-backup"
+    gem.authors = ["Ben Koski"]
+    
+    gem.files += ["lib/s3-backup/base.rb"]
+    
+    gem.add_dependency "aws_credentials", ">= 0.6.0"
+    gem.add_dependency "right_aws", "~> 2.0.0"
+    
+    gem.add_development_dependency "shoulda", ">= 0"
+    gem.add_development_dependency "mocha", ">= 0"
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'hanna/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "s3-backup #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.6.0

data/lib/s3-backup.rb

@@ -0,0 +1,16 @@
+require 'right_aws'
+require 'aws_credentials'
+require 'fileutils'
+
+require File.join(File.dirname(__FILE__), 's3-backup', 'base')
+
+# To suppress "warning: peer certificate won't be verified in this SSL session" errors,
+# borrowed from http://www.5dollarwhitebox.org/drupal/node/64
+class Net::HTTP
+  alias_method :old_initialize, :initialize
+  def initialize(*args)
+    old_initialize(*args)
+    @ssl_context = OpenSSL::SSL::SSLContext.new
+    @ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  end
+end

data/lib/s3-backup/base.rb

@@ -0,0 +1,118 @@
+class S3Backup
+
+  # Array of files or paths to back up
+  attr_accessor :files
+
+  # Array of files that will be deleted post-backup, regardless of success
+  attr_accessor :files_to_cleanup
+
+  # Array of exclude patterns, these are passed to tar as <tt>--exclude</tt> flags
+  attr_accessor :tar_excludes
+  
+  # Number of backups to keep on S3.  Defaults to 5.
+  attr_accessor :copies_to_keep
+  
+  # Prefix for tarball, defaults to "backup".
+  attr_accessor :backup_name
+  
+  # Name of bucket to push to, set on initialize.
+  attr_reader :bucket_name
+  
+  # Initialize with the name of S3 bucket to push to
+  def initialize(bucket_name)
+    @files = []
+    @files_to_cleanup = []
+    @tar_excludes = []
+    @copies_to_keep = 5
+    @bucket_name = bucket_name
+    @backup_name = 'backup'
+    
+    @s3 =  RightAws::S3.new(AWSCredentials.access_key, AWSCredentials.secret_access_key, :logger => Logger.new(nil))
+  end
+
+  # Called before backup runs.  Useful for dumping a database, or creating files
+  # prior to tarball create.  As you create tmpfiles, you can push onto files_to_cleanup
+  # to ensure post-backup cleanup.
+  def before_backup &block
+    @before_backup = block
+  end
+
+  # Called after backup runs.  Useful for restarting services.
+  def after_backup &block
+    @after_backup = block
+  end
+
+  # Runs the backup: creates tarball, pushes to s3, and rotates old backups.
+  def run
+    begin    
+      @before_backup.call unless @before_backup.nil?
+    
+      create_tarball
+      push_to_s3
+      rotate_remote_backups
+    
+      @after_backup.call unless @after_backup.nil?
+    ensure
+      cleanup_files
+    end
+  end
+  
+  private
+  # Name of the tarball created locally
+  def tarball_name
+    @tarbarll_name ||= "/tmp/#{backup_name}-#{Time.now.to_i}.tar.gz"
+    @tarbarll_name
+  end
+  
+  # Name of the file passed to tar -I containing files to back up
+  def include_file_name
+    tarball_name.gsub('.tar.gz','-includes.txt')
+  end
+  
+  def create_tarball
+    raise ArgumentError, "files to backup is empty!" if files.empty?
+    
+    write_include_file
+    files_to_cleanup << include_file_name
+    
+    excludes = tar_excludes.collect { |e| "--exclude #{e}" }
+    run_tar(%{#{excludes.join(" ")} --preserve -I #{include_file_name} -czf #{tarball_name}})
+    
+    files_to_cleanup << tarball_name
+  end
+  
+  def write_include_file
+    File.open(include_file_name, 'w') { |f| f.write(files.join("\n")) }
+  end
+  
+  def run_tar params
+    output = `tar #{params} 2>&1`
+    raise "tar create failed with #{output}" if !$?.success?
+  end
+  
+  def s3_bucket
+    raise ArgumentError, "bucket_name must be set to run a backup!" if @bucket_name.nil?
+    if @s3_bucket.nil?
+      @s3_bucket = @s3.bucket(bucket_name)
+      raise ArgumentError, "bucket #{bucket_name} not found!" if @s3_bucket.nil?
+    end
+    
+    @s3_bucket
+  end
+  
+  def push_to_s3
+    s3_bucket.put(File.basename(tarball_name), File.open(tarball_name))
+  end
+  
+  def rotate_remote_backups
+    all_backups = s3_bucket.keys(:prefix => backup_name).sort_by { |k| k.name }.reverse
+    if all_backups.length > copies_to_keep
+      all_backups[copies_to_keep, all_backups.length - copies_to_keep].each { |k| k.delete }
+    end
+  end
+  
+  def cleanup_files
+    files_to_cleanup.each { |f| FileUtils.rm_rf(f) }
+  end
+  
+end

data/s3-backup.gemspec

@@ -0,0 +1,64 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{s3-backup}
+  s.version = "0.6.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ben Koski"]
+  s.date = %q{2010-10-17}
+  s.description = %q{Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.}
+  s.email = %q{gems@benkoski.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/s3-backup.rb",
+     "lib/s3-backup/base.rb",
+     "s3-backup.gemspec",
+     "test/base_test.rb",
+     "test/helper.rb"
+  ]
+  s.homepage = %q{http://github.com/bkoski/s3-backup}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.}
+  s.test_files = [
+    "test/base_test.rb",
+     "test/helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<aws_credentials>, [">= 0.6.0"])
+      s.add_runtime_dependency(%q<right_aws>, ["~> 2.0.0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+    else
+      s.add_dependency(%q<aws_credentials>, [">= 0.6.0"])
+      s.add_dependency(%q<right_aws>, ["~> 2.0.0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<aws_credentials>, [">= 0.6.0"])
+    s.add_dependency(%q<right_aws>, ["~> 2.0.0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+  end
+end
+

data/test/base_test.rb

@@ -0,0 +1,164 @@
+require 'helper'
+
+class BaseTest < Test::Unit::TestCase
+    
+  def setup
+    @b = S3Backup.new('test')
+    @b.files = ['/tmp/test1.txt']
+  end
+  
+  context "callbacks" do
+    setup do
+      stub_io_methods
+    end
+  
+    should "call before_backup block before backup is run" do
+      s = sequence('backup')
+
+      before_mock = mock()
+      before_mock.expects(:test_cmd).in_sequence(s)
+      @b.before_backup { before_mock.test_cmd }
+
+      @b.expects(:create_tarball).in_sequence(s)
+      
+      @b.run
+    end
+  
+    should "call after_backup block after backup is run" do
+      s = sequence('backup')
+      
+      @b.expects(:push_to_s3).in_sequence(s)
+      
+      after_mock = mock()
+      after_mock.expects(:test_cmd).in_sequence(s)
+      @b.after_backup { after_mock.test_cmd }
+
+      @b.run
+    end
+  end
+  
+  context "cleanup" do
+    setup do
+      stub_io_methods
+    end
+    
+    should "remove everything specified in files_to_delete on success" do
+      @b.files_to_cleanup << '/tmp/testtmpfile.txt'
+      FileUtils.expects(:rm_rf).with('/tmp/testtmpfile.txt')
+      @b.run
+    end
+    
+    should "remove everything specified in files_to_delete even if an exception was raised" do
+      @b.files_to_cleanup << '/tmp/testtmpfile.txt'
+      FileUtils.expects(:rm_rf).with('/tmp/testtmpfile.txt')
+      @b.stubs(:create_tarball).raises('test error')
+      @b.run rescue nil
+    end
+    
+    should "remove local tarfile" do
+      FileUtils.expects(:rm_rf).with(@b.tarball_name)
+      @b.run
+    end
+    
+    should "remove local includes file" do
+      FileUtils.expects(:rm_rf).with(@b.include_file_name)
+      @b.run
+    end
+  end
+  
+  context "tar file" do
+    
+    setup do
+      stub_io_methods(:run_tar, :rotate_remote_backups)
+            
+      @tarball_mock = mock()
+      File.stubs(:open).with(@b.tarball_name).returns(@tarball_mock)
+      
+      @include_file_mock = mock()
+      @include_file_mock.stubs(:write)
+      File.stubs(:open).with(@b.include_file_name, 'w').yields(@include_file_mock)      
+    end
+    
+    context "naming" do
+      should "default to backup" do
+        @b.expects(:run_tar).with(regexp_matches(/backup-\d+.tar.gz$/))
+        @b.run
+      end
+      
+      should "include backup_name if specified" do
+        @b = S3Backup.new('test')
+        
+        @b.files = ['/tmp/test1.txt']
+        @b.backup_name = 'app_data'
+        
+        File.stubs(:open).with(@b.include_file_name, 'w').yields(@include_file_mock)
+        File.stubs(:open).with(anything).returns(mock())
+        
+        @b.expects(:run_tar).with(regexp_matches(/app_data-\d+.tar.gz$/))
+        @b.run
+      end
+    end
+
+    should "include all files named in files" do
+      @include_file_mock.expects(:write).with(@b.files.join("\n"))
+      @b.expects(:run_tar).with(regexp_matches(/-I #{@b.include_file_name}/))
+      @b.run
+    end
+    
+    should "exclude tar_excludes" do
+      @b.tar_excludes = ['*.log','*.txt']
+      @b.expects(:run_tar).with(regexp_matches(/--exclude \*.log --exclude \*.txt/))
+      @b.run
+    end
+    
+    should "be run with --perserve" do
+      @b.expects(:run_tar).with(regexp_matches(/--preserve /))
+      @b.run
+    end
+    
+    should "be pushed to s3" do
+      @s3_bucket.expects(:put).with(File.basename(@b.tarball_name), @tarball_mock)
+      @b.run
+    end
+  end
+  
+  context "rotation" do
+    setup do
+      stub_io_methods(:run_tar, :write_include_file, :push_to_s3)
+    end
+    
+    should "not remove anything if total count is < copies_to_keep" do
+      mock_keys = [mock(),mock()]
+      mock_keys.each_with_index do |k,i|
+        k.stubs(:name).returns(i.to_s)
+        k.expects(:delete).never()
+      end
+      
+      @s3_bucket.stubs(:keys).with(:prefix => @b.backup_name).returns(mock_keys)
+      @b.run
+    end
+    
+    should "remove oldest files if total count is > copies_to_keep" do
+      mock_keys = [mock(:name => 'file-299'),mock(:name => 'file-300'),mock(:name => 'file-301'),mock(:name => 'file-302'),mock(:name => 'file-303')]
+      mock_keys[0,@b.copies_to_keep].each { |k| k.expects(:delete).never() }
+      
+      extra_keys = [mock(:name => 'file-100'),mock(:name => 'file-102')]
+      extra_keys.each { |k| k.expects(:delete) }
+      mock_keys += extra_keys
+      
+      @s3_bucket.stubs(:keys).with(:prefix => @b.backup_name).returns(mock_keys)
+      @b.run
+    end
+  end
+  
+  private
+  def stub_io_methods *methods
+    methods = [:run_tar, :write_include_file, :push_to_s3, :rotate_remote_backups] if methods.empty?
+    methods.each { |io| S3Backup.any_instance.stubs(io) }
+    @s3_bucket = mock()
+    @s3_bucket.stubs(:put)
+    S3Backup.any_instance.stubs(:s3_bucket).returns(@s3_bucket)
+    FileUtils.stubs(:rm_rf)
+  end
+  
+end

data/test/helper.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+require 'ruby-debug'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 's3-backup'
+
+class S3Backup
+  # These methods become relevant in tests
+  public :tarball_name, :include_file_name
+end
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification 
+name: s3-backup
+version: !ruby/object:Gem::Version 
+  hash: 7
+  prerelease: false
+  segments: 
+  - 0
+  - 6
+  - 0
+  version: 0.6.0
+platform: ruby
+authors: 
+- Ben Koski
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-17 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aws_credentials
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 6
+        - 0
+        version: 0.6.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 0
+        - 0
+        version: 2.0.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.
+email: gems@benkoski.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/s3-backup.rb
+- lib/s3-backup/base.rb
+- s3-backup.gemspec
+- test/base_test.rb
+- test/helper.rb
+has_rdoc: true
+homepage: http://github.com/bkoski/s3-backup
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Easy tar-based backups to S3, with backup rotation, cleanup helpers, and pre-/post-backup hooks.
+test_files: 
+- test/base_test.rb
+- test/helper.rb