bostonlogic-safe 0.3.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Astrails Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,227 @@
+astrails-safe
+=============
+
+Simple database and filesystem backups with S3 and Rackspace Cloudfiles support (with optional encryption)
+
+Home: github.com/astrails/safe
+
+Motivation
+----------
+
+We needed a backup solution that will satisfy the following requirements:
+
+* opensource
+* simple to install and configure
+* support for simple ‘tar’ backups of directories (with includes/excludes)
+* support for simple mysqldump of mysql databases
+* support for symmetric or public key encryption
+* support for local filesystem and Amazon S3 for storage and Rackspace Cloudfile storage
+* support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3
+* email notification on backup failure with error and backtrace
+
+And since we didn't find any, we wrote our own :)
+
+Contributions
+-------------
+
+The following functionality was contributed by astrails-safe users:
+
+* PostgreSQL dump using pg_dump (by Mark Mansour <mark@stateofflux.com>)
+* Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
+* SFTP remote storage (by Adam <adam@mediadrive.ca>)
+* benchmarking output (By Neer)
+
+
+Thanks to all :)
+
+Reporting problems
+------------------
+
+Please report problems at the [Issues tracker](http://github.com/astrails/safe/issues)
+
+Usage
+-----
+
+    Usage:
+       astrails-safe [OPTIONS] CONFIG_FILE
+    Options:
+      -h, --help           This help screen
+      -v, --verbose        be verbose, duh!
+      -n, --dry-run        just pretend, don't do anything.
+      -L, --local          skip S3
+
+Note: CONFIG_FILE will be created from template if missing
+
+Encryption
+----------
+
+If you want to encrypt your backups you have 2 options:
+* use simple password encryption
+* use GPG public key encryption
+
+For simple password, just add password entry in gpg section.
+For public key encryption you will need to create a public/secret keypair.
+
+We recommend to create your GPG keys only on your local machine and then
+transfer your public key to the server that will do the backups.
+
+This way the server will only know how to encrypt the backups but only you
+will be able to decrypt them using the secret key you have locally. Of course
+you MUST backup your backup encryption key :)
+We recommend also pringing the hard paper copy of your GPG key 'just in case'.
+
+The procedure to create and transfer the key is as follows:
+
+1. run 'gpg --gen-key' on your local machine and follow onscreen instructions to create the key
+   (you can accept all the defaults).
+
+2. extract your public key into a file (assuming you used test@example.com as your key email):
+   gpg -a --export test@example.com > test@example.com.pub
+
+3. transfer public key to the server
+   scp test@example.com.pub root@example.com:
+
+4. import public key on the remote system:
+<pre>
+   $ gpg --import test@example.com.pub
+   gpg: key 45CA9403: public key "Test Backup <test@example.com>" imported
+   gpg: Total number processed: 1
+   gpg:               imported: 1
+</pre>
+
+5. since we don't keep the secret part of the key on the remote server, gpg has
+   no way to know its yours and can be trusted.
+   To fix that we can sign it with other trusted key, or just directly modify its
+   trust level in gpg (use level 5):
+   <pre>
+     $ gpg --edit-key test@example.com
+     ...
+     Command> trust
+     ...
+     1 = I don't know or won't say
+     2 = I do NOT trust
+     3 = I trust marginally
+     4 = I trust fully
+     5 = I trust ultimately
+     m = back to the main menu
+
+     Your decision? 5
+     ...
+     Command> quit
+   </pre>
+
+6. export your secret key for backup
+   (we recommend to print it on paper and burn to a CD/DVD and store in a safe place):
+   <pre>
+   $ gpg -a --export-secret-key test@example.com > test@example.com.key
+   </pre>
+
+
+Example configuration
+---------------------
+<pre>
+  safe do
+    local :path => "/backup/:kind/:id"
+
+    s3 do
+      key "...................."
+      secret "........................................"
+      bucket "backup.astrails.com"
+      path "servers/alpha/:kind/:id"
+    end
+    
+    rcloud do
+      username "username"
+      api_key "key"
+      container "backups"
+      path ":kind/"
+    end
+
+    sftp do
+      host "sftp.astrails.com"
+      user "astrails"
+      password "ssh password for sftp"
+    end
+
+    gpg do
+      # symmetric encryption key
+      # password "qwe"
+
+      # public GPG key (must be known to GPG, i.e. be on the keyring)
+      key "backup@astrails.com"
+    end
+
+    keep do
+      local 2
+      s3 2
+    end
+    
+    notification do
+      subject "safe backup failure"
+      host "mail.example.com"
+      domain "example.com"
+      username "safe@example.com"
+      password "example"
+      authentication :login
+      port 25
+      from "example@example.com"
+      recipients "developement_staff@example.com"
+    end
+
+    mysqldump do
+      options "-ceKq --single-transaction --create-options"
+
+      user "root"
+      password "............"
+      socket "/var/run/mysqld/mysqld.sock"
+
+      database :blog
+      database :servershape
+      database :astrails_com
+      database :secret_project_com
+
+    end
+
+    svndump do
+      repo :my_repo do
+        repo_path "/home/svn/my_repo"
+      end
+    end
+
+    pgdump do
+      options "-i -x -O"   # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
+
+      user "username"
+      password "............"  # shouldn't be used, instead setup ident.  Current functionality exports a password env to the shell which pg_dump uses - untested!
+
+      database :blog
+      database :stateofflux_com
+    end
+
+    tar do
+      archive "git-repositories", :files => "/home/git/repositories"
+      archive "dot-configs",      :files => "/home/*/.[^.]*"
+      archive "etc",              :files => "/etc", :exclude => "/etc/puppet/other"
+
+      archive "blog-astrails-com" do
+        files "/var/www/blog.astrails.com/"
+        exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
+      end
+
+      archive "astrails-com" do
+        files "/var/www/astrails.com/"
+        exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
+      end
+    end
+  end
+</pre>
+
+Reporting problems
+------------------
+
+http://github.com/astrails/safe/issues
+
+Copyright
+---------
+
+Copyright (c) 2009 Astrails Ltd. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,54 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "safe"
+    gem.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) to Amazon S3 or Rackspace Cloudfiles (with encryption)}
+    gem.description = "Simple tool to backup databases (MySQL and PostgreSQL) and filesystem locally or to Amazon S3 or Rackspace Cloudfiles (with optional encryption)"
+    gem.email = "we@astrails.com"
+    gem.homepage = "http://github.com/astrails/safe"
+    gem.authors  = ["Astrails Ltd.", "Mark Mansour"]
+    gem.files = FileList["[A-Z]*.*", "{bin,examples,generators,lib,rails,spec,test,templates}/**/*", 'Rakefile', 'LICENSE*']
+
+    gem.add_dependency("aws-s3")
+    gem.add_dependency("net-sftp")
+    gem.add_dependency("rackspace-cloudfiles")
+
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+require 'micronaut/rake_task'
+Micronaut::RakeTask.new(:examples) do |examples|
+  examples.pattern = 'examples/**/*_example.rb'
+  examples.ruby_opts << '-Ilib -Iexamples'
+end
+
+Micronaut::RakeTask.new(:rcov) do |examples|
+  examples.pattern = 'examples/**/*_example.rb'
+  examples.rcov_opts = '-Ilib -Iexamples -iastrails -x\/gems\/'
+  examples.rcov = true
+end
+
+
+task :default => :examples
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "safe #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:minor: 3
+:patch: 0
+:major: 0

data/bin/astrails-safe

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+
+#require 'ruby-debug'
+#$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require File.dirname(__FILE__) + '/../lib/astrails/safe'
+include Astrails::Safe
+
+def die(msg)
+  puts "ERROR: #{msg}"
+  exit 1
+end
+
+def usage
+  puts <<-END
+Usage: astrails-safe [OPTIONS] CONFIG_FILE
+Options:
+  -h, --help           This help screen
+  -v, --verbose        be verbose, duh!
+  -n, --dry-run        just pretend, don't do anything.
+  -L, --local          skip S3, skip Rackspace Cloud
+
+Note: config file will be created from template if missing
+END
+  exit 1
+end
+
+def process_options
+  usage if ARGV.delete("-h") || ARGV.delete("--help")
+  $_VERBOSE = ARGV.delete("-v") || ARGV.delete("--verbose")
+  $DRY_RUN = ARGV.delete("-n") || ARGV.delete("--dry-run")
+  $LOCAL   = ARGV.delete("-L") || ARGV.delete("--local")
+  usage unless ARGV.first
+  $CONFIG_FILE_NAME = File.expand_path(ARGV.first)
+end
+
+def main
+  process_options
+
+  unless File.exists?($CONFIG_FILE_NAME)
+    die "Missing configuration file. NOT CREATED! Rerun w/o the -n argument to create a template configuration file." if $DRY_RUN
+
+    FileUtils.cp File.join(Astrails::Safe::ROOT, "templates", "script.rb"), $CONFIG_FILE_NAME
+
+    die "Created default #{$CONFIG_FILE_NAME}. Please edit and run again."
+  end
+
+  load($CONFIG_FILE_NAME)
+end
+
+main

data/examples/example_helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'micronaut'
+require 'ruby-debug'
+
+SAFE_ROOT = File.dirname(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(SAFE_ROOT, 'lib'))
+
+require 'astrails/safe'
+
+def not_in_editor?
+  !(ENV.has_key?('TM_MODE') || ENV.has_key?('EMACS') || ENV.has_key?('VIM'))
+end
+
+Micronaut.configure do |c|
+  c.color_enabled = not_in_editor?
+  c.filter_run :focused => true
+  c.mock_with :rr
+end

data/examples/integration/archive_integration_example.rb

@@ -0,0 +1,86 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "tmp/archive_backup_example"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p "#{@src}/q/w/e"
+    mkdir_p "#{@src}/a/s/d"
+
+    File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
+    File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
+    File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
+    File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
+
+    File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
+    File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
+    File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        archive :test1 do
+          files src
+          exclude "#{src}/q/w"
+        end
+      end
+    end
+
+    @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar.gz"
+  end
+
+  it "should create backup file" do
+    puts "Expecting: #{@backup}"
+    File.exists?(@backup).should be_true
+  end
+
+  describe "after extracting" do
+    before(:all) do
+      # prepare target dir
+      @target = "#{@root}/test"
+      mkdir_p @target
+      system "tar -zxvf #{@backup} -C #{@target}"
+
+      @test = "#{@target}/#{@root}/src"
+      puts @test
+    end
+
+    it "should include asd1/2/3" do
+      File.exists?("#{@test}/asd1").should be_true
+      File.exists?("#{@test}/a/asd2").should be_true
+      File.exists?("#{@test}/a/s/asd3").should be_true
+    end
+
+    it "should only include qwe 1 and 2 (no 3)" do
+      File.exists?("#{@test}/qwe1").should be_true
+      File.exists?("#{@test}/q/qwe2").should be_true
+    end
+
+    it "should preserve file content" do
+      File.read("#{@test}/qwe1").should == "qwe"
+      File.read("#{@test}/q/qwe2").should == "qweqwe"
+      File.read("#{@test}/a/s/asd3").should == "asdasdasd"
+    end
+  end
+
+end