netguru-safe 0.2.9

data/.autotest

@@ -0,0 +1,42 @@
+require 'autotest'
+
+Autotest.add_hook :initialize do |at|
+  at.clear_mappings
+  at.failed_results_re = /^\d+\)\n(?:\e\[\d*m)?(?:.*?Error in )?'([^\n]*)'(?: FAILED)?(?:\e\[\d*m)?\n(.*?)\n\n/m
+  at.completed_re = /\n(?:\e\[\d*m)?\d* examples?/m
+
+  at.add_mapping(%r%^examples/.*_example.rb$%) { |filename, _|
+    filename
+  }
+
+  at.add_mapping(%r%^lib/(.*)\.rb$%) { |filename, m|
+    ["examples/lib/#{m[1]}_example.rb"]
+  }
+
+  at.add_mapping(%r%^examples/(example_helper|shared/.*)\.rb$%) {
+    at.files_matching %r%^examples/.*_example\.rb$%
+  }
+end
+
+class MicronautCommandError < StandardError; end
+
+class Autotest::Micronaut < Autotest
+  #remove_method :consolidate_failures, :make_test_cmd
+  #remove_method :make_test_cmd
+
+  def consolidate_failures(failed)
+    filters = new_hash_of_arrays
+    failed.each do |spec, trace|
+      if trace =~ /\n(\.\/)?(.*example\.rb):[\d]+:\Z?/
+        filters[$2] << spec
+      end
+    end
+    return filters
+  end
+
+  def make_test_cmd(files_to_test)
+    return '' if files_to_test.size == 0
+    "bin/micronaut #{files_to_test.keys.sort.join(' ')}"
+  end
+
+end

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,11 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+tmp
+*~
+tags
+log/
+.bundle/
+vendor/bundle

data/CHANGELOG

@@ -0,0 +1,25 @@
+0.2.8
+
+* ruby 1.9.2 compatibility (tests mostly)
+* code review, and tons of small fixes
+* check file size before attempting to upload to cloudfiles
+* testing framework changed from micronaut to rspec
+
+0.2.7
+
+* default options for gpg now include '--no-use-agent'
+* support for 'command' option for gpg
+* quote values in mysql password file
+* add 'lib' to $:
+* [EXPERIMENTAL] Rackspace Cloud Files support
+
+0.2.6
+
+* fix typo in the template config file. (change option to options in pgdump)
+* add example 'options' for tar in the template config file.
+* do not try to upload more then 5G of data to S3. print error instead
+
+0.2.5
+
+* Safety mesure: Disable overwrite of existing configuration keys except for multi-value keys
+  supported multi-value keys: skip_tables, exclude, files

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+gemspec
+
+gem 'rake', '0.8.7'

data/Gemfile.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    astrails-safe (0.2.9)
+      aws-sdk (~> 1.2.3)
+      cloudfiles (~> 1.4.7)
+      net-sftp (~> 2.0.4)
+      toadhopper (~> 2.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aws-sdk (1.2.3)
+      httparty (~> 0.7)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    cloudfiles (1.4.18)
+      mime-types (>= 1.16)
+    httparty (0.8.1)
+      multi_json
+      multi_xml
+    json (1.6.2)
+    mime-types (1.17.2)
+    multi_json (1.0.4)
+    multi_xml (0.4.1)
+    net-sftp (2.0.5)
+      net-ssh (>= 2.0.9)
+    net-ssh (2.2.1)
+    nokogiri (1.5.0)
+    rake (0.8.7)
+    rr (1.0.4)
+    rspec (1.3.2)
+    toadhopper (2.0)
+    uuidtools (2.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  astrails-safe!
+  rake (= 0.8.7)
+  rr (~> 1.0.4)
+  rspec (~> 1.3.2)

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Astrails Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,237 @@
+astrails-safe
+=============
+
+Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption)
+
+* Home: [http://astrails.com/opensource/astrails-safe](http://astrails.com/opensource/astrails-safe)
+* Code: [http://github.com/astrails/safe](http://github.com/astrails/safe)
+* Blog: [http://blog.astrails.com/astrails-safe](http://blog.astrails.com/astrails-safe)
+
+
+Motivation
+----------
+
+We needed a backup solution that will satisfy the following requirements:
+
+* opensource
+* simple to install and configure
+* support for simple ‘tar’ backups of directories (with includes/excludes)
+* support for simple mysqldump of mysql databases
+* support for symmetric or public key encryption
+* support for local filesystem, Amazon S3, and Rackspace Cloud Files for storage
+* support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3 or Cloud Files
+
+And since we didn't find any, we wrote our own :)
+
+Contributions
+-------------
+
+The following functionality was contributed by astrails-safe users:
+
+* PostgreSQL dump using `pg_dump` (by Mark Mansour <mark@stateofflux.com>)
+* Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
+* SFTP remote storage (by Adam <adam@mediadrive.ca>)
+* benchmarking output (By Neer)
+* README fixes (by Bobby Wilson)
+* improved config file parsing (by Fedor Kocherga <fkocherga@gmail.com>)
+* mysql password file quoting (by Jonathan Sutherland <jonathan.sutherland@gmail.com>)
+* Rackspace Cloud Files support (by H. Wade Minter <minter@lunenburg.org>)
+
+Thanks to all :)
+
+Installation
+------------
+
+    sudo gem install astrails-safe --source http://gemcutter.org
+
+Reporting problems
+------------------
+
+Please report problems at the [Issues tracker](http://github.com/astrails/safe/issues)
+
+Usage
+-----
+
+    Usage:
+       astrails-safe [OPTIONS] CONFIG_FILE
+    Options:
+      -h, --help           This help screen
+      -v, --verbose        be verbose, duh!
+      -n, --dry-run        just pretend, don't do anything.
+      -L, --local          skip remote storage, only do local backups
+
+Note: CONFIG\_FILE will be created from template if missing
+
+Encryption
+----------
+
+If you want to encrypt your backups you have 2 options:
+* use simple password encryption
+* use GPG public key encryption
+
+> IMPORTANT: some gpg installations automatically set 'use-agent' option in the default
+> configuration file that is created when you run gpg for the first time. This will cause
+> gpg to fail on the 2nd run if you don't have the agent running. The result is that
+> 'astrails-safe' will work ONCE when you manually test it and then fail on any subsequent run.
+> The solution is to remove the 'use-agent' from the config file (usually /root/.gnupg/gpg.conf)
+> To mitigate this problem for the gpg 1.x series '--no-use-agent' option is added by defaults
+> to the autogenerated config file, but for gpg2 is doesn't work. as the manpage says it:
+> "This is dummy option. gpg2 always requires the agent." :(
+
+For simple password, just add password entry in gpg section.
+For public key encryption you will need to create a public/secret keypair.
+
+We recommend to create your GPG keys only on your local machine and then
+transfer your public key to the server that will do the backups.
+
+This way the server will only know how to encrypt the backups but only you
+will be able to decrypt them using the secret key you have locally. Of course
+you MUST backup your backup encryption key :)
+We recommend also pringing the hard paper copy of your GPG key 'just in case'.
+
+The procedure to create and transfer the key is as follows:
+
+1. run 'gpg --gen-key' on your local machine and follow onscreen instructions to create the key
+   (you can accept all the defaults).
+
+2. extract your public key into a file (assuming you used test@example.com as your key email):
+   `gpg -a --export test@example.com > test@example.com.pub`
+
+3. transfer public key to the server
+   `scp test@example.com.pub root@example.com:`
+
+4. import public key on the remote system:
+
+    $ gpg --import test@example.com.pub
+    gpg: key 45CA9403: public key "Test Backup <test@example.com>" imported
+    gpg: Total number processed: 1
+    gpg:               imported: 1
+
+5. since we don't keep the secret part of the key on the remote server, gpg has
+   no way to know its yours and can be trusted.
+   To fix that we can sign it with other trusted key, or just directly modify its
+   trust level in gpg (use level 5):
+
+     $ gpg --edit-key test@example.com
+     ...
+     Command> trust
+     ...
+     1 = I don't know or won't say
+     2 = I do NOT trust
+     3 = I trust marginally
+     4 = I trust fully
+     5 = I trust ultimately
+     m = back to the main menu
+
+     Your decision? 5
+     ...
+     Command> quit
+
+6. export your secret key for backup
+   (we recommend to print it on paper and burn to a CD/DVD and store in a safe place):
+
+    $ gpg -a --export-secret-key test@example.com > test@example.com.key
+
+
+
+Example configuration
+---------------------
+
+    safe do
+      local :path => "/backup/:kind/:id"
+
+      s3 do
+        key "...................."
+        secret "........................................"
+        bucket "backup.astrails.com"
+        path "servers/alpha/:kind/:id"
+      end
+
+      cloudfiles do
+        user "..........."
+        api_key "................................."
+        container "safe_backup"
+        path ":kind/" # this is default
+        service_net false
+      end
+
+      sftp do
+        host "sftp.astrails.com"
+        user "astrails"
+        # port 8023
+        password "ssh password for sftp"
+      end
+
+      gpg do
+        command "/usr/local/bin/gpg"
+        options  "--no-use-agent"
+        # symmetric encryption key
+        # password "qwe"
+
+        # public GPG key (must be known to GPG, i.e. be on the keyring)
+        key "backup@astrails.com"
+      end
+
+      keep do
+        local 20
+        s3 100
+        cloudfiles 100
+        sftp 100
+      end
+
+      mysqldump do
+        options "-ceKq --single-transaction --create-options"
+
+        user "root"
+        password "............"
+        socket "/var/run/mysqld/mysqld.sock"
+
+        database :blog
+        database :servershape
+        database :astrails_com
+        database :secret_project_com do
+          skip_tables "foo"
+          skip_tables ["bar", "baz"]
+        end
+
+      end
+
+      svndump do
+        repo :my_repo do
+          repo_path "/home/svn/my_repo"
+        end
+      end
+
+      pgdump do
+        options "-i -x -O"   # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
+
+        user "username"
+        password "............"  # shouldn't be used, instead setup ident.  Current functionality exports a password env to the shell which pg_dump uses - untested!
+
+        database :blog
+        database :stateofflux_com
+      end
+
+      tar do
+        options "-h" # dereference symlinks
+        archive "git-repositories", :files => "/home/git/repositories"
+        archive "dot-configs",      :files => "/home/*/.[^.]*"
+        archive "etc",              :files => "/etc", :exclude => "/etc/puppet/other"
+
+        archive "blog-astrails-com" do
+          files "/var/www/blog.astrails.com/"
+          exclude "/var/www/blog.astrails.com/log"
+          exclude "/var/www/blog.astrails.com/tmp"
+        end
+
+        archive "astrails-com" do
+          files "/var/www/astrails.com/"
+          exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
+        end
+      end
+    end
+
+Copyright
+---------
+
+Copyright (c) 2010 Astrails Ltd. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'rake'
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_opts = ['--options', "\"spec/spec.opts\""]
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "safe #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/TODO

@@ -0,0 +1,11 @@
+- add 'silent'
+- handle errors from mysqldump
+- check that gpg is installed
+- support percona XtraBackup as an option instead of mysqldump [patches anyone :) ?]
+- backup validation:
+  - support for 'minsize' opition in backup that will check that produced backup is at least the expected size
+    this should catch many backup failure scenarious (like broken mysql connection, insufficient disk space etc.
+- support differencial backups
+  - it should be fairly easy for filesystem backups using tar's built in incremental functionality.
+  - for mysql need to use XtraBackup
+  - or we can keep the previous dump locally and store only diff with the latest dump