astrails-safe 0.2.6 → 0.2.7

data/README.markdown

@@ -1,7 +1,7 @@
 astrails-safe
 =============
 
-Simple database and filesystem backups with S3 support (with optional encryption)
+Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption)
 
 Home: http://blog.astrails.com/astrails-safe
 
@@ -15,8 +15,8 @@ We needed a backup solution that will satisfy the following requirements:
 * support for simple ‘tar’ backups of directories (with includes/excludes)
 * support for simple mysqldump of mysql databases
 * support for symmetric or public key encryption
-* support for local filesystem and Amazon S3 for storage
-* support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3
+* support for local filesystem, Amazon S3, and Rackspace Cloud Files for storage
+* support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3 or Cloud Files
 
 And since we didn't find any, we wrote our own :)
 
@@ -29,6 +29,10 @@ The following functionality was contributed by astrails-safe users:
 * Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
 * SFTP remote storage (by Adam <adam@mediadrive.ca>)
 * benchmarking output (By Neer)
+* README fixes (by Bobby Wilson)
+* improved config file parsing (by Fedor Kocherga <fkocherga@gmail.com>)
+* mysql password file quoting (by Jonathan Sutherland <jonathan.sutherland@gmail.com>)
+* Rackspace Cloud Files support (by H. Wade Minter <minter@lunenburg.org>)
 
 Thanks to all :)
 
@@ -51,7 +55,7 @@ Usage
       -h, --help           This help screen
       -v, --verbose        be verbose, duh!
       -n, --dry-run        just pretend, don't do anything.
-      -L, --local          skip S3
+      -L, --local          skip remote storage, only do local backups
 
 Note: CONFIG_FILE will be created from template if missing
 
@@ -62,6 +66,15 @@ If you want to encrypt your backups you have 2 options:
 * use simple password encryption
 * use GPG public key encryption
 
+> IMPORTANT: some gpg installations automatically set 'use-agent' option in the default
+> configuration file that is created when you run gpg for the first time. This will cause
+> gpg to fail on the 2nd run if you don't have the agent running. The result is that
+> 'astrails-safe' will work ONCE when you manually test it and then fail on any subsequent run.
+> The solution is to remove the 'use-agent' from the config file (usually /root/.gnupg/gpg.conf)
+> To mitigate this problem for the gpg 1.x series '--no-use-agent' option is added by defaults
+> to the autogenerated config file, but for gpg2 is doesn't work. as the manpage says it:
+> "This is dummy option. gpg2 always requires the agent." :(
+
 For simple password, just add password entry in gpg section.
 For public key encryption you will need to create a public/secret keypair.
 
@@ -133,6 +146,14 @@ Example configuration
       path "servers/alpha/:kind/:id"
     end
 
+    cloudfiles do
+      username "..........."
+      api_key "................................."
+      container "safe_backup"
+      path ":kind/" # this is default
+      service_net false
+    end
+
     sftp do
       host "sftp.astrails.com"
       user "astrails"
@@ -141,6 +162,8 @@ Example configuration
     end
 
     gpg do
+      command "/usr/local/bin/gpg"
+      options  "--no-use-agent"
       # symmetric encryption key
       # password "qwe"
 
@@ -150,7 +173,9 @@ Example configuration
 
     keep do
       local 20
-      s3 30
+      s3 100
+      cloudfiles 100
+      sftp 100
     end
 
     mysqldump do

data/Rakefile

@@ -5,14 +5,19 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "astrails-safe"
-    gem.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) to Amazon S3 (with encryption)}
-    gem.description = "Simple tool to backup databases (MySQL and PostgreSQL) and filesystem locally or to Amazon S3 (with optional encryption)"
+    gem.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
+    gem.description = <<-DESC
+Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+Backups can be stored locally or remotely and can be enctypted.
+Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
+DESC
     gem.email = "we@astrails.com"
     gem.homepage = "http://blog.astrails.com/astrails-safe"
-    gem.authors  = ["Astrails Ltd.", "Mark Mansour"]
+    gem.authors  = ["Astrails Ltd."]
     gem.files = FileList["[A-Z]*.*", "{bin,examples,generators,lib,rails,spec,test,templates}/**/*", 'Rakefile', 'LICENSE*']
 
     gem.add_dependency("aws-s3")
+    gem.add_dependency("cloudfiles")
     gem.add_dependency("net-sftp")
 
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :major: 0
 :minor: 2
-:patch: 6
+:patch: 7

data/bin/astrails-safe

@@ -2,8 +2,8 @@
 
 require 'rubygems'
 
-#require 'ruby-debug'
-#$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+# require 'ruby-debug'
+$:.unshift File.expand_path("../../lib", __FILE__)
 
 require 'astrails/safe'
 include Astrails::Safe
@@ -20,7 +20,7 @@ Options:
   -h, --help           This help screen
   -v, --verbose        be verbose, duh!
   -n, --dry-run        just pretend, don't do anything.
-  -L, --local          skip S3
+  -L, --local          skip S3 and Cloud Files
 
 Note: config file will be created from template if missing
 END

data/examples/unit/gpg_example.rb

@@ -102,28 +102,38 @@ describe Astrails::Safe::Gpg do
   describe :pipe do
 
     describe "with key" do
-      before(:each) do
-        @gpg = gpg(:gpg => {:key => "foo", :options => "GPG-OPT"}, :options => "OPT")
+      def kgpg(extra={})
+        gpg({:gpg => {:key => "foo", :options => "GPG-OPT"}.merge(extra), :options => "OPT"})
       end
 
       it "should not call gpg_password_file" do
-        dont_allow(@gpg).gpg_password_file(anything)
-        @gpg.send(:pipe)
+        g = kgpg
+        dont_allow(g).gpg_password_file(anything)
+        g.send(:pipe)
       end
 
       it "should use '-r' and :options" do
-        @gpg.send(:pipe).should == "|gpg GPG-OPT -e -r foo"
+        kgpg.send(:pipe).should == "|gpg GPG-OPT -e -r foo"
+      end
+
+      it "should use the 'command' options" do
+        kgpg(:command => 'other-gpg').send(:pipe).should == "|other-gpg GPG-OPT -e -r foo"
       end
     end
 
     describe "with password" do
-      before(:each) do
-        @gpg = gpg(:gpg => {:password => "bar", :options => "GPG-OPT"}, :options => "OPT")
-        stub(@gpg).gpg_password_file(anything) {"pass-file"}
+      def pgpg(extra = {})
+        returning(gpg({:gpg => {:password => "bar", :options => "GPG-OPT"}.merge(extra), :options => "OPT"})) do |g|
+          stub(g).gpg_password_file(anything) {"pass-file"}
+        end
       end
 
       it "should use '--passphrase-file' and :options" do
-        @gpg.send(:pipe).should == "|gpg GPG-OPT -c --passphrase-file pass-file"
+        pgpg.send(:pipe).should == "|gpg GPG-OPT -c --passphrase-file pass-file"
+      end
+
+      it "should use the 'command' options" do
+        pgpg(:command => 'other-gpg').send(:pipe).should == "|other-gpg GPG-OPT -c --passphrase-file pass-file"
       end
     end
   end

data/examples/unit/mysqldump_example.rb

@@ -2,7 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
 
 describe Astrails::Safe::Mysqldump do
 
-  def def_config
+  def def_config(extra = {})
     {
       :options => "OPTS",
       :user => "User",
@@ -11,7 +11,7 @@ describe Astrails::Safe::Mysqldump do
       :port => 7777,
       :socket => "socket",
       :skip_tables => [:bar, :baz]
-    }
+    }.merge(extra)
   end
 
   def mysqldump(id = :foo, config = def_config)
@@ -66,16 +66,16 @@ describe Astrails::Safe::Mysqldump do
   end
 
   describe :mysql_password_file do
-    it "should create passwords file" do
-      m = mysqldump
+    it "should create passwords file with quoted values" do
+      m = mysqldump(:foo, def_config(:password => '#qwe"asd\'zxc'))
       file = m.send(:mysql_password_file)
       File.exists?(file).should == true
       File.read(file).should == <<-PWD
 [mysqldump]
-user = User
-password = pwd
-socket = socket
-host = localhost
+user = "User"
+password = "#qwe\\"asd'zxc"
+socket = "socket"
+host = "localhost"
 port = 7777
       PWD
     end

data/lib/astrails/safe.rb

@@ -1,4 +1,5 @@
 require "aws/s3"
+require "cloudfiles"
 require 'net/sftp'
 require 'fileutils'
 require 'benchmark'
@@ -30,6 +31,7 @@ require 'astrails/safe/gzip'
 require 'astrails/safe/sink'
 require 'astrails/safe/local'
 require 'astrails/safe/s3'
+require 'astrails/safe/cloudfiles'
 require 'astrails/safe/sftp'
 
 module Astrails
@@ -48,7 +50,7 @@ module Astrails
       ].each do |klass, path|
         if collection = config[*path]
           collection.each do |name, config|
-            klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :sftp)
+            klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
           end
         end
       end

data/lib/astrails/safe/cloudfiles.rb

@@ -0,0 +1,64 @@
+module Astrails
+  module Safe
+    class Cloudfiles < Sink
+
+      protected
+
+      def active?
+        container && user && api_key
+      end
+
+      def path
+        @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        # needed in cleanup even on dry run
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
+        puts "Uploading #{container}:#{full_path} from #{@backup.path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          benchmark = Benchmark.realtime do
+            cf_container = cf.create_container(container)
+            o = cf_container.create_object(full_path,true)
+            o.write(open(@backup.path))
+          end
+          puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :cloudfiles]
+
+        puts "listing files: #{container}:#{base}*" if $_VERBOSE
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
+        files = cf.container(container).objects(:prefix => base)
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing Cloud File #{container}:#{f}" if $DRY_RUN || $_VERBOSE
+          cf.container(container).delete_object(f) unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def container
+        @config[:cloudfiles, :container]
+      end
+
+      def user
+        @config[:cloudfiles, :user]
+      end
+
+      def api_key
+        @config[:cloudfiles, :api_key]
+      end
+
+      def service_net
+        @config[:cloudfiles, :service_net] || false
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -3,7 +3,7 @@ module Astrails
     module Config
       class Builder
         COLLECTIONS = %w/database archive repo/
-        ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump pgdump options
+        ITEMS = %w/s3 cloudfiles key secret bucket api_key container service_net path gpg password keep local mysqldump pgdump command options
         user host port socket skip_tables tar files exclude filename svndump repo_path sftp/
         NAMES = COLLECTIONS + ITEMS
         def initialize(node)

data/lib/astrails/safe/gpg.rb

@@ -9,10 +9,11 @@ module Astrails
       end
 
       def pipe
+        command = @config[:gpg, :command] || 'gpg'
         if key
-          "|gpg #{@config[:gpg, :options]} -e -r #{key}"
+          "|#{command} #{@config[:gpg, :options]} -e -r #{key}"
         elsif password
-          "|gpg #{@config[:gpg,:options]} -c --passphrase-file #{gpg_password_file(password)}"
+          "|#{command} #{@config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}"
         end
       end
 

data/lib/astrails/safe/mysqldump.rb

@@ -15,7 +15,8 @@ module Astrails
           file.puts "[mysqldump]"
           %w/user password socket host port/.each do |k|
             v = @config[k]
-            file.puts "#{k} = #{v}" if v
+            # values are quoted if needed
+            file.puts "#{k} = #{v.inspect}" if v
           end
         end
       end

data/lib/astrails/safe/s3.rb

@@ -41,7 +41,6 @@ module Astrails
 
         return unless keep = @config[:keep, :s3]
 
-
         puts "listing files: #{bucket}:#{base}*" if $_VERBOSE
         files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
         puts files.collect {|x| x.key} if $_VERBOSE

data/templates/script.rb

@@ -12,7 +12,6 @@ safe do
 
   ## uncomment to enable uploads to Amazon S3
   ## Amazon S3 auth (optional)
-  ## don't forget to add :s3 to the 'store' list
   # s3 do
   #   key YOUR_S3_KEY
   #   secret YOUR_S3_SECRET
@@ -24,6 +23,20 @@ safe do
   ## alternative style:
   # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET, :path => ":kind/"
 
+  ## uncomment to enable uploads to Rackspace Cloud Files
+  ## http://www.rackspacecloud.com/cloud_hosting_products/files
+  ## Rackspace auth (optional)
+  # cloudfiles do
+  #   user "YOUR_RACKSPACE_CLOUD_USERNAME"
+  #   api_key "YOUR_RACKSPACE_API_KEY"
+  #   container "YOUR_CONTAINER_NAME"
+  #   # path for uploads to Cloud Files, supports same substitution like :local/:path
+  #   path ":kind/" # this is default
+  #   # If you are running the backup from a system within the Rackspace/Slicehost network and would like
+  #   # to back up over the private (unbilled) service net, set this value to true.
+  #   # service_net true
+  # end
+
   ## uncomment to enable uploads via SFTP
   # sftp do
   #   host "YOUR_REMOTE_HOSTNAME"
@@ -36,6 +49,12 @@ safe do
   ## uncomment to enable GPG encryption.
   ## Note: you can use public 'key' or symmetric password but not both!
   # gpg do
+  #   # you can specify your own gpg executable with the 'command' options
+  #   # this can be useful for example to choose b/w gpg and gpg2 if both are installed
+  #   # some gpg installations will automatically set 'use-agent' option in the
+  #   # config file on the 1st run. see README for more details
+  #   options "--no-use-agent"
+  #   # command "/usr/local/bin/gpg"
   #   # key "backup@astrails.com"
   #   password "astrails"
   # end

metadata

@@ -1,16 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: astrails-safe
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors: 
 - Astrails Ltd.
-- Mark Mansour
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2009-12-21 00:00:00 +02:00
+date: 2010-01-21 00:00:00 +02:00
 default_executable: astrails-safe
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -23,6 +22,16 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
+- !ruby/object:Gem::Dependency 
+  name: cloudfiles
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 - !ruby/object:Gem::Dependency 
   name: net-sftp
   type: :runtime
@@ -33,7 +42,11 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
-description: Simple tool to backup databases (MySQL and PostgreSQL) and filesystem locally or to Amazon S3 (with optional encryption)
+description: |
+  Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+  Backups can be stored locally or remotely and can be enctypted.
+  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
+
 email: we@astrails.com
 executables: 
 - astrails-safe
@@ -63,6 +76,7 @@ files:
 - lib/astrails/safe.rb
 - lib/astrails/safe/archive.rb
 - lib/astrails/safe/backup.rb
+- lib/astrails/safe/cloudfiles.rb
 - lib/astrails/safe/config/builder.rb
 - lib/astrails/safe/config/node.rb
 - lib/astrails/safe/gpg.rb
@@ -107,7 +121,7 @@ rubyforge_project:
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
-summary: Backup filesystem and databases (MySQL and PostgreSQL) to Amazon S3 (with encryption)
+summary: Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)
 test_files: 
 - examples/example_helper.rb
 - examples/integration/archive_integration_example.rb