astrails-safe 0.1.6 → 0.1.7

data/README.markdown

@@ -1,7 +1,9 @@
 astrails-safe
 =============
 
-Simple mysql and filesystem backups with S3 support (with optional encryption)
+Simple database and filesystem backups with S3 support (with optional encryption)
+
+Home: github.com/astrails/safe
 
 Motivation
 ----------
@@ -18,6 +20,13 @@ We needed a backup solution that will satisfy the following requirements:
 
 And since we didn't find any, we wrote our own :)
 
+Note
+----
+
+Support for pg_dump and svndump was contributed but since I don't personally use them i don't have an easy
+way of testing it. So if you use Subversion or PostgreSQL please test the new functionality and report if
+there are any problems.
+
 Usage
 -----
 
@@ -62,7 +71,7 @@ The procedure to create and transfer the key is as follows:
 
 4. import public key on the remote system:
 <pre>
-   $ gpg --import test@example.com.pub 
+   $ gpg --import test@example.com.pub
    gpg: key 45CA9403: public key "Test Backup <test@example.com>" imported
    gpg: Total number processed: 1
    gpg:               imported: 1
@@ -83,7 +92,7 @@ The procedure to create and transfer the key is as follows:
      4 = I trust fully
      5 = I trust ultimately
      m = back to the main menu
-     
+
      Your decision? 5
      ...
      Command> quit
@@ -136,6 +145,22 @@ Example configuration
 
     end
 
+    svndump do
+      repo :my_repo do
+        repo_path "/home/svn/my_repo"
+      end
+    end
+
+    pgdump do
+      options "-i -x -O"   # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format
+
+      user "username"
+      password "............"  # shouldn't be used, instead setup ident.  Current functionality exports a password env to the shell which pg_dump uses - untested!
+
+      database :blog
+      database :stateofflux_com
+    end
+
     tar do
       archive "git-repositories", :files => "/home/git/repositories"
       archive "dot-configs",      :files => "/home/*/.[^.]*"
@@ -154,6 +179,11 @@ Example configuration
   end
 </pre>
 
+Reporting problems
+------------------
+
+http://github.com/astrails/safe/issues
+
 Copyright
 ---------
 

data/Rakefile

@@ -5,11 +5,11 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "safe"
-    gem.summary = %Q{Backup filesystem and MySQL to Amazon S3 (with encryption)}
-    gem.description = "Simple tool to backup MySQL databases and filesystem locally or to Amazon S3 (with optional encryption)"
+    gem.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) to Amazon S3 (with encryption)}
+    gem.description = "Simple tool to backup databases (MySQL and PostgreSQL) and filesystem locally or to Amazon S3 (with optional encryption)"
     gem.email = "we@astrails.com"
     gem.homepage = "http://github.com/astrails/safe"
-    gem.authors  = ["Astrails Ltd."]
+    gem.authors  = ["Astrails Ltd.", "Mark Mansour"]
     gem.files = FileList["[A-Z]*.*", "{bin,examples,generators,lib,rails,spec,test,templates}/**/*", 'Rakefile', 'LICENSE*']
 
     gem.add_dependency("aws-s3")

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :major: 0
 :minor: 1
-:patch: 6
+:patch: 7

data/bin/astrails-safe

@@ -1,11 +1,6 @@
 #!/usr/bin/env ruby
 
-
-require 'tempfile'
 require 'rubygems'
-require 'fileutils'
-require "aws/s3"
-require 'yaml'
 
 #require 'ruby-debug'
 #$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
@@ -52,8 +47,7 @@ def main
     die "Created default #{$CONFIG_FILE_NAME}. Please edit and run again."
   end
 
-
   load($CONFIG_FILE_NAME)
 end
 
-main
+main

data/examples/example_helper.rb

@@ -2,8 +2,9 @@ require 'rubygems'
 require 'micronaut'
 require 'ruby-debug'
 
+SAFE_ROOT = File.dirname(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.join(SAFE_ROOT, 'lib'))
 
 require 'astrails/safe'
 
@@ -15,5 +16,4 @@ Micronaut.configure do |c|
   c.color_enabled = not_in_editor?
   c.filter_run :focused => true
   c.mock_with :rr
-end
-
+end

data/examples/integration/archive_integration_example.rb

@@ -0,0 +1,86 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "tmp/archive_backup_example"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p "#{@src}/q/w/e"
+    mkdir_p "#{@src}/a/s/d"
+
+    File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
+    File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
+    File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
+    File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
+
+    File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
+    File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
+    File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        archive :test1 do
+          files src
+          exclude "#{src}/q/w"
+        end
+      end
+    end
+
+    @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar.gz"
+  end
+
+  it "should create backup file" do
+    puts "Expecting: #{@backup}"
+    File.exists?(@backup).should be_true
+  end
+
+  describe "after extracting" do
+    before(:all) do
+      # prepare target dir
+      @target = "#{@root}/test"
+      mkdir_p @target
+      system "tar -zxvf #{@backup} -C #{@target}"
+
+      @test = "#{@target}/#{@root}/src"
+      puts @test
+    end
+
+    it "should include asd1/2/3" do
+      File.exists?("#{@test}/asd1").should be_true
+      File.exists?("#{@test}/a/asd2").should be_true
+      File.exists?("#{@test}/a/s/asd3").should be_true
+    end
+
+    it "should only include qwe 1 and 2 (no 3)" do
+      File.exists?("#{@test}/qwe1").should be_true
+      File.exists?("#{@test}/q/qwe2").should be_true
+    end
+
+    it "should preserve file content" do
+      File.read("#{@test}/qwe1").should == "qwe"
+      File.read("#{@test}/q/qwe2").should == "qweqwe"
+      File.read("#{@test}/a/s/asd3").should == "asdasdasd"
+    end
+  end
+
+end

data/examples/unit/archive_example.rb

@@ -0,0 +1,67 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+describe Astrails::Safe::Archive do
+
+  def def_config
+    {
+      :options => "OPTS",
+      :files   => "apples",
+      :exclude => "oranges"
+    }
+  end
+
+  def archive(id = :foo, config = def_config)
+    Astrails::Safe::Archive.new(id, Astrails::Safe::Config::Node.new(nil, config))
+  end
+
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @archive = archive
+      stub(@archive).timestamp {"NOW"}
+    end
+
+    {
+      :id => "foo",
+      :kind => "archive",
+      :extension => ".tar",
+      :filename => "archive-foo.NOW",
+      :command => "tar -cf - OPTS --exclude=oranges apples",
+    }.each do |k, v|
+      it "should set #{k} to #{v}" do
+        @archive.backup.send(k).should == v
+      end
+    end
+  end
+
+  describe :tar_exclude_files do
+    it "should return '' when no excludes" do
+      archive(:foo, {}).send(:tar_exclude_files).should == ''
+    end
+
+    it "should accept single exclude as string" do
+      archive(:foo, {:exclude => "bar"}).send(:tar_exclude_files).should == '--exclude=bar'
+    end
+
+    it "should accept multiple exclude as array" do
+      archive(:foo, {:exclude => ["foo", "bar"]}).send(:tar_exclude_files).should == '--exclude=foo --exclude=bar'
+    end
+  end
+
+  describe :tar_files do
+    it "should raise RuntimeError when no files" do
+      lambda {
+        archive(:foo, {}).send(:tar_files)
+      }.should raise_error(RuntimeError, "missing files for tar")
+    end
+
+    it "should accept single file as string" do
+      archive(:foo, {:files => "foo"}).send(:tar_files).should == "foo"
+    end
+
+    it "should accept multiple files as array" do
+      archive(:foo, {:files => ["foo", "bar"]}).send(:tar_files).should == "foo bar"
+    end
+  end
+end

data/examples/unit/config_example.rb

@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
 
 describe Astrails::Safe::Config do
-  it "foo" do
+  it "should parse example config" do
     config = Astrails::Safe::Config::Node.new do
       local do
         path "path"
@@ -47,6 +47,29 @@ describe Astrails::Safe::Config do
 
       end
 
+      pgdump do
+        options "-i -x -O"
+
+        user "astrails"
+        password ""
+        host "localhost"
+        port 5432
+
+        database :blog
+
+        database :production do
+          keep :local => 3
+
+          skip_tables [:logger_exceptions, :request_logs]
+        end
+
+      end
+
+      svndump do
+        repo :my_repo do
+          repo_path "/home/svn/my_repo"
+        end
+      end
 
       tar do
         archive "git-repositories" do
@@ -105,6 +128,31 @@ describe Astrails::Safe::Config do
           },
         },
       },
+
+      "pgdump" => {
+        "options" => "-i -x -O",
+        "user" => "astrails",
+        "password" => "",
+        "host" => "localhost",
+        "port" => 5432,
+
+        "databases" => {
+          "blog" => {},
+          "production" => {
+           "keep" => {"local" => 3},
+            "skip_tables" => ["logger_exceptions", "request_logs"],
+          },
+        },
+      },
+
+      "svndump" => {
+        "repos" => {
+          "my_repo"=> {
+            "repo_path" => "/home/svn/my_repo"
+          }
+        }
+      },
+
       "tar" => {
         "archives" => {
           "git-repositories" => {"files" => "/home/git/repositories"},
@@ -120,7 +168,5 @@ describe Astrails::Safe::Config do
     }
 
     config.to_hash.should == expected
-
   end
 end
-

data/examples/unit/gpg_example.rb

@@ -0,0 +1,138 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+describe Astrails::Safe::Gpg do
+  def def_backup
+    {
+      :compressed => false,
+      :command => "command",
+      :extension => ".foo",
+      :filename => "qweqwe"
+    }
+  end
+
+  def gpg(config = {}, backup = def_backup)
+    Astrails::Safe::Gpg.new(
+      @config = Astrails::Safe::Config::Node.new(nil, config),
+      @backup = Astrails::Safe::Backup.new(backup)
+    )
+  end
+
+  before(:each) do
+    @gpg = gpg()
+    stub(@gpg).gpg_password_file {"pwd-file"}
+    stub(@gpg).pipe {"|gpg -BLAH"}
+  end
+
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+  describe :process do
+
+    describe "when active" do
+      before(:each) do
+        stub(@gpg).active? {true}
+      end
+
+      it "should add .gpg extension" do
+        mock(@backup.extension) << '.gpg'
+        @gpg.process
+      end
+
+      it "should add command pipe" do
+        mock(@backup.command) << (/\|gpg -/)
+        @gpg.process
+      end
+
+      it "should set compressed" do
+        mock(@backup).compressed = true
+        @gpg.process
+      end
+    end
+
+    describe "when inactive" do
+      before(:each) do
+        stub(@gpg).active? {false}
+      end
+
+      it "should not touch extension" do
+        dont_allow(@backup.extension) << anything
+        @gpg.process
+      end
+
+      it "should not touch command" do
+        dont_allow(@backup.command) << anything
+        @gpg.process
+      end
+
+      it "should not touch compressed" do
+        dont_allow(@backup).compressed = anything
+        @gpg.process
+      end
+    end
+  end
+
+  describe :active? do
+
+    describe "with key" do
+      it "should be true" do
+        gpg(:gpg => {:key => :foo}).should be_active
+      end
+    end
+
+    describe "with password" do
+      it "should be true" do
+        gpg(:gpg => {:password => :foo}).should be_active
+      end
+    end
+
+    describe "without key & password" do
+      it "should be false" do
+        gpg.should_not be_active
+      end
+    end
+
+    describe "with key & password" do
+      it "should raise RuntimeError" do
+        lambda {
+          gpg(:gpg => {:key => "foo", :password => "bar"}).send :active?
+        }.should raise_error(RuntimeError, "can't use both gpg password and pubkey")
+      end
+    end
+  end
+
+  describe :pipe do
+
+    describe "with key" do
+      before(:each) do
+        @gpg = gpg(:gpg => {:key => "foo"}, :options => "OPT")
+      end
+
+      it "should not call gpg_password_file" do
+        dont_allow(@gpg).gpg_password_file(anything)
+        @gpg.send(:pipe)
+      end
+
+      it "should use '-r' and options" do
+        @gpg.send(:pipe).should == "|gpg OPT -e -r foo"
+      end
+    end
+
+    describe "with password" do
+      before(:each) do
+        @gpg = gpg(:gpg => {:password => "bar"}, :options => "OPT")
+        stub(@gpg).gpg_password_file(anything) {"pass-file"}
+      end
+
+      it "should use '--passphrase-file' and :options" do
+        @gpg.send(:pipe).should == "|gpg OPT -c --passphrase-file pass-file"
+      end
+    end
+  end
+
+  describe :gpg_password_file do
+    it "should create password file" do
+      file = gpg.send(:gpg_password_file, "foo")
+      File.exists?(file).should be_true
+      File.read(file).should == "foo"
+    end
+  end
+end