akupchanko-astrails-safe 0.3.1

data/spec/astrails/safe/pgdump_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe Astrails::Safe::Pgdump do
+
+  def def_config
+    {
+      :options => "OPTS",
+      :user => "User",
+      :password => "pwd",
+      :host => "localhost",
+      :port => 7777,
+      :skip_tables => [:bar, :baz]
+    }
+  end
+
+  def pgdump(id = :foo, config = def_config)
+    Astrails::Safe::Pgdump.new(id, Astrails::Safe::Config::Node.new(nil, config))
+  end
+
+  before(:each) do
+    stub(Time).now.stub!.strftime {"NOW"}
+  end
+
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @pg = pgdump
+    end
+
+    {
+      :id => "foo",
+      :kind => "pgdump",
+      :extension => ".sql",
+      :filename => "pgdump-foo.NOW",
+      :command => "pg_dump OPTS --username='User' --host='localhost' --port='7777' foo",
+    }.each do |k, v|
+      it "should set #{k} to #{v}" do
+        @pg.backup.send(k).should == v
+      end
+    end
+
+  end
+
+end

data/spec/astrails/safe/s3_spec.rb

@@ -0,0 +1,168 @@
+require 'spec_helper'
+
+describe Astrails::Safe::S3 do
+
+  def def_config
+    {
+      :s3 => {
+        :bucket => "_bucket",
+        :key    => "_key",
+        :secret => "_secret",
+      },
+      :keep => {
+        :s3 => 2
+      }
+    }
+  end
+
+  def def_backup(extra = {})
+    {
+      :kind      => "_kind",
+      :filename  => "/backup/somewhere/_kind-_id.NOW.bar",
+      :extension => ".bar",
+      :id        => "_id",
+      :timestamp => "NOW"
+    }.merge(extra)
+  end
+
+  def s3(config = def_config, backup = def_backup)
+    Astrails::Safe::S3.new(
+      Astrails::Safe::Config::Node.new.merge(config),
+      Astrails::Safe::Backup.new(backup)
+    )
+  end
+
+  describe :cleanup do
+
+    before(:each) do
+      @s3 = s3
+
+      @files = [4,1,3,2].map { |i| stub(o = {}).key {"aaaaa#{i}"}; o }
+
+      stub(AWS::S3::Bucket).objects("_bucket", :prefix => "_kind/_id/_kind-_id.", :max_keys => 4) {@files}
+      stub(AWS::S3::Bucket).objects("_bucket", :prefix => anything).stub![0].stub!.delete
+    end
+
+    it "should check [:keep, :s3]" do
+      @s3.config[:keep].data["s3"] = nil
+      dont_allow(@s3.backup).filename
+      @s3.send :cleanup
+    end
+
+    it "should delete extra files" do
+      mock(AWS::S3::Bucket).objects("_bucket", :prefix => "aaaaa1").mock![0].mock!.delete
+      mock(AWS::S3::Bucket).objects("_bucket", :prefix => "aaaaa2").mock![0].mock!.delete
+      @s3.send :cleanup
+    end
+
+  end
+
+  describe :active do
+    before(:each) do
+      @s3 = s3
+    end
+
+    it "should be true when all params are set" do
+      @s3.should be_active
+    end
+
+    it "should be false if bucket is missing" do
+      @s3.config[:s3].data["bucket"] = nil
+      @s3.should_not be_active
+    end
+
+    it "should be false if key is missing" do
+      @s3.config[:s3].data["key"] = nil
+      @s3.should_not be_active
+    end
+
+    it "should be false if secret is missing" do
+      @s3.config[:s3].data["secret"] = nil
+      @s3.should_not be_active
+    end
+  end
+
+  describe :path do
+    before(:each) do
+      @s3 = s3
+    end
+    it "should use s3/path 1st" do
+      @s3.config[:s3].data["path"] = "s3_path"
+      @s3.config[:local] = {:path => "local_path"}
+      @s3.send(:path).should == "s3_path"
+    end
+
+    it "should use local/path 2nd" do
+      @s3.config.merge local: {path: "local_path"}
+      @s3.send(:path).should == "local_path"
+    end
+
+    it "should use constant 3rd" do
+      @s3.send(:path).should == "_kind/_id"
+    end
+
+  end
+
+  describe :save do
+    def add_stubs(*stubs)
+      stubs.each do |s|
+        case s
+        when :connection
+          stub(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true)
+        when :stat
+          stub(File).stat("foo").stub!.size {123}
+        when :create_bucket
+          stub(AWS::S3::Bucket).find('_bucket') { raise_error AWS::S3::NoSuchBucket }
+          stub(AWS::S3::Bucket).create
+        when :file_open
+          stub(File).open("foo") {|f, block| block.call(:opened_file)}
+        when :s3_store
+          stub(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket")
+        end
+      end
+    end
+
+    before(:each) do
+      @s3 = s3(def_config, def_backup(:path => "foo"))
+      @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar"
+    end
+
+    it "should fail if no backup.file is set" do
+      @s3.backup.path = nil
+      proc {@s3.send(:save)}.should raise_error(RuntimeError)
+    end
+
+    it "should establish s3 connection" do
+      mock(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true)
+      add_stubs(:stat, :create_bucket, :file_open, :s3_store)
+      @s3.send(:save)
+    end
+
+    it "should open local file" do
+      add_stubs(:connection, :stat, :create_bucket)
+      mock(File).open("foo")
+      @s3.send(:save)
+    end
+
+    it "should upload file" do
+      add_stubs(:connection, :stat, :create_bucket, :file_open)
+      mock(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket")
+      @s3.send(:save)
+    end
+
+    it "should fail on files bigger then 5G" do
+      add_stubs(:connection)
+      mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
+      mock(STDERR).puts(anything)
+      dont_allow(Benchmark).realtime
+      @s3.send(:save)
+    end
+
+    it 'should not create a bucket that already exists' do
+      add_stubs(:connection, :stat, :file_open, :s3_store)
+      stub(AWS::S3::Bucket).find('_bucket') { true }
+      dont_allow(AWS::S3::Bucket).create
+      @s3.send(:save)
+    end
+  end
+end

data/spec/astrails/safe/svndump_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe Astrails::Safe::Svndump do
+  def def_config
+    {
+      :options => "OPTS",
+      :repo_path => "bar/baz"
+    }
+  end
+
+  def svndump(id = :foo, config = def_config)
+    Astrails::Safe::Svndump.new(id, Astrails::Safe::Config::Node.new(nil, config))
+  end
+
+  before(:each) do
+    stub(Time).now.stub!.strftime {"NOW"}
+  end
+
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @svn = svndump
+    end
+
+    {
+      :id => "foo",
+      :kind => "svndump",
+      :extension => ".svn",
+      :filename => "svndump-foo.NOW",
+      :command => "svnadmin dump OPTS bar/baz",
+    }.each do |k, v|
+      it "should set #{k} to #{v}" do
+        @svn.backup.send(k).should == v
+      end
+    end
+
+  end
+end

data/spec/integration/archive_integration_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "tmp/archive_backup_example"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p "#{@src}/q/w/e"
+    mkdir_p "#{@src}/a/s/d"
+
+    File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") }
+    File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) }
+    File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) }
+    File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) }
+
+    File.open("#{@src}/asd1", "w") {|f| f.write("asd") }
+    File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) }
+    File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p @dst
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    config = Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        archive :test1 do
+          files src
+          exclude "#{src}/q/w"
+          exclude "#{src}/q/w/e"
+        end
+      end
+    end
+    Astrails::Safe.process config
+
+    @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar.gz"
+  end
+
+  it "should create backup file" do
+    File.exists?(@backup).should be_true
+  end
+
+  describe "after extracting" do
+    before(:all) do
+      # prepare target dir
+      @target = "#{@root}/test"
+      mkdir_p @target
+      system "tar -zxvf #{@backup} -C #{@target}"
+
+      @test = "#{@target}/#{@root}/src"
+      puts @test
+    end
+
+    it "should include asd1/2/3" do
+      File.exists?("#{@test}/asd1").should be_true
+      File.exists?("#{@test}/a/asd2").should be_true
+      File.exists?("#{@test}/a/s/asd3").should be_true
+    end
+
+    it "should only include qwe 1 and 2 (no 3)" do
+      File.exists?("#{@test}/qwe1").should be_true
+      File.exists?("#{@test}/q/qwe2").should be_true
+      File.exists?("#{@test}/q/w/qwe3").should be_false
+      File.exists?("#{@test}/q/w/e/qwe4").should be_false
+    end
+
+    it "should preserve file content" do
+      File.read("#{@test}/qwe1").should == "qwe"
+      File.read("#{@test}/q/qwe2").should == "qweqwe"
+      File.read("#{@test}/a/s/asd3").should == "asdasdasd"
+    end
+  end
+
+end

data/spec/integration/cleanup_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "tmp/cleanup_example"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p src
+
+    File.open(qwe = "#{@src}/qwe", "w") {|f| f.write("qwe") }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p "#{@dst}/archive"
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    cp qwe, "#{dst}/archive/archive-foo.000001.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foo.000002.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foobar.000001.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foobar.000002.tar.gz"
+
+    config = Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        keep :local => 1 # only leave the latest
+        archive :foo do
+          files src
+        end
+      end
+    end
+    Astrails::Safe.process config
+
+    @backup = "#{dst}/archive/archive-foo.#{@timestamp}.tar.gz"
+  end
+
+  it "should create backup file" do
+    File.exists?(@backup).should be_true
+  end
+
+  it "should remove old backups" do
+    Dir["#{@dst}/archive/archive-foo.*"].should == [@backup]
+  end
+
+  it "should NOT remove backups with base having same prefix" do
+    Dir["#{@dst}/archive/archive-foobar.*"].sort.should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'astrails/safe'
+require 'debugger'
+
+RSpec.configure do |config|
+  config.mock_with :rr
+end

data/templates/script.rb

@@ -0,0 +1,183 @@
+safe do
+  # same as --verbose on the command line
+  # verbose true
+
+  # same as --local on the command line
+  # local_only true
+
+  # same as --dry-run on the command line
+  # dry_run true
+
+  # backup file path (not including filename)
+  # supported substitutions:
+  #   :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
+  #   :id -> backup 'id', e.g. "blog", "production", etc.
+  #   :timestamp -> current run timestamp (same for all the backups in the same 'run')
+  # you can set separate :path for all backups (or once globally here)
+  local do
+    path "/backup/:kind"
+  end
+
+  ## uncomment to enable uploads to Amazon S3
+  ## Amazon S3 auth (optional)
+  # s3 do
+  #   key YOUR_S3_KEY
+  #   secret YOUR_S3_SECRET
+  #   bucket S3_BUCKET
+  #   # path for uploads to S3. supports same substitution like :local/:path
+  #   path ":kind/" # this is default
+  # end
+
+  ## alternative style:
+  # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET, :path => ":kind/"
+
+  ## uncomment to enable uploads to Rackspace Cloud Files
+  ## http://www.rackspacecloud.com/cloud_hosting_products/files
+  ## Rackspace auth (optional)
+  # cloudfiles do
+  #   user "YOUR_RACKSPACE_CLOUD_USERNAME"
+  #   api_key "YOUR_RACKSPACE_API_KEY"
+  #   container "YOUR_CONTAINER_NAME"
+  #   # path for uploads to Cloud Files, supports same substitution like :local/:path
+  #   path ":kind/" # this is default
+  #   # If you are running the backup from a system within the Rackspace/Slicehost network and would like
+  #   # to back up over the private (unbilled) service net, set this value to true.
+  #   # service_net true
+  # end
+
+  ## uncomment to enable uploads via SFTP
+  # sftp do
+  #   host "YOUR_REMOTE_HOSTNAME"
+  #   user "YOUR_REMOTE_USERNAME"
+  #   # port "NON STANDARD SSH PORT"
+  #   password "YOUR_REMOTE_PASSWORD"
+  #   path ":kind/:id" # this is the default
+  # end
+
+  ## uncomment to enable uploads via FTP
+  # ftp do
+  #   host "YOUR_REMOTE_HOSTNAME"
+  #   user "YOUR_REMOTE_USERNAME"
+  #   # port "NON STANDARD FTP PORT"
+  #   password "YOUR_REMOTE_PASSWORD"
+  #   path ":kind/:id" # this is the default
+  # end
+
+  ## uncomment to enable GPG encryption.
+  ## Note: you can use public 'key' or symmetric password but not both!
+  # gpg do
+  #   # you can specify your own gpg executable with the 'command' options
+  #   # this can be useful for example to choose b/w gpg and gpg2 if both are installed
+  #   # some gpg installations will automatically set 'use-agent' option in the
+  #   # config file on the 1st run. see README for more details
+  #   options "--no-use-agent"
+  #   # command "/usr/local/bin/gpg"
+  #   # key "backup@astrails.com"
+  #   password "astrails"
+  # end
+
+  ## uncomment to enable backup rotation. keep only given number of latest
+  ## backups. remove the rest
+  # keep do
+  #   local 4 # keep 4 local backups
+  #   s3 20 # keep 20 S3 backups
+  # end
+
+  # backup mysql databases with mysqldump
+  mysqldump do
+    # you can override any setting from parent in a child:
+    options "-ceKq --single-transaction --create-options"
+
+    user "astrails"
+    password ""
+    # host "localhost"
+    # port 3306
+    socket "/var/run/mysqld/mysqld.sock"
+
+    # database is a 'collection' element. it must have a hash or block parameter
+    # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+    # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql configuration 'nodes'
+
+    # backup database with default values
+    # database :blog
+
+    # backup overriding some values
+    # database :production do
+    #   # you can override 'partially'
+    #   keep :local => 3
+    #   # keep/local is 3, and keep/s3 is 20 (from parent)
+
+    #   # local override for gpg password
+    #   gpg do
+    #     password "custom-production-pass"
+    #   end
+    #   # skip those tables during backup
+    #   # you can pass an array
+    #   skip_tables [:logger_exceptions, :request_logs]
+    #   # or pass them all separately
+    #   skip_tables :test1
+    #   skip_tables :test2
+    # end
+
+  end
+
+  # # uncomment to enable
+  # # backup PostgreSQL databases with pg_dump
+  # pgdump do
+  #   options "-i -x -O"
+  #
+  #   user "markmansour"
+  #   # password "" - leave this out if you have ident setup
+  #
+  #   # database is a 'collection' element. it must have a hash or block parameter
+  #   # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+  #   database :blog
+  #   database :production
+  # end
+
+  tar do
+    # options "-h" # uncomment this to dereference symbolic links
+
+    # 'archive' is a collection item, just like 'database'
+    # archive "git-repositories" do
+    #   # files and directories to backup
+    #   files "/home/git/repositories"
+    #   # can have more then one 'files' lines or/and use an array
+    #   files ["/home/dev/work/foo", "/home/dev/work/bar"]
+    # end
+
+    # archive "etc-files" do
+    #   files "/etc"
+    #   # exlude those files/directories
+    #   exclude "/etc/puppet/other"
+    #   # can have multiple 'exclude' lines or/and use an array
+    #   exclude ["/etc/tmp/a", "/etc/tmp/b"]
+    # end
+
+    # archive "dot-configs" do
+    #   files "/home/*/.[^.]*"
+    # end
+
+    # archive "blog" do
+    #   files "/var/www/blog.astrails.com/"
+    #   # specify multiple files/directories as array
+    #   exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
+    # end
+
+    # archive "site" do
+    #   files "/var/www/astrails.com/"
+    #   exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
+    # end
+
+    # archive :misc do
+    #   files [ "/backup/*.rb" ]
+    # end
+  end
+
+  # svndump do
+  #   repo :my_repo do
+  #     repo_path "/home/svn/my_repo"
+  #   end
+  # end
+
+end