akupchanko-astrails-safe 0.3.1

data/lib/astrails/safe/stream.rb

@@ -0,0 +1,32 @@
+module Astrails
+  module Safe
+    class Stream
+
+      attr_accessor :config, :backup
+      def initialize(config, backup)
+        @config, @backup = config, backup
+      end
+      # FIXME: move to Backup
+      def expand(path)
+        path .
+        gsub(/:kind\b/, @backup.kind.to_s) .
+        gsub(/:id\b/, @backup.id.to_s) .
+        gsub(/:timestamp\b/, @backup.timestamp)
+      end
+
+      private
+
+      def verbose?
+        config[:verbose]
+      end
+
+      def local_only?
+        config[:local_only]
+      end
+
+      def dry_run?
+        config[:dry_run]
+      end
+    end
+  end
+end

data/lib/astrails/safe/svndump.rb

@@ -0,0 +1,13 @@
+module Astrails
+  module Safe
+    class Svndump < Source
+
+      def command
+        "svnadmin dump #{config[:options]} #{config[:repo_path]}"
+      end
+
+      def extension; '.svn'; end
+
+    end
+  end
+end

data/lib/astrails/safe/tmp_file.rb

@@ -0,0 +1,48 @@
+require 'tmpdir'
+module Astrails
+  module Safe
+    module TmpFile
+      @keep_files = []
+
+      def self.tmproot
+        @tmproot ||= Dir.mktmpdir
+      end
+
+      def self.cleanup
+        begin
+          FileUtils.remove_entry_secure tmproot
+        rescue ArgumentError => e
+          if e.message =~ /parent directory is world writable/
+            puts <<-ERR
+
+
+********************************************************************************
+It looks like you have wrong permissions on your TEMP directory.  The usual
+case is when you have world writable TEMP directory withOUT the sticky bit.
+
+Try "chmod +t" on it.
+
+********************************************************************************
+
+ERR
+          else
+            raise
+          end
+        end
+        @tmproot = nil
+      end
+
+      def self.create(name)
+        # create temp directory
+
+        file = Tempfile.new(name, tmproot)
+
+        yield file
+
+        file.close
+        @keep_files << file # so that it will not get gcollected and removed from filesystem until the end
+        file.path
+      end
+    end
+  end
+end

data/lib/astrails/safe/version.rb

@@ -0,0 +1,5 @@
+module Astrails
+  module Safe
+    VERSION = "0.3.1"
+  end
+end

data/lib/extensions/mktmpdir.rb

@@ -0,0 +1,45 @@
+require 'tmpdir'
+
+unless Dir.respond_to?(:mktmpdir)
+  # backward compat for 1.8.6
+  class Dir
+    def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
+      case prefix_suffix
+      when nil
+        prefix = "d"
+        suffix = ""
+      when String
+        prefix = prefix_suffix
+        suffix = ""
+      when Array
+        prefix = prefix_suffix[0]
+        suffix = prefix_suffix[1]
+      else
+        raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+      end
+      tmpdir ||= Dir.tmpdir
+      t = Time.now.strftime("%Y%m%d")
+      n = nil
+      begin
+        path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+        path << "-#{n}" if n
+        path << suffix
+        Dir.mkdir(path, 0700)
+      rescue Errno::EEXIST
+        n ||= 0
+        n += 1
+        retry
+      end
+
+      if block_given?
+        begin
+          yield path
+        ensure
+          FileUtils.remove_entry_secure path
+        end
+      else
+        path
+      end
+    end
+  end
+end

data/spec/astrails/safe/archive_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe Astrails::Safe::Archive do
+
+  def def_config
+    {
+      :options => "OPTS",
+      :files   => "apples",
+      :exclude => "oranges"
+    }
+  end
+
+  def archive(id = :foo, config = def_config)
+    Astrails::Safe::Archive.new(id, Astrails::Safe::Config::Node.new(nil, config))
+  end
+
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @archive = archive
+      stub(@archive).timestamp {"NOW"}
+    end
+
+    {
+      :id => "foo",
+      :kind => "archive",
+      :extension => ".tar",
+      :filename => "archive-foo.NOW",
+      :command => "tar -cf - OPTS --exclude=oranges apples",
+    }.each do |k, v|
+      it "should set #{k} to #{v}" do
+        @archive.backup.send(k).should == v
+      end
+    end
+  end
+
+  describe :tar_exclude_files do
+    it "should return '' when no excludes" do
+      archive(:foo, {}).send(:tar_exclude_files).should == ''
+    end
+
+    it "should accept single exclude as string" do
+      archive(:foo, {:exclude => "bar"}).send(:tar_exclude_files).should == '--exclude=bar'
+    end
+
+    it "should accept multiple exclude as array" do
+      archive(:foo, {:exclude => ["foo", "bar"]}).send(:tar_exclude_files).should == '--exclude=foo --exclude=bar'
+    end
+  end
+
+  describe :tar_files do
+    it "should raise RuntimeError when no files" do
+      lambda {
+        archive(:foo, {}).send(:tar_files)
+      }.should raise_error(RuntimeError, "missing files for tar")
+    end
+
+    it "should accept single file as string" do
+      archive(:foo, {:files => "foo"}).send(:tar_files).should == "foo"
+    end
+
+    it "should accept multiple files as array" do
+      archive(:foo, {:files => ["foo", "bar"]}).send(:tar_files).should == "foo bar"
+    end
+  end
+end

data/spec/astrails/safe/cloudfiles_spec.rb

@@ -0,0 +1,175 @@
+require 'spec_helper'
+
+describe Astrails::Safe::Cloudfiles do
+
+  def def_config
+    {
+      cloudfiles: {
+        container: '_container',
+        user:      '_user',
+        api_key:   '_api_key',
+      },
+      keep: { cloudfiles: 2 }
+    }
+  end
+
+  def def_backup(extra = {})
+    {
+      kind:      '_kind',
+      filename:  '/backup/somewhere/_kind-_id.NOW.bar',
+      extension: '.bar',
+      id:        '_id',
+      timestamp: 'NOW'
+    }.merge(extra)
+  end
+
+  def cloudfiles(config = def_config, backup = def_backup)
+    Astrails::Safe::Cloudfiles.new(
+      Astrails::Safe::Config::Node.new.merge(config),
+      Astrails::Safe::Backup.new(backup)
+    )
+  end
+
+  describe :cleanup do
+
+    before(:each) do
+      @cloudfiles = cloudfiles
+
+      @files = [4,1,3,2].map { |i| "aaaaa#{i}" }
+
+      @container = "container"
+
+      stub(@container).objects(prefix: "_kind/_id/_kind-_id.") { @files }
+      stub(@container).delete_object(anything)
+
+      stub(CloudFiles::Connection).
+        new('_user', '_api_key', true, false).stub!.
+        container('_container') {@container}
+    end
+
+    it "should check [:keep, :cloudfiles]" do
+      @cloudfiles.config[:keep].data["cloudfiles"] = nil
+      dont_allow(@cloudfiles.backup).filename
+      @cloudfiles.send :cleanup
+    end
+
+    it "should delete extra files" do
+      mock(@container).delete_object('aaaaa1')
+      mock(@container).delete_object('aaaaa2')
+      @cloudfiles.send :cleanup
+    end
+
+  end
+
+  describe :active do
+    before(:each) do
+      @cloudfiles = cloudfiles
+    end
+
+    it "should be true when all params are set" do
+      @cloudfiles.should be_active
+    end
+
+    it "should be false if container is missing" do
+      @cloudfiles.config[:cloudfiles].data["container"] = nil
+      @cloudfiles.should_not be_active
+    end
+
+    it "should be false if user is missing" do
+      @cloudfiles.config[:cloudfiles].data["user"] = nil
+      @cloudfiles.should_not be_active
+    end
+
+    it "should be false if api_key is missing" do
+      @cloudfiles.config[:cloudfiles].data["api_key"] = nil
+      @cloudfiles.should_not be_active
+    end
+  end
+
+  describe :path do
+    before(:each) do
+      @cloudfiles = cloudfiles
+    end
+    it "should use cloudfiles/path 1st" do
+      @cloudfiles.config[:cloudfiles].data["path"] = "cloudfiles_path"
+      @cloudfiles.config[:local] = {path: "local_path"}
+      @cloudfiles.send(:path).should == "cloudfiles_path"
+    end
+
+    it "should use local/path 2nd" do
+      @cloudfiles.config.merge local: {path: 'local_path'}
+      @cloudfiles.send(:path).should == 'local_path'
+    end
+
+    it "should use constant 3rd" do
+      @cloudfiles.send(:path).should == "_kind/_id"
+    end
+
+  end
+
+  describe :save do
+    def add_stubs(*stubs)
+      stubs.each do |s|
+        case s
+        when :connection
+          @connection = "connection"
+          stub(CloudFiles::Authentication).new
+          stub(CloudFiles::Connection).
+            new('_user', '_api_key', true, false) {@connection}
+        when :file_size
+          stub(@cloudfiles).get_file_size("foo") {123}
+        when :create_container
+          @container = "container"
+          stub(@container).create_object("_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar", true) {@object}
+          stub(@connection).create_container {@container}
+        when :file_open
+          stub(File).open("foo")
+        when :cloudfiles_store
+          @object = "object"
+          stub(@object).write(nil) {true}
+        end
+      end
+    end
+
+    before(:each) do
+      @cloudfiles = cloudfiles(def_config, def_backup(path: 'foo'))
+      @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar"
+    end
+
+    it "should fail if no backup.file is set" do
+      @cloudfiles.backup.path = nil
+      proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError)
+    end
+
+    it "should establish Cloud Files connection" do
+      add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
+      @cloudfiles.send(:save)
+    end
+
+    it "should open local file" do
+      add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
+      mock(File).open("foo")
+      @cloudfiles.send(:save)
+    end
+
+    it "should call write on the cloudfile object with files' descriptor" do
+      add_stubs(:connection, :file_size, :create_container, :cloudfiles_store)
+      stub(File).open("foo") {"qqq"}
+      mock(@object).write("qqq") {true}
+      @cloudfiles.send(:save)
+    end
+
+    it "should upload file" do
+      add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store)
+      @cloudfiles.send(:save)
+    end
+
+    it "should fail on files bigger then 5G" do
+      add_stubs(:connection)
+      mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
+      mock(STDERR).puts(anything)
+      dont_allow(Benchmark).realtime
+      @cloudfiles.send(:save)
+    end
+  end
+end

data/spec/astrails/safe/config_spec.rb

@@ -0,0 +1,307 @@
+require 'spec_helper'
+
+describe Astrails::Safe::Config do
+  it "should parse example config" do
+    config = Astrails::Safe::Config::Node.new do
+
+      dry_run false
+      local_only true
+      verbose true
+
+      local do
+        path "path"
+      end
+
+      s3 do
+        key "s3 key"
+        secret "secret"
+        bucket "bucket"
+        path "path1"
+      end
+
+      sftp do
+        user "sftp user"
+        password "sftp password"
+        host "sftp host"
+      end
+
+      gpg do
+        password "astrails"
+        key "gpg-key"
+      end
+
+      keep do
+        s3 20
+        local 4
+      end
+
+      mysqldump do
+        options "-ceKq --single-transaction --create-options"
+
+        user "astrails"
+        password ""
+        host "localhost"
+        port 3306
+        socket "/var/run/mysqld/mysqld.sock"
+
+        database :blog
+
+        database :production do
+          keep :local => 3
+
+          gpg do
+            password "custom-production-pass"
+          end
+
+          skip_tables [:logger_exceptions, :request_logs]
+        end
+
+      end
+
+      pgdump do
+        options "-i -x -O"
+
+        user "astrails"
+        password ""
+        host "localhost"
+        port 5432
+
+        database :blog
+
+        database :production do
+          keep :local => 3
+
+          skip_tables [:logger_exceptions, :request_logs]
+        end
+
+      end
+
+      svndump do
+        repo :my_repo do
+          repo_path "/home/svn/my_repo"
+        end
+      end
+
+      tar do
+        archive "git-repositories" do
+          files "/home/git/repositories"
+        end
+
+        archive "etc-files" do
+          files "/etc"
+          exclude "/etc/puppet/other"
+        end
+
+        archive "dot-configs" do
+          files "/home/*/.[^.]*"
+        end
+
+        archive "blog" do
+          files "/var/www/blog.astrails.com/"
+          exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
+        end
+
+        archive :misc do
+          files [ "/backup/*.rb" ]
+        end
+      end
+
+      mongodump do
+        host "host"
+        database "database"
+        user "user"
+        password "password"
+      end
+    end
+
+    expected = {
+      "dry_run" => false,
+      "local_only" => true,
+      "verbose" => true,
+
+      "local" => {"path" => "path"},
+
+      "s3" => {
+        "key" => "s3 key",
+        "secret" => "secret",
+        "bucket" => "bucket",
+        "path" => "path1",
+      },
+
+      "sftp" => {
+        "user" => "sftp user",
+        "password" => "sftp password",
+        "host" => "sftp host",
+      },
+
+      "gpg" => {"password" => "astrails", "key" => "gpg-key"},
+
+      "keep" => {"s3" => 20, "local" => 4},
+
+      "mysqldump" => {
+        "options" => "-ceKq --single-transaction --create-options",
+        "user" => "astrails",
+        "password" => "",
+        "host" => "localhost",
+        "port" => 3306,
+        "socket" => "/var/run/mysqld/mysqld.sock",
+
+        "databases" => {
+          "blog" => {},
+          "production" => {
+           "keep" => {"local" => 3},
+           "gpg" => {"password" => "custom-production-pass"},
+            "skip_tables" => ["logger_exceptions", "request_logs"],
+          },
+        },
+      },
+
+      "pgdump" => {
+        "options" => "-i -x -O",
+        "user" => "astrails",
+        "password" => "",
+        "host" => "localhost",
+        "port" => 5432,
+
+        "databases" => {
+          "blog" => {},
+          "production" => {
+           "keep" => {"local" => 3},
+            "skip_tables" => ["logger_exceptions", "request_logs"],
+          },
+        },
+      },
+
+      "svndump" => {
+        "repos" => {
+          "my_repo"=> {
+            "repo_path" => "/home/svn/my_repo"
+          }
+        }
+      },
+
+      "tar" => {
+        "archives" => {
+          "git-repositories" => {"files" => ["/home/git/repositories"]},
+          "etc-files" => {"files" => ["/etc"], "exclude" => ["/etc/puppet/other"]},
+          "dot-configs" => {"files" => ["/home/*/.[^.]*"]},
+          "blog" => {
+            "files" => ["/var/www/blog.astrails.com/"],
+            "exclude" => ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"],
+          },
+          "misc" => { "files" => ["/backup/*.rb"] },
+        },
+      },
+
+      "mongodump" => {
+        "host" => "host",
+        "databases" => {
+          "database" => {}
+        },
+        "user" => "user",
+        "password" => "password"
+      }
+    }
+
+    config.to_hash.should == expected
+  end
+
+  it "should make an array from multivalues" do
+    config = Astrails::Safe::Config::Node.new do
+      skip_tables "a"
+      skip_tables "b"
+      files "/foo"
+      files "/bar"
+      exclude "/foo/bar"
+      exclude "/foo/bar/baz"
+    end
+
+    expected = {
+      "skip_tables" => ["a", "b"],
+      "files" => ["/foo", "/bar"],
+      "exclude" => ["/foo/bar", "/foo/bar/baz"],
+    }
+
+    config.to_hash.should == expected
+  end
+
+  it "should raise error on key duplication" do
+    proc do
+      Astrails::Safe::Config::Node.new do
+        path "foo"
+        path "bar"
+      end
+    end.should raise_error(ArgumentError, "duplicate value for 'path'")
+  end
+
+  it "should accept hash as data" do
+    Astrails::Safe::Config::Node.new do
+      tar do
+        archive 'blog', files: 'foo', exclude: ['aaa', 'bbb']
+      end
+    end.to_hash.should == {
+      'tar' => {
+        'archives' => {
+          'blog' => {
+            'files' => ['foo'],
+            'exclude' => ['aaa', 'bbb']
+          }
+        }
+      }
+    }
+  end
+
+  it "should accept hash as data and a block" do
+    Astrails::Safe::Config::Node.new do
+      tar do
+        archive 'blog', files: 'foo' do
+          exclude ['aaa', 'bbb']
+        end
+      end
+    end.to_hash.should == {
+      'tar' => {
+        'archives' => {
+          'blog' => {
+            'files' => ['foo'],
+            'exclude' => ['aaa', 'bbb']
+          }
+        }
+      }
+    }
+  end
+
+  it 'should accept multiple levels of data hash' do
+    config = Astrails::Safe::Config::Node.new nil, tar: {
+      s3: { bucket: '_bucket', key: '_key', secret: '_secret', },
+      keep: { s3: 2 }
+    }
+
+    config.to_hash.should == {
+      'tar' => {
+        's3' => { 'bucket' => '_bucket', 'key' => '_key', 'secret' => '_secret', },
+        'keep' => { 's3' => 2 }
+      }
+    }
+  end
+  
+  it 'should set multi value as array' do
+    config = Astrails::Safe::Config::Node.new do
+      tar do
+        archive 'foo' do
+          files 'bar'
+        end
+      end
+    end
+
+    config.to_hash.should == {
+      'tar' => {
+        'archives' => {
+          'foo' => {
+            'files' => ['bar']
+          }
+        }
+      }
+    }
+  end
+
+end