backup 3.0.20 → 3.0.21

data/spec/storage/cloudfiles_spec.rb

@@ -3,118 +3,228 @@
 require File.expand_path('../../spec_helper.rb', __FILE__)
 
 describe Backup::Storage::CloudFiles do
-
-  let(:cf) do
-    Backup::Storage::CloudFiles.new do |cf|
+  let(:model)   { Backup::Model.new(:test_trigger, 'test label') }
+  let(:storage) do
+    Backup::Storage::CloudFiles.new(model) do |cf|
       cf.username  = 'my_username'
       cf.api_key   = 'my_api_key'
-      cf.container = 'my_container'
-      cf.path      = 'backups'
-      cf.keep      = 20
       cf.auth_url  = 'lon.auth.api.rackspacecloud.com'
+      cf.container = 'my_container'
+      cf.keep      = 5
     end
   end
 
-  before do
-    Backup::Configuration::Storage::CloudFiles.clear_defaults!
-  end
-
-  it 'should have defined the configuration properly' do
-    cf.username.should  == 'my_username'
-    cf.api_key.should   == 'my_api_key'
-    cf.container.should == 'my_container'
-    cf.path.should      == 'backups'
-    cf.keep.should      == 20
-    cf.auth_url.should  == 'lon.auth.api.rackspacecloud.com'
-  end
-
-  it 'should use the defaults if a particular attribute has not been defined' do
-    Backup::Configuration::Storage::CloudFiles.defaults do |cf|
-      cf.username = 'my_username'
-      cf.api_key  = 'my_api_key'
+  describe '#initialize' do
+    it 'should set the correct values' do
+      storage.username.should   == 'my_username'
+      storage.api_key.should    == 'my_api_key'
+      storage.auth_url.should   == 'lon.auth.api.rackspacecloud.com'
+      storage.container.should  == 'my_container'
+      storage.servicenet.should == false
+      storage.path.should       == 'backups'
+
+      storage.storage_id.should be_nil
+      storage.keep.should       == 5
     end
 
-    cf = Backup::Storage::CloudFiles.new do |cf|
-      cf.container = 'my_container'
-      cf.path      = 'my/backups'
+    it 'should set a storage_id if given' do
+      cf = Backup::Storage::CloudFiles.new(model, 'my storage_id')
+      cf.storage_id.should == 'my storage_id'
     end
 
-    cf.username.should  == 'my_username'
-    cf.api_key.should   == 'my_api_key'
-    cf.container.should == 'my_container'
-    cf.path.should      == 'my/backups'
-  end
+    context 'when setting configuration defaults' do
+      after { Backup::Configuration::Storage::CloudFiles.clear_defaults! }
+
+      it 'should use the configured defaults' do
+        Backup::Configuration::Storage::CloudFiles.defaults do |cf|
+          cf.username   = 'some_username'
+          cf.api_key    = 'some_api_key'
+          cf.auth_url   = 'some_auth_url'
+          cf.container  = 'some_container'
+          cf.servicenet = true
+          cf.path       = 'some_path'
+          cf.keep       = 15
+        end
+        storage = Backup::Storage::CloudFiles.new(model)
+        storage.username.should   == 'some_username'
+        storage.api_key.should    == 'some_api_key'
+        storage.auth_url.should   == 'some_auth_url'
+        storage.container.should  == 'some_container'
+        storage.servicenet.should == true
+        storage.path.should       == 'some_path'
+
+        storage.storage_id.should be_nil
+        storage.keep.should       == 15
+      end
+
+      it 'should override the configured defaults' do
+        Backup::Configuration::Storage::CloudFiles.defaults do |cf|
+          cf.username   = 'old_username'
+          cf.api_key    = 'old_api_key'
+          cf.auth_url   = 'old_auth_url'
+          cf.container  = 'old_container'
+          cf.servicenet = true
+          cf.path       = 'old_path'
+          cf.keep       = 15
+        end
+        storage = Backup::Storage::CloudFiles.new(model) do |cf|
+          cf.username   = 'new_username'
+          cf.api_key    = 'new_api_key'
+          cf.auth_url   = 'new_auth_url'
+          cf.container  = 'new_container'
+          cf.servicenet = false
+          cf.path       = 'new_path'
+          cf.keep       = 10
+        end
+
+        storage.username.should   == 'new_username'
+        storage.api_key.should    == 'new_api_key'
+        storage.auth_url.should   == 'new_auth_url'
+        storage.container.should  == 'new_container'
+        storage.servicenet.should == false
+        storage.path.should       == 'new_path'
+
+        storage.storage_id.should be_nil
+        storage.keep.should       == 10
+      end
+    end # context 'when setting configuration defaults'
+
+  end # describe '#initialize'
 
-  describe '#perform' do
-    it 'should invoke transfer! and cycle!' do
-      cf.expects(:transfer!)
-      cf.expects(:cycle!)
-      cf.perform!
+  describe '#provider' do
+    it 'should set the Fog provider' do
+      storage.send(:provider).should == 'Rackspace'
     end
   end
 
   describe '#connection' do
-    it 'should establish and re-use a connection to Rackspace Cloud Files' do
-      Fog::Storage.expects(:new).once.with({
-        :provider             => 'Rackspace',
-        :rackspace_username   => 'my_username',
-        :rackspace_api_key    => 'my_api_key',
-        :rackspace_auth_url   => 'lon.auth.api.rackspacecloud.com',
-        :rackspace_servicenet => false
-      }).returns(true)
-
-      cf.send(:connection)
-      cf.send(:connection)
+    let(:connection) { mock }
+
+    context 'when @servicenet is set to false' do
+      it 'should create a new standard connection' do
+        Fog::Storage.expects(:new).once.with(
+          :provider             => 'Rackspace',
+          :rackspace_username   => 'my_username',
+          :rackspace_api_key    => 'my_api_key',
+          :rackspace_auth_url   => 'lon.auth.api.rackspacecloud.com',
+          :rackspace_servicenet => false
+        ).returns(connection)
+        storage.send(:connection).should == connection
+      end
     end
 
-    context 'with LAN (servicenet) enabled' do
-      it 'should establish and re-use a connection to Rackspace Cloud Files' do
-        Fog::Storage.expects(:new).once.with({
+    context 'when @servicenet is set to true' do
+      before do
+        storage.servicenet = true
+      end
+
+      it 'should create a new servicenet connection' do
+        Fog::Storage.expects(:new).once.with(
           :provider             => 'Rackspace',
           :rackspace_username   => 'my_username',
           :rackspace_api_key    => 'my_api_key',
           :rackspace_auth_url   => 'lon.auth.api.rackspacecloud.com',
           :rackspace_servicenet => true
-        }).returns(true)
-
-        cf.servicenet = true
-        cf.send(:connection)
-        cf.send(:connection)
+        ).returns(connection)
+        storage.send(:connection).should == connection
       end
     end
-  end
 
-  describe '#provider' do
-    it 'should be Rackspace' do
-      cf.provider == 'Rackspace'
+    it 'should return an existing connection' do
+      Fog::Storage.expects(:new).once.returns(connection)
+      storage.send(:connection).should == connection
+      storage.send(:connection).should == connection
     end
-  end
+
+  end # describe '#connection'
 
   describe '#transfer!' do
-    let(:connection) { mock('Fog::Storage') }
+    let(:connection) { mock }
+    let(:package) { mock }
+    let(:file) { mock }
+    let(:s) { sequence '' }
+
     before do
-      Fog::Storage.expects(:new).once.returns(connection)
+      storage.instance_variable_set(:@package, package)
+      storage.stubs(:storage_name).returns('Storage::CloudFiles')
+      storage.stubs(:local_path).returns('/local/path')
+      storage.stubs(:connection).returns(connection)
     end
 
-    it 'should transfer the provided file to the container' do
-      Backup::Model.new('blah', 'blah') {}
-      file = mock("Backup::Storage::CloudFiles::File")
-      File.expects(:open).with("#{File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER}")}.tar").returns(file)
-      connection.expects(:put_object).with('my_container', "backups/myapp/#{ Backup::TIME }/#{ Backup::TRIGGER }.tar", file)
-      cf.send(:transfer!)
+    it 'should transfer the package files' do
+      storage.expects(:remote_path_for).in_sequence(s).with(package).
+          returns('remote/path')
+      storage.expects(:files_to_transfer_for).in_sequence(s).with(package).
+        multiple_yields(
+        ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
+        ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
+      )
+      # first yield
+      Backup::Logger.expects(:message).in_sequence(s).with(
+        "Storage::CloudFiles started transferring " +
+        "'2011.12.31.11.00.02.backup.tar.enc-aa'."
+      )
+      File.expects(:open).in_sequence(s).with(
+        File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-aa'), 'r'
+      ).yields(file)
+      connection.expects(:put_object).in_sequence(s).with(
+        'my_container', File.join('remote/path', 'backup.tar.enc-aa'), file
+      )
+      # second yield
+      Backup::Logger.expects(:message).in_sequence(s).with(
+        "Storage::CloudFiles started transferring " +
+        "'2011.12.31.11.00.02.backup.tar.enc-ab'."
+      )
+      File.expects(:open).in_sequence(s).with(
+        File.join('/local/path', '2011.12.31.11.00.02.backup.tar.enc-ab'), 'r'
+      ).yields(file)
+      connection.expects(:put_object).in_sequence(s).with(
+        'my_container', File.join('remote/path', 'backup.tar.enc-ab'), file
+      )
+
+      storage.send(:transfer!)
     end
-  end
+  end # describe '#transfer!'
 
   describe '#remove!' do
-    let(:connection) { mock('Fog::Storage') }
+    let(:package) { mock }
+    let(:connection) { mock }
+    let(:s) { sequence '' }
+
     before do
-      Fog::Storage.expects(:new).once.returns(connection)
+      storage.stubs(:storage_name).returns('Storage::CloudFiles')
+      storage.stubs(:connection).returns(connection)
     end
 
-    it 'should remove the file from the container' do
-      connection.expects(:delete_object).with('my_container', "backups/myapp/#{ Backup::TIME }/#{ Backup::TRIGGER }.tar")
-      cf.send(:remove!)
+    it 'should remove the package files' do
+      storage.expects(:remote_path_for).in_sequence(s).with(package).
+          returns('remote/path')
+      storage.expects(:transferred_files_for).in_sequence(s).with(package).
+        multiple_yields(
+        ['2011.12.31.11.00.02.backup.tar.enc-aa', 'backup.tar.enc-aa'],
+        ['2011.12.31.11.00.02.backup.tar.enc-ab', 'backup.tar.enc-ab']
+      )
+      # first yield
+      Backup::Logger.expects(:message).in_sequence(s).with(
+        "Storage::CloudFiles started removing " +
+        "'2011.12.31.11.00.02.backup.tar.enc-aa' " +
+        "from container 'my_container'."
+      )
+      connection.expects(:delete_object).in_sequence(s).with(
+        'my_container', File.join('remote/path', 'backup.tar.enc-aa')
+      )
+      # second yield
+      Backup::Logger.expects(:message).in_sequence(s).with(
+        "Storage::CloudFiles started removing " +
+        "'2011.12.31.11.00.02.backup.tar.enc-ab' " +
+        "from container 'my_container'."
+      )
+      connection.expects(:delete_object).in_sequence(s).with(
+        'my_container', File.join('remote/path', 'backup.tar.enc-ab')
+      )
+
+      storage.send(:remove!, package)
     end
-  end
+  end # describe '#remove!'
 
 end

data/spec/storage/cycler_spec.rb

@@ -0,0 +1,239 @@
+# encoding: utf-8
+
+require File.expand_path('../../spec_helper.rb', __FILE__)
+
+describe 'Backup::Storage::Cycler' do
+  let(:cycler)        { Backup::Storage::Cycler }
+  let(:storage)       { mock }
+  let(:package)       { mock }
+  let(:storage_file)  { mock }
+  let(:pkg_a)         { mock }
+  let(:pkg_b)         { mock }
+  let(:pkg_c)         { mock }
+  let(:s)             { sequence '' }
+
+  describe '#cycle!' do
+    it 'should setup variables and initiate cycling' do
+      cycler.expects(:storage_file).in_sequence(s).returns(storage_file)
+      cycler.expects(:update_storage_file!).in_sequence(s)
+      cycler.expects(:remove_packages!).in_sequence(s)
+      cycler.cycle!(storage, package)
+      cycler.instance_variable_get(:@storage).should be(storage)
+      cycler.instance_variable_get(:@package).should be(package)
+      cycler.instance_variable_get(:@storage_file).should be(storage_file)
+    end
+  end
+
+  describe 'update_storage_file!' do
+    before do
+      storage.stubs(:keep).returns(2)
+      cycler.instance_variable_set(:@storage, storage)
+      cycler.instance_variable_set(:@package, package)
+      cycler.instance_variable_set(:@storage_file, storage_file)
+    end
+
+    it 'should remove entries and set @packages_to_remove' do
+      cycler.expects(:yaml_load).in_sequence(s).returns([pkg_a, pkg_b, pkg_c])
+      cycler.expects(:yaml_save).in_sequence(s).with([package, pkg_a])
+      cycler.send(:update_storage_file!)
+      cycler.instance_variable_get(:@packages_to_remove).should == [pkg_b, pkg_c]
+    end
+  end
+
+  describe '#remove_packages!' do
+    before do
+      cycler.instance_variable_set(:@storage, storage)
+      cycler.instance_variable_set(:@packages_to_remove, [pkg_a, pkg_b, pkg_c])
+    end
+
+    it 'should call the @storage to remove the old packages' do
+      storage.expects(:remove!).in_sequence(s).with(pkg_a)
+      storage.expects(:remove!).in_sequence(s).with(pkg_b)
+      storage.expects(:remove!).in_sequence(s).with(pkg_c)
+      cycler.send(:remove_packages!)
+    end
+
+    context 'when errors occur removing packages' do
+      before do
+        pkg_b.stubs(:trigger).returns('pkg_trigger')
+        pkg_b.stubs(:time).returns('pkg_time')
+        pkg_b.stubs(:filenames).returns(['file1', 'file2'])
+      end
+
+      it 'should warn and continue' do
+        storage.expects(:remove!).in_sequence(s).with(pkg_a)
+        storage.expects(:remove!).in_sequence(s).with(pkg_b).raises('error message')
+        Backup::Logger.expects(:warn).with do |err|
+          err.should be_an_instance_of Backup::Errors::Storage::CyclerError
+          err.message.should == 'Storage::CyclerError: ' +
+              "There was a problem removing the following package:\n" +
+              "  Trigger: pkg_trigger :: Dated: pkg_time\n" +
+              "  Package included the following 2 file(s):\n" +
+              "  file1\n" +
+              "  file2\n" +
+              "  Reason: RuntimeError\n" +
+              "  error message"
+        end
+        storage.expects(:remove!).in_sequence(s).with(pkg_c)
+
+        expect do
+          cycler.send(:remove_packages!)
+        end.not_to raise_error
+      end
+    end
+
+  end # describe '#remove_packages!'
+
+  describe '#storage_file' do
+    before do
+      cycler.instance_variable_set(:@storage, storage)
+      storage.stubs(:class).returns('Backup::Storage::S3')
+      cycler.instance_variable_set(:@package, package)
+      package.stubs(:trigger).returns('pkg_trigger')
+    end
+
+    context 'when the @storage.storage_id is not set' do
+      before { storage.stubs(:storage_id).returns(nil) }
+      it 'should return the path to the YAML storage file with no suffix' do
+        cycler.send(:storage_file).should ==
+            File.join(Backup::Config.data_path, 'pkg_trigger', 'S3.yml')
+      end
+    end
+
+    context 'when the @storage.storage_id is set' do
+      before { storage.stubs(:storage_id).returns('Storage #1') }
+      it 'should sanitize the storage_id to use as a filename suffix' do
+        cycler.send(:storage_file).should ==
+            File.join(Backup::Config.data_path, 'pkg_trigger', 'S3-Storage__1.yml')
+      end
+    end
+
+  end # describe '#storage_file'
+
+  describe '#yaml_load' do
+    let(:obj_a) { mock }
+    let(:obj_b) { mock }
+    let(:obj_c) { mock }
+    let(:unsorted_objects)  { [obj_a, obj_c, obj_b] }
+    let(:sorted_objects)    { [obj_c, obj_b, obj_a] }
+
+    before do
+      cycler.instance_variable_set(:@storage_file, storage_file)
+      obj_a.instance_variable_set(:@time, '2012.01.01.07.00.00')
+      obj_b.instance_variable_set(:@time, '2012.01.01.08.00.00')
+      obj_c.instance_variable_set(:@time, '2012.01.01.09.00.00')
+    end
+
+    context 'when the storage file exists' do
+      before { File.expects(:exist?).with(storage_file).returns(true) }
+      context 'when the file is not empty' do
+        before { File.expects(:zero?).with(storage_file).returns(false) }
+        it 'should return YAML deserialized objects in an array, sorted by time DESC' do
+          YAML.expects(:load_file).with(storage_file).returns(unsorted_objects)
+          cycler.expects(:check_upgrade).with(sorted_objects).returns(sorted_objects)
+          cycler.send(:yaml_load).should be(sorted_objects)
+        end
+      end
+      context 'when the file is empty' do
+        before { File.expects(:zero?).with(storage_file).returns(true) }
+        it 'should return an empty array' do
+          cycler.send(:yaml_load).should == []
+        end
+      end
+    end
+
+    context 'when the storage file does not exist' do
+      before { File.expects(:exist?).with(storage_file).returns(false) }
+      it 'should return an empty array' do
+        cycler.send(:yaml_load).should == []
+      end
+    end
+  end # describe '#yaml_load'
+
+  describe '#yaml_save' do
+    let(:file) { mock }
+    let(:pkgs) { [ [1, 2, 3], [4, 5, 6] ] }
+    let(:pkgs_to_yaml) { pkgs.to_yaml }
+
+    it 'should save the given packages to the storage file in YAML format' do
+      cycler.instance_variable_set(:@storage_file, storage_file)
+      File.expects(:open).with(storage_file, 'w').yields(file)
+      file.expects(:write).with(pkgs_to_yaml)
+      cycler.send(:yaml_save, pkgs)
+    end
+  end # describe '#yaml_save'
+
+  describe '#check_upgrade' do
+    let(:yaml_v3_0_19) do
+      <<-EOF.gsub(/^ +/,'  ').gsub(/^  (-\W\W)/, "\\1")
+        ---
+        - !ruby/object:Backup::Storage::Local
+          time: 2011.11.01.12.01.00
+          remote_file: 2011.11.01.12.01.00.backup.tar.gz
+        - !ruby/object:Backup::Storage::Local
+          time: 2011.11.01.12.02.00
+          remote_file: 2011.11.01.12.02.00.backup.tar.gz
+        - !ruby/object:Backup::Storage::Local
+          time: 2011.11.01.12.03.00
+          remote_file: 2011.11.01.12.03.00.backup.tar.gz
+      EOF
+    end
+    let(:yaml_v3_0_20) do
+      <<-EOF.gsub(/^ +/,'  ').gsub(/^  (-\W\W)/, "\\1")
+        ---
+        - !ruby/object:Backup::Storage::Local
+          time: 2011.12.01.12.01.00
+          chunk_suffixes: []
+          filename: 2011.12.01.12.01.00.backup.tar.gz
+          version: 3.0.20
+        - !ruby/object:Backup::Storage::Local
+          time: 2011.12.01.12.02.00
+          chunk_suffixes: []
+          filename: 2011.12.01.12.02.00.backup.tar.gz
+          version: 3.0.20
+        - !ruby/object:Backup::Storage::Local
+          time: 2011.12.01.12.03.00
+          chunk_suffixes:
+          - aa
+          - ab
+          filename: 2011.12.01.12.03.00.backup.tar.gz
+          version: 3.0.20
+      EOF
+    end
+
+    before do
+      model = Backup::Model.new('foo', 'foo')
+      cycler.instance_variable_set(:@storage, storage)
+      storage.instance_variable_set(:@model, model)
+    end
+
+    it 'should upgrade v3.0.20 objects' do
+      objects = YAML.load(yaml_v3_0_20)
+      packages = cycler.send(:check_upgrade, objects)
+
+      packages.all? {|pkg| pkg.is_a? Backup::Package }.should be_true
+      packages.all? {|pkg| pkg.extension == 'tar.gz' }.should be_true
+
+      packages[0].time.should == '2011.12.01.12.01.00'
+      packages[0].chunk_suffixes.should == []
+      packages[1].time.should == '2011.12.01.12.02.00'
+      packages[1].chunk_suffixes.should == []
+      packages[2].time.should == '2011.12.01.12.03.00'
+      packages[2].chunk_suffixes.should == ['aa', 'ab']
+    end
+
+    it 'should upgrade v3.0.19 objects' do
+      objects = YAML.load(yaml_v3_0_19)
+      packages = cycler.send(:check_upgrade, objects)
+
+      packages.all? {|pkg| pkg.is_a? Backup::Package }.should be_true
+      packages.all? {|pkg| pkg.extension == 'tar.gz' }.should be_true
+      packages.all? {|pkg| pkg.chunk_suffixes == []  }.should be_true
+
+      packages[0].time.should == '2011.11.01.12.01.00'
+      packages[1].time.should == '2011.11.01.12.02.00'
+      packages[2].time.should == '2011.11.01.12.03.00'
+    end
+  end # describe '#check_upgrade'
+
+end