backup 3.0.19 → 3.0.20

data/spec/storage/s3_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require File.dirname(__FILE__) + '/../spec_helper'
+require File.expand_path('../../spec_helper.rb', __FILE__)
 
 ##
 # available S3 regions:
@@ -50,38 +50,46 @@ describe Backup::Storage::S3 do
     s3.keep.should              == 500                # comes from the default configuration
   end
 
+  describe '#provider' do
+    it 'should be AWS' do
+      s3.provider == 'AWS'
+    end
+  end
+
+  describe '#perform' do
+    it 'should invoke transfer! and cycle!' do
+      s3.expects(:transfer!)
+      s3.expects(:cycle!)
+      s3.perform!
+    end
+  end
+
   describe '#connection' do
-    it 'should establish a connection to Amazon S3 using the provided credentials' do
-      Fog::Storage.expects(:new).with({
+    it 'should establish and re-use a connection to Amazon S3' do
+      Fog::Storage.expects(:new).once.with({
         :provider               => 'AWS',
         :aws_access_key_id      => 'my_access_key_id',
         :aws_secret_access_key  => 'my_secret_access_key',
         :region                 => 'us-east-1'
-      })
+      }).returns(true)
 
       s3.send(:connection)
-    end
-  end
-
-  describe '#provider' do
-    it 'should be AWS' do
-      s3.provider == 'AWS'
+      s3.send(:connection)
     end
   end
 
   describe '#transfer!' do
     let(:connection) { mock('Fog::Storage') }
     before do
-      Fog::Storage.stubs(:new).returns(connection)
+      Fog::Storage.expects(:new).once.returns(connection)
     end
 
     it 'should transfer the provided file to the bucket' do
       Backup::Model.new('blah', 'blah') {}
       file = mock("Backup::Storage::S3::File")
       File.expects(:open).with("#{File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER}")}.tar").returns(file)
-      s3.expects(:remote_file).returns("#{ Backup::TIME }.#{ Backup::TRIGGER }.tar").twice
       connection.expects(:sync_clock)
-      connection.expects(:put_object).with('my-bucket', "backups/myapp/#{ Backup::TIME }.#{ Backup::TRIGGER }.tar", file)
+      connection.expects(:put_object).with('my-bucket', "backups/myapp/#{ Backup::TIME }/#{ Backup::TRIGGER }.tar", file)
       s3.send(:transfer!)
     end
   end
@@ -89,23 +97,14 @@ describe Backup::Storage::S3 do
   describe '#remove!' do
     let(:connection) { mock('Fog::Storage') }
     before do
-      Fog::Storage.stubs(:new).returns(connection)
+      Fog::Storage.expects(:new).once.returns(connection)
     end
 
     it 'should remove the file from the bucket' do
-      s3.expects(:remote_file).returns("#{ Backup::TIME }.#{ Backup::TRIGGER }.tar")
       connection.expects(:sync_clock)
-      connection.expects(:delete_object).with('my-bucket', "backups/myapp/#{ Backup::TIME }.#{ Backup::TRIGGER }.tar")
+      connection.expects(:delete_object).with('my-bucket', "backups/myapp/#{ Backup::TIME }/#{ Backup::TRIGGER }.tar")
       s3.send(:remove!)
     end
   end
 
-  describe '#perform' do
-    it 'should invoke transfer! and cycle!' do
-      s3.expects(:transfer!)
-      s3.expects(:cycle!)
-      s3.perform!
-    end
-  end
-
 end

data/spec/storage/scp_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require File.dirname(__FILE__) + '/../spec_helper'
+require File.expand_path('../../spec_helper.rb', __FILE__)
 
 describe Backup::Storage::SCP do
 
@@ -52,76 +52,151 @@ describe Backup::Storage::SCP do
     scp.path.should == 'backups'
   end
 
+  describe '#perform' do
+    it 'should invoke transfer! and cycle!' do
+      scp.expects(:transfer!)
+      scp.expects(:cycle!)
+      scp.perform!
+    end
+  end
+
   describe '#connection' do
-    it 'should establish a connection to the remote server using the provided ip address and credentials' do
-      Net::SSH.expects(:start).with('123.45.678.90', 'my_username', :password => 'my_password', :port => 22)
-      scp.send(:connection)
+    it 'should establish a connection to the remote server' do
+      connection = mock
+      Net::SSH.expects(:start).with(
+        '123.45.678.90',
+        'my_username',
+        :password => 'my_password',
+        :port => 22
+      ).yields(connection)
+
+      scp.send(:connection) do |ssh|
+        ssh.should be connection
+      end
     end
   end
 
   describe '#transfer!' do
-    let(:connection) { mock('Net::SCP') }
 
     before do
-      Net::SSH.stubs(:start).returns(connection)
-      scp.stubs(:create_remote_directories!)
+      scp.stubs(:storage_name).returns('Storage::SCP')
     end
 
-    it 'should transfer the provided file to the path' do
-      Backup::Model.new('blah', 'blah') {}
-      file = mock("Backup::Storage::SCP::File")
+    context 'when file chunking is not used' do
+      it 'should create remote paths and transfer using a single connection' do
+        ssh, ssh_scp = mock, mock
+        local_file  = "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar"
+        remote_file = "#{ Backup::TRIGGER }.tar"
 
-      scp.expects(:create_remote_directories!)
+        scp.expects(:connection).yields(ssh)
+        scp.expects(:create_remote_directories).with(ssh)
 
-      ssh_scp = mock('Net::SSH::SCP')
-      connection.expects(:scp).returns(ssh_scp)
+        Backup::Logger.expects(:message).with(
+          "Storage::SCP started transferring '#{local_file}' to '#{scp.ip}'."
+        )
 
-      ssh_scp.expects(:upload!).with(
-        File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar"),
-        File.join('backups/myapp', "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar")
-      )
+        ssh.expects(:scp).returns(ssh_scp)
+        ssh_scp.expects(:upload!).with(
+          File.join(Backup::TMP_PATH, local_file),
+          File.join('backups/myapp', Backup::TIME, remote_file)
+        )
 
-      scp.send(:transfer!)
+        scp.send(:transfer!)
+      end
     end
-  end
+
+    context 'when file chunking is used' do
+      it 'should transfer all the provided files using a single connection' do
+        s = sequence ''
+        ssh, ssh_scp = mock, mock
+
+        scp.expects(:connection).in_sequence(s).yields(ssh)
+        scp.expects(:create_remote_directories).in_sequence(s).with(ssh)
+
+        scp.expects(:files_to_transfer).in_sequence(s).multiple_yields(
+          ['local_file1', 'remote_file1'], ['local_file2', 'remote_file2']
+        )
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Storage::SCP started transferring 'local_file1' to '#{scp.ip}'."
+        )
+        ssh.expects(:scp).in_sequence(s).returns(ssh_scp)
+        ssh_scp.expects(:upload!).in_sequence(s).with(
+          File.join(Backup::TMP_PATH, 'local_file1'),
+          File.join('backups/myapp', Backup::TIME, 'remote_file1')
+        )
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Storage::SCP started transferring 'local_file2' to '#{scp.ip}'."
+        )
+        ssh.expects(:scp).in_sequence(s).returns(ssh_scp)
+        ssh_scp.expects(:upload!).in_sequence(s).with(
+          File.join(Backup::TMP_PATH, 'local_file2'),
+          File.join('backups/myapp', Backup::TIME, 'remote_file2')
+        )
+
+        scp.send(:transfer!)
+      end
+    end
+
+  end # describe '#transfer!'
 
   describe '#remove!' do
-    let(:connection) { mock('Net::SCP') }
 
     before do
-      Net::SSH.stubs(:start).returns(connection)
+      scp.stubs(:storage_name).returns('Storage::SCP')
     end
 
-    it 'should remove the file from the remote server path' do
-      connection.expects(:exec!).with("rm backups/myapp/#{ Backup::TIME }.#{ Backup::TRIGGER }.tar")
+    it 'should remove all remote files with a single logger call' do
+      ssh = mock
+
+      scp.expects(:transferred_files).multiple_yields(
+        ['local_file1', 'remote_file1'], ['local_file2', 'remote_file2']
+      )
+
+      Backup::Logger.expects(:message).with(
+        "Storage::SCP started removing 'local_file1' from '#{scp.ip}'.\n" +
+        "Storage::SCP started removing 'local_file2' from '#{scp.ip}'."
+      )
+
+      scp.expects(:connection).yields(ssh)
+      ssh.expects(:exec!).with("rm -r 'backups/myapp/#{ Backup::TIME }'")
+
       scp.send(:remove!)
     end
-  end
 
-  describe '#create_remote_directories!' do
-    let(:connection) { mock('Net::SSH') }
+    it 'should raise an error if Net::SSH reports errors' do
+      ssh = mock
 
-    before do
-      Net::SSH.stubs(:start).returns(connection)
+      scp.expects(:transferred_files)
+      Backup::Logger.expects(:message)
+
+      scp.expects(:connection).yields(ssh)
+      ssh.expects(:exec!).yields('', :stderr, 'error message')
+
+      expect do
+        scp.send(:remove!)
+      end.to raise_error(
+        Backup::Errors::Storage::SCP::SSHError,
+        "Storage::SCP::SSHError: Net::SSH reported the following errors:\n" +
+        "  error message"
+      )
     end
 
+  end # describe '#remove!'
+
+  describe '#create_remote_directories' do
     it 'should properly create remote directories one by one' do
+      ssh = mock
       scp.path = 'backups/some_other_folder/another_folder'
 
-      connection.expects(:exec!).with("mkdir 'backups'")
-      connection.expects(:exec!).with("mkdir 'backups/some_other_folder'")
-      connection.expects(:exec!).with("mkdir 'backups/some_other_folder/another_folder'")
-      connection.expects(:exec!).with("mkdir 'backups/some_other_folder/another_folder/myapp'")
+      ssh.expects(:exec!).with("mkdir 'backups'")
+      ssh.expects(:exec!).with("mkdir 'backups/some_other_folder'")
+      ssh.expects(:exec!).with("mkdir 'backups/some_other_folder/another_folder'")
+      ssh.expects(:exec!).with("mkdir 'backups/some_other_folder/another_folder/myapp'")
+      ssh.expects(:exec!).with("mkdir 'backups/some_other_folder/another_folder/myapp/#{ Backup::TIME }'")
 
-      scp.send(:create_remote_directories!)
-    end
-  end
-
-  describe '#perform' do
-    it 'should invoke transfer! and cycle!' do
-      scp.expects(:transfer!)
-      scp.expects(:cycle!)
-      scp.perform!
+      scp.send(:create_remote_directories, ssh)
     end
   end
 

data/spec/storage/sftp_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require File.dirname(__FILE__) + '/../spec_helper'
+require File.expand_path('../../spec_helper.rb', __FILE__)
 
 describe Backup::Storage::SFTP do
 
@@ -52,72 +52,150 @@ describe Backup::Storage::SFTP do
     sftp.path.should == 'backups'
   end
 
-  describe '#connection' do
-    it 'should establish a connection to the remote server using the provided ip address and credentials' do
-      Net::SFTP.expects(:start).with('123.45.678.90', 'my_username', :password => 'my_password', :port => 22)
-      sftp.send(:connection)
+  describe '#perform' do
+    it 'should invoke transfer! and cycle!' do
+      sftp.expects(:transfer!)
+      sftp.expects(:cycle!)
+      sftp.perform!
     end
   end
 
+  describe '#connection' do
+    it 'should establish a connection to the remote server' do
+      connection = mock
+      Net::SFTP.expects(:start).with(
+        '123.45.678.90',
+        'my_username',
+        :password  => 'my_password',
+        :port      => 22
+      ).yields(connection)
+
+      sftp.send(:connection) do |conn|
+        conn.should be connection
+      end
+    end
+  end # describe '#connection'
+
   describe '#transfer!' do
-    let(:connection) { mock('Fog::Storage') }
+    let(:connection) { mock }
 
     before do
-      Net::SFTP.stubs(:start).returns(connection)
-      sftp.stubs(:create_remote_directories!)
+      sftp.stubs(:storage_name).returns('Storage::SFTP')
     end
 
-    it 'should transfer the provided file to the path' do
-      Backup::Model.new('blah', 'blah') {}
-      file = mock("Backup::Storage::SFTP::File")
+    context 'when file chunking is not used' do
+      it 'should create remote paths and transfer using a single connection' do
+        local_file  = "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar"
+        remote_file = "#{ Backup::TRIGGER }.tar"
 
-      sftp.expects(:create_remote_directories!)
-      connection.expects(:upload!).with(
-        File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar"),
-        File.join('backups/myapp', "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar")
-      )
+        sftp.expects(:connection).yields(connection)
+        sftp.expects(:create_remote_directories).with(connection)
 
-      sftp.send(:transfer!)
-    end
-  end
+        Backup::Logger.expects(:message).with(
+          "Storage::SFTP started transferring '#{local_file}' to '#{sftp.ip}'."
+        )
 
-  describe '#remove!' do
-    let(:connection) { mock('Net::SFTP') }
+        connection.expects(:upload!).with(
+          File.join(Backup::TMP_PATH, local_file),
+          File.join('backups/myapp', Backup::TIME, remote_file)
+        )
 
-    before do
-      Net::SFTP.stubs(:start).returns(connection)
+        sftp.send(:transfer!)
+      end
     end
 
-    it 'should remove the file from the remote server path' do
-      connection.expects(:remove!).with("backups/myapp/#{ Backup::TIME }.#{ Backup::TRIGGER }.tar")
-      sftp.send(:remove!)
+    context 'when file chunking is used' do
+      it 'should transfer all the provided files using a single connection' do
+        s = sequence ''
+
+        sftp.expects(:connection).in_sequence(s).yields(connection)
+        sftp.expects(:create_remote_directories).in_sequence(s).with(connection)
+
+        sftp.expects(:files_to_transfer).in_sequence(s).multiple_yields(
+          ['local_file1', 'remote_file1'], ['local_file2', 'remote_file2']
+        )
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Storage::SFTP started transferring 'local_file1' to '#{sftp.ip}'."
+        )
+        connection.expects(:upload!).in_sequence(s).with(
+          File.join(Backup::TMP_PATH, 'local_file1'),
+          File.join('backups/myapp', Backup::TIME, 'remote_file1')
+        )
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Storage::SFTP started transferring 'local_file2' to '#{sftp.ip}'."
+        )
+        connection.expects(:upload!).in_sequence(s).with(
+          File.join(Backup::TMP_PATH, 'local_file2'),
+          File.join('backups/myapp', Backup::TIME, 'remote_file2')
+        )
+
+        sftp.send(:transfer!)
+      end
     end
-  end
+  end # describe '#transfer'
+
+  describe '#remove!' do
+    it 'should remove all remote files with a single FTP connection' do
+      s = sequence ''
+      connection = mock
+      remote_path = "backups/myapp/#{ Backup::TIME }"
+      sftp.stubs(:storage_name).returns('Storage::SFTP')
 
-  describe '#create_remote_directories!' do
-    let(:connection) { mock('Net::SFTP') }
+      sftp.expects(:connection).in_sequence(s).yields(connection)
 
-    before do
-      Net::SFTP.stubs(:start).returns(connection)
-    end
+      sftp.expects(:transferred_files).in_sequence(s).multiple_yields(
+        ['local_file1', 'remote_file1'], ['local_file2', 'remote_file2']
+      )
 
-    it 'should properly create remote directories one by one' do
-      sftp.path = 'backups/some_other_folder/another_folder'
+      Backup::Logger.expects(:message).in_sequence(s).with(
+        "Storage::SFTP started removing 'local_file1' from '#{sftp.ip}'."
+      )
+      connection.expects(:remove!).in_sequence(s).with(
+        File.join(remote_path, 'remote_file1')
+      )
 
-      connection.expects(:mkdir!).with('backups')
-      connection.expects(:mkdir!).with('backups/some_other_folder')
-      connection.expects(:mkdir!).with('backups/some_other_folder/another_folder')
-      connection.expects(:mkdir!).with('backups/some_other_folder/another_folder/myapp')
+      Backup::Logger.expects(:message).in_sequence(s).with(
+        "Storage::SFTP started removing 'local_file2' from '#{sftp.ip}'."
+      )
+      connection.expects(:remove!).in_sequence(s).with(
+        File.join(remote_path, 'remote_file2')
+      )
 
-      sftp.send(:create_remote_directories!)
-    end
-  end
+      connection.expects(:rmdir!).in_sequence(s).with(remote_path)
 
-  describe '#perform' do
-    it 'should invoke transfer! and cycle!' do
-      sftp.expects(:transfer!)
-      sftp.expects(:cycle!)
-      sftp.perform!
+      sftp.send(:remove!)
+    end
+  end # describe '#remove!'
+
+  describe '#create_remote_directories' do
+    let(:connection) { mock }
+
+    context 'while properly creating remote directories one by one' do
+      it 'should rescue any SFTP::StatusExceptions' do
+        s = sequence ''
+        sftp.path = '~/backups/some_other_folder/another_folder'
+        sftp_response = stub(:code => 11, :message => nil)
+        sftp_status_exception = Net::SFTP::StatusException.new(sftp_response)
+
+        connection.expects(:mkdir!).in_sequence(s).
+            with("~").raises(sftp_status_exception)
+        connection.expects(:mkdir!).in_sequence(s).
+            with("~/backups").raises(sftp_status_exception)
+        connection.expects(:mkdir!).in_sequence(s).
+            with("~/backups/some_other_folder")
+        connection.expects(:mkdir!).in_sequence(s).
+            with("~/backups/some_other_folder/another_folder")
+        connection.expects(:mkdir!).in_sequence(s).
+            with("~/backups/some_other_folder/another_folder/myapp")
+        connection.expects(:mkdir!).in_sequence(s).
+            with("~/backups/some_other_folder/another_folder/myapp/#{ Backup::TIME }")
+
+        expect do
+          sftp.send(:create_remote_directories, connection)
+        end.not_to raise_error
+      end
     end
   end