backup 3.0.23 → 3.0.24

data/spec-live/syncer/cloud/s3_spec.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+
+require File.expand_path('../../../spec_helper.rb', __FILE__)
+
+describe 'Syncer::Cloud::S3 - No Concurrency',
+    :if => Backup::SpecLive::CONFIG['syncer']['cloud']['s3']['specs_enabled'] do
+  let(:trigger) { 'syncer_cloud_s3' }
+  let(:model) { h_set_trigger(trigger) }
+
+  before do
+    model # trigger model initialization so Fog is available
+    create_sync_files
+    clean_remote
+  end
+
+  after do
+    clean_sync_dir
+    clean_remote
+  end
+
+  it 'should work' do
+    model.perform!
+    remote_files.map {|file| [file.key, file.etag] }.sort.should == [
+      ["backups/dir_a/one.file",          "d3b07384d113edec49eaa6238ad5ff00"],
+      ["backups/dir_b/dir_c/three.file",  "d3b07384d113edec49eaa6238ad5ff00"],
+      ["backups/dir_b/two.file",          "d3b07384d113edec49eaa6238ad5ff00"]
+    ]
+
+    update_sync_files
+
+    model.perform!
+    remote_files.map {|file| [file.key, file.etag] }.sort.should == [
+      ["backups/dir_a/dir_d/two.new",     "14758f1afd44c09b7992073ccf00b43d"],
+      ["backups/dir_a/one.file",          "14758f1afd44c09b7992073ccf00b43d"],
+      ["backups/dir_b/dir_c/three.file",  "d3b07384d113edec49eaa6238ad5ff00"],
+      ["backups/dir_b/one.new",           "14758f1afd44c09b7992073ccf00b43d"]
+    ]
+  end
+
+  private
+
+  ##
+  # Initial Files are MD5: d3b07384d113edec49eaa6238ad5ff00
+  #
+  # ├── dir_a
+  # │   └── one.file
+  # └── dir_b
+  #     ├── dir_c
+  #     │   └── three.file
+  #     ├── bad\xFFfile
+  #     └── two.file
+  def create_sync_files
+    clean_sync_dir
+
+    %w{ dir_a dir_b/dir_c }.each do |dir|
+      path = File.join(Backup::SpecLive::SYNC_PATH, dir)
+      FileUtils.mkdir_p(path)
+    end
+
+    %W{ dir_a/one.file
+        dir_b/two.file
+        dir_b/bad\xFFfile
+        dir_b/dir_c/three.file }.each do |file|
+      path = File.join(Backup::SpecLive::SYNC_PATH, file)
+      File.open(path, 'w') {|file| file.puts 'foo' }
+    end
+  end
+
+  ##
+  # Added/Updated Files are MD5: 14758f1afd44c09b7992073ccf00b43d
+  #
+  # ├── dir_a
+  # │   ├── dir_d           (add)
+  # │   │   └── two.new     (add)
+  # │   └── one.file        (update)
+  # └── dir_b
+  #     ├── dir_c
+  #     │   └── three.file
+  #     ├── bad\377file
+  #     ├── one.new         (add)
+  #     └── two.file        (remove)
+  def update_sync_files
+    FileUtils.mkdir_p(File.join(Backup::SpecLive::SYNC_PATH, 'dir_a/dir_d'))
+    %w{ dir_a/one.file
+        dir_b/one.new
+        dir_a/dir_d/two.new }.each do |file|
+      path = File.join(Backup::SpecLive::SYNC_PATH, file)
+      File.open(path, 'w') {|file| file.puts 'foobar' }
+    end
+
+    path = File.join(Backup::SpecLive::SYNC_PATH, 'dir_b/two.file')
+    h_safety_check(path)
+    FileUtils.rm(path)
+  end
+
+  def clean_sync_dir
+    path = Backup::SpecLive::SYNC_PATH
+    if File.directory?(path)
+      h_safety_check(path)
+      FileUtils.rm_r(path)
+    end
+  end
+
+  # use a new connection for each request
+  def connection
+    @opts = Backup::SpecLive::CONFIG['syncer']['cloud']['s3']
+    Fog::Storage.new(
+      :provider              => 'AWS',
+      :aws_access_key_id     => @opts['access_key_id'],
+      :aws_secret_access_key => @opts['secret_access_key'],
+      :region                => @opts['region']
+    )
+  end
+
+  def remote_files
+    bucket = connection.directories.get(@opts['bucket'])
+    bucket.files.all(:prefix => 'backups')
+  end
+
+  def clean_remote
+    remote_files.each {|file| file.destroy }
+  end
+
+end

data/spec/archive_spec.rb

@@ -134,30 +134,41 @@ describe Backup::Archive do
     end
     let(:paths) { ['/path/to/add', '/another/path/to/add'] }
     let(:excludes) { ['/path/to/exclude', '/another/path/to/exclude'] }
+    let(:pipeline) { mock }
+    let(:s) { sequence '' }
 
     before do
+      archive.instance_variable_set(:@paths, paths)
       archive.expects(:utility).with(:tar).returns('tar')
       FileUtils.expects(:mkdir_p).with(archive_path)
+      Backup::Pipeline.expects(:new).returns(pipeline)
     end
 
     context 'when both #paths and #excludes were added' do
       before do
-        archive.instance_variable_set(:@paths, paths)
         archive.instance_variable_set(:@excludes, excludes)
       end
 
       it 'should render the syntax for both' do
-        Backup::Logger.expects(:message).with(
-          "Backup::Archive started packaging and archiving:\n" +
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive has started archiving:\n" +
           "  /path/to/add\n" +
           "  /another/path/to/add"
         )
 
-        archive.expects(:run).with(
-          "tar  -cf - " +
+        pipeline.expects(:<<).in_sequence(s).with(
+          "tar  -cPf - " +
           "--exclude='/path/to/exclude' --exclude='/another/path/to/exclude' " +
-          "'/path/to/add' '/another/path/to/add' " +
-          "> '#{ File.join(archive_path, 'test_archive.tar') }'"
+          "'/path/to/add' '/another/path/to/add'"
+        )
+        pipeline.expects(:<<).in_sequence(s).with(
+          "cat > '#{ File.join(archive_path, 'test_archive.tar') }'"
+        )
+        pipeline.expects(:run).in_sequence(s)
+        pipeline.expects(:success?).in_sequence(s).returns(true)
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive Complete!"
         )
 
         archive.perform!
@@ -165,45 +176,56 @@ describe Backup::Archive do
     end # context 'when both #paths and #excludes were added'
 
     context 'when no excludes were added' do
-      before do
-        archive.instance_variable_set(:@paths, paths)
-      end
-
       it 'should render only the syntax for adds' do
-        Backup::Logger.expects(:message).with(
-          "Backup::Archive started packaging and archiving:\n" +
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive has started archiving:\n" +
           "  /path/to/add\n" +
           "  /another/path/to/add"
         )
 
-        archive.expects(:run).with(
-          "tar  -cf -  " +
-          "'/path/to/add' '/another/path/to/add' " +
-          "> '#{ File.join(archive_path, 'test_archive.tar') }'"
+        pipeline.expects(:<<).in_sequence(s).with(
+          "tar  -cPf -  '/path/to/add' '/another/path/to/add'"
+        )
+        pipeline.expects(:<<).in_sequence(s).with(
+          "cat > '#{ File.join(archive_path, 'test_archive.tar') }'"
+        )
+        pipeline.expects(:run).in_sequence(s)
+        pipeline.expects(:success?).in_sequence(s).returns(true)
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive Complete!"
         )
 
         archive.perform!
       end
     end # context 'when no excludes were added'
+
     context 'with #paths, #excludes and #tar_args' do
       before do
-        archive.instance_variable_set(:@paths, paths)
         archive.instance_variable_set(:@excludes, excludes)
         archive.instance_variable_set(:@tar_args, '-h --xattrs')
       end
 
       it 'should render the syntax for all three' do
-        Backup::Logger.expects(:message).with(
-          "Backup::Archive started packaging and archiving:\n" +
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive has started archiving:\n" +
           "  /path/to/add\n" +
           "  /another/path/to/add"
         )
 
-        archive.expects(:run).with(
-          "tar -h --xattrs -cf - " +
+        pipeline.expects(:<<).in_sequence(s).with(
+          "tar -h --xattrs -cPf - " +
           "--exclude='/path/to/exclude' --exclude='/another/path/to/exclude' " +
-          "'/path/to/add' '/another/path/to/add' " +
-          "> '#{ File.join(archive_path, 'test_archive.tar') }'"
+          "'/path/to/add' '/another/path/to/add'"
+        )
+        pipeline.expects(:<<).in_sequence(s).with(
+          "cat > '#{ File.join(archive_path, 'test_archive.tar') }'"
+        )
+        pipeline.expects(:run).in_sequence(s)
+        pipeline.expects(:success?).in_sequence(s).returns(true)
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive Complete!"
         )
 
         archive.perform!
@@ -212,7 +234,6 @@ describe Backup::Archive do
 
     context 'with #paths, #excludes, #tar_args and a Gzip Compressor' do
       before do
-        archive.instance_variable_set(:@paths, paths)
         archive.instance_variable_set(:@excludes, excludes)
         archive.instance_variable_set(:@tar_args, '-h --xattrs')
         compressor = mock
@@ -221,23 +242,57 @@ describe Backup::Archive do
       end
 
       it 'should render the syntax with compressor modifications' do
-        Backup::Logger.expects(:message).with(
-          "Backup::Archive started packaging and archiving:\n" +
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive has started archiving:\n" +
           "  /path/to/add\n" +
           "  /another/path/to/add"
         )
 
-        archive.expects(:run).with(
-          "tar -h --xattrs -cf - " +
+        pipeline.expects(:<<).in_sequence(s).with(
+          "tar -h --xattrs -cPf - " +
           "--exclude='/path/to/exclude' --exclude='/another/path/to/exclude' " +
-          "'/path/to/add' '/another/path/to/add' " +
-          "| gzip > '#{ File.join(archive_path, 'test_archive.tar.gz') }'"
+          "'/path/to/add' '/another/path/to/add'"
+        )
+        pipeline.expects(:<<).in_sequence(s).with('gzip')
+        pipeline.expects(:<<).in_sequence(s).with(
+          "cat > '#{ File.join(archive_path, 'test_archive.tar.gz') }'"
+        )
+        pipeline.expects(:run).in_sequence(s)
+        pipeline.expects(:success?).in_sequence(s).returns(true)
+
+        Backup::Logger.expects(:message).in_sequence(s).with(
+          "Backup::Archive Complete!"
         )
 
         archive.perform!
       end
     end # context 'with #paths, #excludes, #tar_args and a Gzip Compressor'
 
+    context 'when pipeline command fails' do
+      before do
+        pipeline.stubs(:<<)
+        pipeline.expects(:run)
+        pipeline.expects(:success?).returns(false)
+        pipeline.expects(:error_messages).returns('pipeline_errors')
+      end
+
+      it 'should raise an error' do
+        Backup::Logger.expects(:message).with(
+          "Backup::Archive has started archiving:\n" +
+          "  /path/to/add\n" +
+          "  /another/path/to/add"
+        )
+
+        expect do
+          archive.perform!
+        end.to raise_error(
+          Backup::Errors::Archive::PipelineError,
+          "Archive::PipelineError: Failed to Create Backup Archive\n" +
+          "  pipeline_errors"
+        )
+      end
+    end # context 'when pipeline command fails'
+
   end # describe '#perform!'
 
   describe '#paths_to_package' do

data/spec/cleaner_spec.rb

@@ -221,6 +221,10 @@ describe 'Backup::Cleaner' do
       FileUtils.unstub(:mkdir_p)
     end
 
+    after do
+      Backup::Config.send(:reset!)
+    end
+
     context 'when files exist in the packaging folder' do
       it 'should return true' do
         Dir.mktmpdir do |path|
@@ -253,6 +257,10 @@ describe 'Backup::Cleaner' do
       FileUtils.unstub(:touch)
     end
 
+    after do
+      Backup::Config.send(:reset!)
+    end
+
     context 'when packaging files exist in the tmp_path' do
       it 'should return the files' do
         Dir.mktmpdir do |path|

data/spec/cli/helpers_spec.rb

@@ -6,55 +6,190 @@ describe Backup::CLI::Helpers do
   let(:helpers) { Module.new.extend(Backup::CLI::Helpers) }
 
   describe '#run' do
-    let(:stdin)   { mock }
-    let(:stdout)  { mock }
-    let(:stderr)  { mock }
-    let(:process_status) { mock }
-
-    it 'should run the given command using POpen4' do
-      Open4.expects(:popen4).with('/path/to/command args').
-        returns([123, stdin, stdout, stderr])
-      Process.expects(:waitpid2).with(123).returns([123, process_status])
-      stdout.expects(:read).returns('stdout message')
-      stderr.expects(:read).returns('stderr message')
-
-      helpers.expects(:command_name).with('/path/to/command args').
-          returns('command')
-      helpers.expects(:raise_if_command_failed!).with(
-        'command',
-        {:status => process_status,
-         :stdout => 'stdout message',
-         :stderr => 'stderr message',
-         :ignore_exit_codes => [0]}
-      )
+    let(:stdout_io) { stub(:read => stdout_messages) }
+    let(:stderr_io) { stub(:read => stderr_messages) }
+    let(:stdin_io)  { stub(:close) }
+    let(:process_status) { stub(:success? => process_success) }
+    let(:command) { '/path/to/cmd_name arg1 arg2' }
 
-      helpers.run('/path/to/command args').should == 'stdout message'
-    end
+    context 'when the command is successful' do
+      let(:process_success) { true }
 
-    it 'should accept ignore_exit_codes and add 0 to the list' do
-      Open4.expects(:popen4).with('/path/to/command args').
-        returns([123, stdin, stdout, stderr])
-      Process.expects(:waitpid2).with(123).returns([123, process_status])
-      stdout.expects(:read).returns('stdout message')
-      stderr.expects(:read).returns('stderr message')
-
-      helpers.expects(:command_name).with('/path/to/command args').
-          returns('command')
-      helpers.expects(:raise_if_command_failed!).with(
-        'command',
-        {:status => process_status,
-         :stdout => 'stdout message',
-         :stderr => 'stderr message',
-         :ignore_exit_codes => [1, 2, 0]}
-      )
+      before do
+        Backup::Logger.expects(:message).with(
+          "Running system utility 'cmd_name'..."
+        )
 
-      helpers.run(
-        '/path/to/command args', :ignore_exit_codes => [1, 2]
-      ).should == 'stdout message'
-    end
-  end
+        Open4.expects(:popen4).with(command).yields(
+          nil, stdin_io, stdout_io, stderr_io
+        ).returns(process_status)
+      end
+
+      context 'and generates no messages' do
+        let(:stdout_messages) { '' }
+        let(:stderr_messages) { '' }
+
+        it 'should generate no additional log messages' do
+          helpers.run(command).should be_nil
+        end
+      end
+
+      context 'and generates only stdout messages' do
+        let(:stdout_messages) { "out line1\nout line2\n" }
+        let(:stderr_messages) { '' }
+
+        it 'should log the stdout messages' do
+          Backup::Logger.expects(:message).with(
+            "cmd_name:STDOUT: out line1\ncmd_name:STDOUT: out line2"
+          )
+          helpers.run(command).should be_nil
+        end
+      end
+
+      context 'and generates only stderr messages' do
+        let(:stdout_messages) { '' }
+        let(:stderr_messages) { "err line1\nerr line2\n" }
+
+        it 'should log the stderr messages' do
+          Backup::Logger.expects(:warn).with(
+            "cmd_name:STDERR: err line1\ncmd_name:STDERR: err line2"
+          )
+          helpers.run(command).should be_nil
+        end
+      end
+
+      context 'and generates messages on both stdout and stderr' do
+        let(:stdout_messages) { "out line1\nout line2\n" }
+        let(:stderr_messages) { "err line1\nerr line2\n" }
+
+        it 'should log both stdout and stderr messages' do
+          Backup::Logger.expects(:message).with(
+            "cmd_name:STDOUT: out line1\ncmd_name:STDOUT: out line2"
+          )
+          Backup::Logger.expects(:warn).with(
+            "cmd_name:STDERR: err line1\ncmd_name:STDERR: err line2"
+          )
+          helpers.run(command).should be_nil
+        end
+      end
+    end # context 'when the command is successful'
+
+    context 'when the command is not successful' do
+      let(:process_success) { false }
+      let(:message_head) do
+        "CLI::SystemCallError: 'cmd_name' Failed on #{ RUBY_PLATFORM }\n" +
+        "  The following information should help to determine the problem:\n" +
+        "  Command was: /path/to/cmd_name arg1 arg2\n" +
+        "  Exit Status: 1\n"
+      end
+
+      before do
+        Backup::Logger.expects(:message).with(
+          "Running system utility 'cmd_name'..."
+        )
+
+        Open4.expects(:popen4).with(command).yields(
+          nil, stdin_io, stdout_io, stderr_io
+        ).returns(process_status)
+
+        process_status.stubs(:exitstatus).returns(1)
+      end
+
+      context 'and generates no messages' do
+        let(:stdout_messages) { '' }
+        let(:stderr_messages) { '' }
+
+        it 'should raise an error reporting no messages' do
+          expect do
+            helpers.run(command)
+          end.to raise_error {|err|
+            err.message.should == message_head +
+              "  STDOUT Messages: None\n" +
+              "  STDERR Messages: None"
+          }
+        end
+      end
+
+      context 'and generates only stdout messages' do
+        let(:stdout_messages) { "out line1\nout line2\n" }
+        let(:stderr_messages) { '' }
+
+        it 'should raise an error and report the stdout messages' do
+          expect do
+            helpers.run(command)
+          end.to raise_error {|err|
+            err.message.should == message_head +
+              "  STDOUT Messages: \n" +
+              "  out line1\n" +
+              "  out line2\n" +
+              "  STDERR Messages: None"
+          }
+        end
+      end
+
+      context 'and generates only stderr messages' do
+        let(:stdout_messages) { '' }
+        let(:stderr_messages) { "err line1\nerr line2\n" }
+
+        it 'should raise an error and report the stderr messages' do
+          expect do
+            helpers.run(command)
+          end.to raise_error {|err|
+            err.message.should == message_head +
+              "  STDOUT Messages: None\n" +
+              "  STDERR Messages: \n" +
+              "  err line1\n" +
+              "  err line2"
+          }
+        end
+      end
+
+      context 'and generates messages on both stdout and stderr' do
+        let(:stdout_messages) { "out line1\nout line2\n" }
+        let(:stderr_messages) { "err line1\nerr line2\n" }
+
+        it 'should raise an error and report the stdout and stderr messages' do
+          expect do
+            helpers.run(command)
+          end.to raise_error {|err|
+            err.message.should == message_head +
+              "  STDOUT Messages: \n" +
+              "  out line1\n" +
+              "  out line2\n" +
+              "  STDERR Messages: \n" +
+              "  err line1\n" +
+              "  err line2"
+          }
+        end
+      end
+    end # context 'when the command is not successful'
+
+    context 'when the system fails to execute the command' do
+      before do
+        Backup::Logger.expects(:message).with(
+          "Running system utility 'cmd_name'..."
+        )
+
+        Open4.expects(:popen4).raises("exec call failed")
+      end
+
+      it 'should raise an error wrapping the system error raised' do
+        expect do
+          helpers.run(command)
+        end.to raise_error {|err|
+          err.message.should == "CLI::SystemCallError: " +
+            "Failed to execute system command on #{ RUBY_PLATFORM }\n" +
+            "  Command was: /path/to/cmd_name arg1 arg2\n" +
+            "  Reason: RuntimeError\n" +
+            "  exec call failed"
+        }
+      end
+    end # context 'when the system fails to execute the command'
+  end # describe '#run'
 
   describe '#utility' do
+    after { Backup::CLI::Helpers::UTILITY.clear }
+
     context 'when a system path for the utility is available' do
       it 'should return the system path with newline removed' do
         helpers.expects(:`).with('which foo 2>/dev/null').returns("system_path\n")
@@ -79,7 +214,6 @@ describe Backup::CLI::Helpers do
       end
     end
 
-
     context 'when a system path for the utility is not available' do
       it 'should raise an error' do
         helpers.expects(:`).with('which unknown 2>/dev/null').returns("\n")
@@ -87,7 +221,7 @@ describe Backup::CLI::Helpers do
         expect do
           helpers.utility(:unknown)
         end.to raise_error(Backup::Errors::CLI::UtilityNotFoundError) {|err|
-          err.message.should match(/Path to 'unknown' could not be found/)
+          err.message.should match(/Could not locate 'unknown'/)
         }
       end
 
@@ -97,16 +231,34 @@ describe Backup::CLI::Helpers do
         expect do
           helpers.utility(:not_cached)
         end.to raise_error(Backup::Errors::CLI::UtilityNotFoundError) {|err|
-          err.message.should match(/Path to 'not_cached' could not be found/)
+          err.message.should match(/Could not locate 'not_cached'/)
         }
 
         expect do
           helpers.utility(:not_cached)
         end.to raise_error(Backup::Errors::CLI::UtilityNotFoundError) {|err|
-          err.message.should match(/Path to 'not_cached' could not be found/)
+          err.message.should match(/Could not locate 'not_cached'/)
         }
       end
     end
+
+    it 'should raise an error if name is nil' do
+      expect do
+        helpers.utility(nil)
+      end.to raise_error(
+        Backup::Errors::CLI::UtilityNotFoundError,
+          'CLI::UtilityNotFoundError: Utility Name Empty'
+      )
+    end
+
+    it 'should raise an error if name is empty' do
+      expect do
+        helpers.utility(' ')
+      end.to raise_error(
+        Backup::Errors::CLI::UtilityNotFoundError,
+          'CLI::UtilityNotFoundError: Utility Name Empty'
+      )
+    end
   end # describe '#utility'
 
   describe '#command_name' do
@@ -146,31 +298,4 @@ describe Backup::CLI::Helpers do
     end
   end # describe '#command_name'
 
-  describe '#raise_if_command_failed!' do
-
-    it 'returns nil if status exit code is in ignore_exit_codes' do
-      process_data = { :status => '3', :ignore_exit_codes => [1,3,5] }
-      helpers.raise_if_command_failed!('foo', process_data).should be_nil
-    end
-
-    it 'raises an error with stdout/stderr data' do
-      process_data = { :status => '3', :ignore_exit_codes => [2,4,6],
-                       :stdout => 'stdout data', :stderr => 'stderr data' }
-
-      expect do
-        helpers.raise_if_command_failed!('utility_name', process_data)
-      end.to raise_error(
-        Backup::Errors::CLI::SystemCallError,
-        "CLI::SystemCallError: Failed to run utility_name on #{RUBY_PLATFORM}\n" +
-        "  The following information should help to determine the problem:\n" +
-        "  Exit Code: 3\n" +
-        "  STDERR:\n" +
-        "  stderr data\n" +
-        "  STDOUT:\n" +
-        "  stdout data"
-      )
-    end
-
-  end
-
 end