backup-agoddard 3.0.27

data/spec-live/storage/dropbox_spec.rb

@@ -0,0 +1,151 @@
+# encoding: utf-8
+
+require File.expand_path('../../spec_helper.rb', __FILE__)
+
+describe 'Storage::Dropbox',
+    :if => Backup::SpecLive::CONFIG['storage']['dropbox']['specs_enabled'] do
+  let(:trigger) { 'archive_dropbox' }
+
+  def remote_files_for(storage, package)
+    remote_path = storage.send(:remote_path_for, package)
+
+    files = []
+    storage.send(:transferred_files_for, package) do |local_file, remote_file|
+      files << File.join(remote_path, remote_file)
+    end
+    files
+  end
+
+  def check_remote_for(storage, package, expectation = true)
+    remote_path = storage.send(:remote_path_for, package)
+
+    # search the remote_path folder for the trigger (base file name)
+    metadata = storage.send(:connection).search(
+      remote_path, package.trigger
+    )
+    files_found = metadata.map {|entry| File.basename(entry['path']) }
+
+    files = remote_files_for(storage, package).map {|file| File.basename(file) }
+
+    if expectation
+      files_found.sort.should == files.sort
+    else
+      files_found.should be_empty
+    end
+  end
+
+  def clean_remote!(storage, package)
+    storage.send(:remove!, package)
+  end
+
+  it 'should store the archive on the remote', :init => true do
+    model = h_set_trigger(trigger)
+
+    model.perform!
+
+    storage = model.storages.first
+    package = model.package
+    files = remote_files_for(storage, package)
+    files.count.should == 1
+
+    check_remote_for(storage, package)
+
+    clean_remote!(storage, package)
+  end
+
+  describe 'Storage::Dropbox Cycling' do
+    context 'when archives exceed `keep` setting' do
+      it 'should remove the oldest archive' do
+        packages = []
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+        sleep 1
+
+        check_remote_for(storage, packages[0])
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+        sleep 1
+
+        check_remote_for(storage, packages[1])
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+
+        check_remote_for(storage, packages[2])
+        clean_remote!(storage, packages[2])
+
+        check_remote_for(storage, packages[1])
+        clean_remote!(storage, packages[1])
+
+        check_remote_for(storage, packages[0], false)
+      end
+    end
+
+    context 'when an archive to be removed does not exist' do
+      it 'should log a warning and continue' do
+        packages = []
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+        sleep 1
+
+        check_remote_for(storage, packages[0])
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+
+        check_remote_for(storage, packages[1])
+
+        # remove archive directory cycle! will attempt to remove
+        clean_remote!(storage, packages[0])
+
+        check_remote_for(storage, packages[0], false)
+
+        check_remote_for(storage, packages[1])
+
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        expect do
+          model.perform!
+        end.not_to raise_error
+
+        Backup::Logger.has_warnings?.should be_true
+
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+
+        check_remote_for(storage, packages[1])
+        clean_remote!(storage, packages[1])
+
+        check_remote_for(storage, packages[2])
+        clean_remote!(storage, packages[2])
+      end
+    end
+
+  end # describe 'Storage::SCP Cycling'
+
+end

data/spec-live/storage/local_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+
+require File.expand_path('../../spec_helper.rb', __FILE__)
+
+describe 'Storage::Local' do
+  let(:trigger) { 'archive_local' }
+
+  def archive_file_for(model)
+    File.join(
+      Backup::SpecLive::TMP_PATH,
+      "#{model.trigger}", model.time, "#{model.trigger}.tar"
+    )
+  end
+
+  it 'should store a local archive' do
+    model = h_set_trigger(trigger)
+    model.perform!
+    File.exist?(archive_file_for(model)).should be_true
+  end
+
+  describe 'Storage::Local Cycling' do
+
+    context 'when archives exceed `keep` setting' do
+      it 'should remove the oldest archive' do
+        archives = []
+
+        model = h_set_trigger(trigger)
+        model.perform!
+        archives << archive_file_for(model)
+        sleep 1
+
+        model = h_set_trigger(trigger)
+        model.perform!
+        archives << archive_file_for(model)
+        sleep 1
+
+        model = h_set_trigger(trigger)
+        model.perform!
+        archives << archive_file_for(model)
+
+        File.exist?(archives[0]).should be_false
+        File.exist?(archives[1]).should be_true
+        File.exist?(archives[2]).should be_true
+      end
+    end
+
+    context 'when an archive to be removed does not exist' do
+      it 'should log a warning and continue' do
+        archives = []
+
+        model = h_set_trigger(trigger)
+        model.perform!
+        archives << archive_file_for(model)
+        sleep 1
+
+        model = h_set_trigger(trigger)
+        model.perform!
+        archives << archive_file_for(model)
+        sleep 1
+
+        File.exist?(archives[0]).should be_true
+        File.exist?(archives[1]).should be_true
+        # remove archive directory cycle! will attempt to remove
+        dir = archives[0].split('/')[0...-1].join('/')
+        h_safety_check(dir)
+        FileUtils.rm_r(dir)
+        File.exist?(archives[0]).should be_false
+
+        expect do
+          model = h_set_trigger(trigger)
+          model.perform!
+          archives << archive_file_for(model)
+        end.not_to raise_error
+
+        Backup::Logger.has_warnings?.should be_true
+
+        File.exist?(archives[1]).should be_true
+        File.exist?(archives[2]).should be_true
+      end
+    end
+
+  end # describe 'Storage::Local Cycling'
+end

data/spec-live/storage/scp_spec.rb

@@ -0,0 +1,193 @@
+# encoding: utf-8
+
+require File.expand_path('../../spec_helper.rb', __FILE__)
+
+describe 'Storage::SCP',
+    :if => Backup::SpecLive::CONFIG['storage']['scp']['specs_enabled'] do
+  let(:trigger) { 'archive_scp' }
+
+  def remote_files_for(storage, package)
+    remote_path = storage.send(:remote_path_for, package)
+
+    files = []
+    storage.send(:transferred_files_for, package) do |local_file, remote_file|
+      files << File.join(remote_path, remote_file)
+    end
+    files
+  end
+
+  def check_remote_for(storage, files)
+    if (storage.username == Backup::Config.user) &&
+        (storage.ip == 'localhost')
+      files.each do |file|
+        if !File.exist?(file)
+          return false
+        end
+      end
+      true
+    else
+      errors = []
+      storage.send(:connection) do |ssh|
+        files.each do |file|
+          ssh.exec!("ls '#{file}'") do |ch, stream, data|
+            errors << data if stream == :stderr
+          end
+        end
+      end
+      errors.empty?
+    end
+  end
+
+  def clean_remote!(storage, package)
+    return if (storage.username == Backup::Config.user) &&
+        (storage.ip == 'localhost') &&
+        (storage.path == Backup::SpecLive::TMP_PATH)
+
+    remote_path = storage.send(:remote_path_for, package)
+    h_safety_check(remote_path)
+    storage.send(:connection) do |ssh|
+      ssh.exec!("rm -r '#{remote_path}'")
+    end
+  end
+
+  it 'should store the archive on the remote' do
+    model = h_set_trigger(trigger)
+
+    model.perform!
+
+    storage = model.storages.first
+    package = model.package
+    files = remote_files_for(storage, package)
+    files.count.should == 1
+
+    check_remote_for(storage, files).should be_true
+
+    clean_remote!(storage, package)
+  end
+
+  describe 'Storage::SCP Cycling' do
+    context 'when archives exceed `keep` setting' do
+      it 'should remove the oldest archive' do
+        packages = []
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+        sleep 1
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[0])
+        ).should be_true
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+        sleep 1
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[1])
+        ).should be_true
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[2])
+        ).should be_true
+        clean_remote!(storage, packages[2])
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[1])
+        ).should be_true
+        clean_remote!(storage, packages[1])
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[0])
+        ).should be_false
+      end
+    end
+
+    context 'when an archive to be removed does not exist' do
+      it 'should log a warning and continue' do
+        packages = []
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+        sleep 1
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[0])
+        ).should be_true
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        model.perform!
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[1])
+        ).should be_true
+
+        # remove archive directory cycle! will attempt to remove
+
+        if (storage.username == Backup::Config.user) &&
+            (storage.ip == 'localhost')
+          remote_path = storage.send(:remote_path_for, packages[0])
+          h_safety_check(remote_path)
+          FileUtils.rm_r(remote_path)
+        else
+          clean_remote!(storage, packages[0])
+        end
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[0])
+        ).should be_false
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[1])
+        ).should be_true
+
+
+        model = h_set_trigger(trigger)
+        storage = model.storages.first
+        expect do
+          model.perform!
+        end.not_to raise_error
+
+        Backup::Logger.has_warnings?.should be_true
+
+        package = model.package
+        package.filenames.count.should == 1
+        packages << package
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[1])
+        ).should be_true
+        clean_remote!(storage, packages[1])
+
+        check_remote_for(
+          storage, remote_files_for(storage, packages[2])
+        ).should be_true
+        clean_remote!(storage, packages[2])
+      end
+    end
+
+  end # describe 'Storage::SCP Cycling'
+
+end

data/spec-live/syncer/cloud/s3_spec.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+
+require File.expand_path('../../../spec_helper.rb', __FILE__)
+
+describe 'Syncer::Cloud::S3 - No Concurrency',
+    :if => Backup::SpecLive::CONFIG['syncer']['cloud']['s3']['specs_enabled'] do
+  let(:trigger) { 'syncer_cloud_s3' }
+  let(:model) { h_set_trigger(trigger) }
+
+  before do
+    model # trigger model initialization so Fog is available
+    create_sync_files
+    clean_remote
+  end
+
+  after do
+    clean_sync_dir
+    clean_remote
+  end
+
+  it 'should work' do
+    model.perform!
+    remote_files.map {|file| [file.key, file.etag] }.sort.should == [
+      ["backups/dir_a/one.file",          "d3b07384d113edec49eaa6238ad5ff00"],
+      ["backups/dir_b/dir_c/three.file",  "d3b07384d113edec49eaa6238ad5ff00"],
+      ["backups/dir_b/two.file",          "d3b07384d113edec49eaa6238ad5ff00"]
+    ]
+
+    update_sync_files
+
+    model.perform!
+    remote_files.map {|file| [file.key, file.etag] }.sort.should == [
+      ["backups/dir_a/dir_d/two.new",     "14758f1afd44c09b7992073ccf00b43d"],
+      ["backups/dir_a/one.file",          "14758f1afd44c09b7992073ccf00b43d"],
+      ["backups/dir_b/dir_c/three.file",  "d3b07384d113edec49eaa6238ad5ff00"],
+      ["backups/dir_b/one.new",           "14758f1afd44c09b7992073ccf00b43d"]
+    ]
+  end
+
+  private
+
+  ##
+  # Initial Files are MD5: d3b07384d113edec49eaa6238ad5ff00
+  #
+  # ├── dir_a
+  # │   └── one.file
+  # └── dir_b
+  #     ├── dir_c
+  #     │   └── three.file
+  #     ├── bad\xFFfile
+  #     └── two.file
+  def create_sync_files
+    clean_sync_dir
+
+    %w{ dir_a dir_b/dir_c }.each do |dir|
+      path = File.join(Backup::SpecLive::SYNC_PATH, dir)
+      FileUtils.mkdir_p(path)
+    end
+
+    %W{ dir_a/one.file
+        dir_b/two.file
+        dir_b/bad\xFFfile
+        dir_b/dir_c/three.file }.each do |file|
+      path = File.join(Backup::SpecLive::SYNC_PATH, file)
+      File.open(path, 'w') {|file| file.puts 'foo' }
+    end
+  end
+
+  ##
+  # Added/Updated Files are MD5: 14758f1afd44c09b7992073ccf00b43d
+  #
+  # ├── dir_a
+  # │   ├── dir_d           (add)
+  # │   │   └── two.new     (add)
+  # │   └── one.file        (update)
+  # └── dir_b
+  #     ├── dir_c
+  #     │   └── three.file
+  #     ├── bad\377file
+  #     ├── one.new         (add)
+  #     └── two.file        (remove)
+  def update_sync_files
+    FileUtils.mkdir_p(File.join(Backup::SpecLive::SYNC_PATH, 'dir_a/dir_d'))
+    %w{ dir_a/one.file
+        dir_b/one.new
+        dir_a/dir_d/two.new }.each do |file|
+      path = File.join(Backup::SpecLive::SYNC_PATH, file)
+      File.open(path, 'w') {|file| file.puts 'foobar' }
+    end
+
+    path = File.join(Backup::SpecLive::SYNC_PATH, 'dir_b/two.file')
+    h_safety_check(path)
+    FileUtils.rm(path)
+  end
+
+  def clean_sync_dir
+    path = Backup::SpecLive::SYNC_PATH
+    if File.directory?(path)
+      h_safety_check(path)
+      FileUtils.rm_r(path)
+    end
+  end
+
+  # use a new connection for each request
+  def connection
+    @opts = Backup::SpecLive::CONFIG['syncer']['cloud']['s3']
+    Fog::Storage.new(
+      :provider              => 'AWS',
+      :aws_access_key_id     => @opts['access_key_id'],
+      :aws_secret_access_key => @opts['secret_access_key'],
+      :region                => @opts['region']
+    )
+  end
+
+  def remote_files
+    bucket = connection.directories.get(@opts['bucket'])
+    bucket.files.all(:prefix => 'backups')
+  end
+
+  def clean_remote
+    remote_files.each {|file| file.destroy }
+  end
+
+end