backs3 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +3 -0
- data/Manifest.txt +21 -0
- data/README.rdoc +51 -0
- data/Rakefile +26 -0
- data/bin/backs3 +100 -0
- data/bin/res3 +90 -0
- data/example.conf +5 -0
- data/lib/backs3.rb +5 -0
- data/lib/backs3/backs3.rb +56 -0
- data/lib/backs3/backup.rb +111 -0
- data/lib/backs3/file_info.rb +51 -0
- data/lib/backs3/restore.rb +70 -0
- data/lib/backs3/version.rb +9 -0
- data/spec/backs3/backup_spec.rb +132 -0
- data/spec/backs3/file_info_spec.rb +60 -0
- data/spec/backs3/restore_spec.rb +82 -0
- data/spec/spec.opts +2 -0
- data/spec/spec_helper.rb +19 -0
- data/tasks/rspec.rake +21 -0
- metadata +106 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
README.rdoc
|
4
|
+
Rakefile
|
5
|
+
example.conf
|
6
|
+
bin/backs3
|
7
|
+
bin/res3
|
8
|
+
lib/backs3.rb
|
9
|
+
lib/backs3/backs3.rb
|
10
|
+
lib/backs3/backup.rb
|
11
|
+
lib/backs3/restore.rb
|
12
|
+
lib/backs3/version.rb
|
13
|
+
lib/backs3/backup.rb
|
14
|
+
lib/backs3/file_info.rb
|
15
|
+
spec/spec.opts
|
16
|
+
spec/spec_helper.rb
|
17
|
+
spec/backs3/backup_spec.rb
|
18
|
+
spec/backs3/restore_spec.rb
|
19
|
+
spec/backs3/backup_spec.rb
|
20
|
+
spec/backs3/file_info_spec.rb
|
21
|
+
tasks/rspec.rake
|
data/README.rdoc
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
= Backs3
|
2
|
+
|
3
|
+
* http://github.com/jemmyw/backs3
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
A simple backup and restore program for S3
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* FIX (list of features or problems)
|
12
|
+
|
13
|
+
== SYNOPSIS:
|
14
|
+
|
15
|
+
backs3 -c s3.conf test
|
16
|
+
res3 -c s3.conf available => [12345, 54321]
|
17
|
+
res3 -c s3.conf restore 12345
|
18
|
+
|
19
|
+
== REQUIREMENTS:
|
20
|
+
|
21
|
+
* ActiveSupport
|
22
|
+
* AWS/S3
|
23
|
+
|
24
|
+
== INSTALL:
|
25
|
+
|
26
|
+
* sudo gem install jemmyw-backs3
|
27
|
+
|
28
|
+
== LICENSE:
|
29
|
+
|
30
|
+
(The MIT License)
|
31
|
+
|
32
|
+
Copyright (c) 2009 Jeremy Wells / Boost Limited (http://www.boost.co.nz)
|
33
|
+
|
34
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
35
|
+
a copy of this software and associated documentation files (the
|
36
|
+
'Software'), to deal in the Software without restriction, including
|
37
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
38
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
39
|
+
permit persons to whom the Software is furnished to do so, subject to
|
40
|
+
the following conditions:
|
41
|
+
|
42
|
+
The above copyright notice and this permission notice shall be
|
43
|
+
included in all copies or substantial portions of the Software.
|
44
|
+
|
45
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
46
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
47
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
48
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
49
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
50
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
51
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
gem 'hoe', '>= 2.1.0'
|
3
|
+
require 'hoe'
|
4
|
+
require 'fileutils'
|
5
|
+
require './lib/backs3'
|
6
|
+
|
7
|
+
Hoe.plugin :newgem
|
8
|
+
# Hoe.plugin :website
|
9
|
+
# Hoe.plugin :cucumberfeatures
|
10
|
+
|
11
|
+
# Generate all the Rake tasks
|
12
|
+
# Run 'rake -T' to see list of generated tasks (from gem root directory)
|
13
|
+
$hoe = Hoe.spec 'backs3' do
|
14
|
+
self.developer 'Jeremy Wells', 'jeremy@boost.co.nz'
|
15
|
+
self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
|
16
|
+
self.description = "S3 backup and restore program"
|
17
|
+
#self.rubyforge_name = self.name # TODO this is default value
|
18
|
+
self.extra_deps = [['activesupport','>= 2.0.2'], ['aws-s3', '>= 0.5.1']]
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'newgem/tasks'
|
22
|
+
Dir['tasks/**/*.rake'].each { |t| load t }
|
23
|
+
|
24
|
+
# TODO - want other tests/tasks run by default? Add them to the list
|
25
|
+
# remove_task :default
|
26
|
+
# task :default => [:spec, :features]
|
data/bin/backs3
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'rubygems'
|
5
|
+
require 'backs3/backs3'
|
6
|
+
rescue LoadError => e
|
7
|
+
require 'lib/backs3/backs3'
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'active_support'
|
11
|
+
require 'getoptlong'
|
12
|
+
|
13
|
+
option_parser = GetoptLong.new(
|
14
|
+
['--help', '-h', GetoptLong::NO_ARGUMENT],
|
15
|
+
['--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
|
16
|
+
['--exclude', GetoptLong::REQUIRED_ARGUMENT],
|
17
|
+
['--id', '-i', GetoptLong::REQUIRED_ARGUMENT],
|
18
|
+
['--key', '-k', GetoptLong::REQUIRED_ARGUMENT],
|
19
|
+
['--bucket', '-b', GetoptLong::REQUIRED_ARGUMENT],
|
20
|
+
['--prefix', '-p', GetoptLong::REQUIRED_ARGUMENT],
|
21
|
+
['--full', '-f', GetoptLong::REQUIRED_ARGUMENT],
|
22
|
+
['--force-full', GetoptLong::NO_ARGUMENT]
|
23
|
+
)
|
24
|
+
|
25
|
+
$options = {}
|
26
|
+
|
27
|
+
def usage(message = nil)
|
28
|
+
$stderr.puts message unless message.blank?
|
29
|
+
|
30
|
+
name = $0.split('/').last
|
31
|
+
$stderr.puts <<"ENDUSAGE"
|
32
|
+
#{name} [options] <directory>
|
33
|
+
--help -h
|
34
|
+
--config -c Configuration file
|
35
|
+
--id -i AWS Access Key ID
|
36
|
+
--key -k AWS Secret Key
|
37
|
+
--bucket -b AWS Bucket name
|
38
|
+
--full=d -f d Number of days between full backups (default: 7)
|
39
|
+
--force-full Force a full backup
|
40
|
+
--prefix -p
|
41
|
+
--exclude="regex" Exclude files based on regex
|
42
|
+
|
43
|
+
ENDUSAGE
|
44
|
+
|
45
|
+
$stderr.puts "Current configuration:"
|
46
|
+
$options.each do |key, value|
|
47
|
+
$stderr.puts " #{key}: \t#{value}"
|
48
|
+
end
|
49
|
+
|
50
|
+
exit!
|
51
|
+
end #usage
|
52
|
+
|
53
|
+
begin
|
54
|
+
option_parser.each do |opt, arg|
|
55
|
+
$options[opt.gsub(/^-*/, '')] = (arg || true)
|
56
|
+
end
|
57
|
+
|
58
|
+
usage if $options['help']
|
59
|
+
$options['folder'] = ARGV[0] unless ARGV[0].blank?
|
60
|
+
|
61
|
+
raise Exception.new("Invalid configuration file #{$options['config']}") unless $options['config'].blank? || File.exists?($options['config'])
|
62
|
+
$options['config'] ||= '/etc/backs3.conf'
|
63
|
+
|
64
|
+
if File.exists?($options['config'])
|
65
|
+
begin
|
66
|
+
puts "Reading configuration from #{$options['config']}"
|
67
|
+
config = YAML::load_file($options['config'])
|
68
|
+
$options = config.merge($options)
|
69
|
+
rescue
|
70
|
+
raise Exception.new("Invalid configuration file #{$options['config']}")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
raise Exception.new("You must specify a directory to backup") if $options['folder'].blank?
|
75
|
+
raise Exception.new("You must specify a bucket") if $options['bucket'].blank?
|
76
|
+
raise Exception.new("You must specify an AWS ID") if $options['id'].blank?
|
77
|
+
raise Exception.new("You must specify an AWS Secret Key") if $options['key'].blank?
|
78
|
+
rescue Exception => e
|
79
|
+
usage(e.to_s)
|
80
|
+
end
|
81
|
+
|
82
|
+
class Backs3::BackupCmd
|
83
|
+
include Backs3
|
84
|
+
|
85
|
+
def initialize(options = {})
|
86
|
+
@options = options
|
87
|
+
@options['prefix'] ||= ''
|
88
|
+
|
89
|
+
establish_connection
|
90
|
+
|
91
|
+
@backups = load_backup_info
|
92
|
+
@backup = Backup.new(@backups, @options)
|
93
|
+
end
|
94
|
+
|
95
|
+
def backup
|
96
|
+
@backup.backup
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
Backs3::BackupCmd.new($options).backup
|
data/bin/res3
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'rubygems'
|
5
|
+
require 'backs3/restore'
|
6
|
+
rescue LoadError => e
|
7
|
+
require 'lib/backs3/restore'
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'active_support'
|
11
|
+
require 'getoptlong'
|
12
|
+
|
13
|
+
option_parser = GetoptLong.new(
|
14
|
+
['--help', '-h', GetoptLong::NO_ARGUMENT],
|
15
|
+
['--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
|
16
|
+
['--id', '-i', GetoptLong::REQUIRED_ARGUMENT],
|
17
|
+
['--key', '-k', GetoptLong::REQUIRED_ARGUMENT],
|
18
|
+
['--bucket', '-b', GetoptLong::REQUIRED_ARGUMENT],
|
19
|
+
['--prefix', '-p', GetoptLong::REQUIRED_ARGUMENT]
|
20
|
+
)
|
21
|
+
|
22
|
+
$options = {}
|
23
|
+
|
24
|
+
def usage(message = nil)
|
25
|
+
$stderr.puts message unless message.blank?
|
26
|
+
|
27
|
+
name = $0.split('/').last
|
28
|
+
$stderr.puts <<"ENDUSAGE"
|
29
|
+
#{name} [options] <directory> <command> [command options]
|
30
|
+
Commands:
|
31
|
+
#{Backs3::Restore.commands.join(", ")}
|
32
|
+
|
33
|
+
--help -h
|
34
|
+
--config -c Configuration file
|
35
|
+
--id -i AWS Access Key ID
|
36
|
+
--key -k AWS Secret Key
|
37
|
+
--bucket -b AWS Bucket name
|
38
|
+
--prefix -p
|
39
|
+
|
40
|
+
ENDUSAGE
|
41
|
+
|
42
|
+
$stderr.puts "Current configuration:"
|
43
|
+
$options.each do |key, value|
|
44
|
+
$stderr.puts " #{key}: \t#{value}"
|
45
|
+
end
|
46
|
+
|
47
|
+
exit!
|
48
|
+
end #usage
|
49
|
+
|
50
|
+
begin
|
51
|
+
option_parser.each do |opt, arg|
|
52
|
+
$options[opt.gsub(/^-*/, '')] = (arg || true)
|
53
|
+
end
|
54
|
+
|
55
|
+
usage if $options['help']
|
56
|
+
|
57
|
+
raise Exception.new("Invalid configuration file #{$options['config']}") unless $options['config'].blank? || File.exists?($options['config'])
|
58
|
+
$options['config'] ||= '/etc/backs3.conf'
|
59
|
+
|
60
|
+
if File.exists?($options['config'])
|
61
|
+
begin
|
62
|
+
puts "Reading configuration from #{$options['config']}"
|
63
|
+
config = YAML::load_file($options['config'])
|
64
|
+
$options = config.merge($options)
|
65
|
+
rescue
|
66
|
+
raise Exception.new("Invalid configuration file #{$options['config']}")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
args = ARGV
|
71
|
+
$options['folder'] = args.shift if $options['folder'].blank?
|
72
|
+
$command = args.shift
|
73
|
+
|
74
|
+
raise Exception.new("You must specify a directory to restore from") if $options['folder'].blank?
|
75
|
+
raise Exception.new("You must specify a bucket") if $options['bucket'].blank?
|
76
|
+
raise Exception.new("You must specify an AWS ID") if $options['id'].blank?
|
77
|
+
raise Exception.new("You must specify an AWS Secret Key") if $options['key'].blank?
|
78
|
+
raise Exception.new("You must specify a valid command") unless Backs3::Restore.commands.include?($command)
|
79
|
+
rescue Exception => e
|
80
|
+
usage(e.to_s)
|
81
|
+
end
|
82
|
+
|
83
|
+
res3 = Backs3::Restore.new($options)
|
84
|
+
|
85
|
+
begin
|
86
|
+
res3.send($command, *args)
|
87
|
+
rescue Exception => e
|
88
|
+
puts e
|
89
|
+
puts e.backtrace.join("\n")
|
90
|
+
end
|
data/example.conf
ADDED
data/lib/backs3.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# #!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems' rescue nil
|
4
|
+
require 'aws/s3'
|
5
|
+
require 'active_support'
|
6
|
+
require 'active_support/dependencies'
|
7
|
+
require 'digest/md5'
|
8
|
+
require 'time'
|
9
|
+
|
10
|
+
unless ActiveSupport::Dependencies.load_paths.include?(File.expand_path(File.dirname(__FILE__) + '/..'))
|
11
|
+
ActiveSupport::Dependencies.load_paths << File.expand_path(File.dirname(__FILE__) + '/..')
|
12
|
+
end
|
13
|
+
|
14
|
+
module Backs3
|
15
|
+
autoload :Backup, 'backs3/backup'
|
16
|
+
autoload :FileInfo, 'backs3/file_info'
|
17
|
+
autoload :Storage, 'backs3/storage'
|
18
|
+
|
19
|
+
def logger
|
20
|
+
logger_output = @options['logger'] || $stdout
|
21
|
+
@logger ||= Logger.new(logger_output)
|
22
|
+
end
|
23
|
+
|
24
|
+
def md5(filename)
|
25
|
+
Digest::MD5.hexdigest(filename)
|
26
|
+
end
|
27
|
+
|
28
|
+
def save_backup_info(info)
|
29
|
+
storage.store('s3backup', YAML.dump(info))
|
30
|
+
logger.info "Backup info has been stored"
|
31
|
+
end
|
32
|
+
|
33
|
+
def load_backup_info
|
34
|
+
@backups ||= begin
|
35
|
+
backup_info_file = storage.read('s3backup') || ''
|
36
|
+
YAML.load(backup_info_file) || []
|
37
|
+
rescue Exception => e
|
38
|
+
puts e.to_s
|
39
|
+
[]
|
40
|
+
end
|
41
|
+
|
42
|
+
unless @backups.respond_to?(:sort) && @backups.respond_to?(:each) && @backups.respond_to?(:reject!)
|
43
|
+
@backups = []
|
44
|
+
end
|
45
|
+
|
46
|
+
@backups.reject! do |backup|
|
47
|
+
!backup.respond_to?(:date)
|
48
|
+
end
|
49
|
+
|
50
|
+
@backups.sort do |a,b|
|
51
|
+
a.date <=> b.date
|
52
|
+
end
|
53
|
+
|
54
|
+
@backups
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
module Backs3
|
2
|
+
class Backup
|
3
|
+
include Backs3
|
4
|
+
include Storage
|
5
|
+
|
6
|
+
attr_reader :date, :files, :full, :options, :last_backup, :last_full_backup, :done
|
7
|
+
|
8
|
+
def initialize(previous, options)
|
9
|
+
@backups = previous.sort{|a,b| a.date <=> b.date } if previous
|
10
|
+
@last_backup = self.backups.last
|
11
|
+
@last_full_backup = self.backups.reverse.detect{|b| b.full == true }
|
12
|
+
|
13
|
+
@date = Time.now.to_i
|
14
|
+
@options = options
|
15
|
+
@options['prefix'] ||= ''
|
16
|
+
@full = @options['force-full'] || first_backup? || @date - @last_full_backup.date > (@options['full'] || 7).days
|
17
|
+
end
|
18
|
+
|
19
|
+
def ==(other_obj)
|
20
|
+
other_obj.date == self.date && other_obj.full == self.full
|
21
|
+
end
|
22
|
+
|
23
|
+
def backups
|
24
|
+
@backups ||= load_backup_info.sort{|a,b| a.date <=> b.date } || []
|
25
|
+
end
|
26
|
+
|
27
|
+
# All of the files for a backup. If the backup is partial this function will
|
28
|
+
# find the files from the last full backup to this one.
|
29
|
+
def all_files
|
30
|
+
if !full && @last_full_backup
|
31
|
+
backups = self.backups.select{|b| b.date >= @last_full_backup.date && b.date <= self.date }
|
32
|
+
backups << self unless backups.include?(self)
|
33
|
+
|
34
|
+
rfiles = backups.collect{|b| b.files}.flatten.uniq
|
35
|
+
rfiles.reject! do |first_file|
|
36
|
+
rfiles.detect{|second_file| second_file.path == first_file.path && second_file.backup_info.date > first_file.backup_info.date }
|
37
|
+
end
|
38
|
+
rfiles
|
39
|
+
else
|
40
|
+
self.files
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def key
|
45
|
+
@options['prefix'] + @date.to_s
|
46
|
+
end
|
47
|
+
|
48
|
+
def first_backup?
|
49
|
+
@last_full_backup.nil?
|
50
|
+
end
|
51
|
+
|
52
|
+
def backup
|
53
|
+
raise "Cannot backup again!" if @done
|
54
|
+
|
55
|
+
logger.info "Backing up #{@options['folder']} in key #{self.key}"
|
56
|
+
|
57
|
+
@files = collect_files
|
58
|
+
@files.each do |file|
|
59
|
+
file.backup
|
60
|
+
end
|
61
|
+
|
62
|
+
update_backup_info
|
63
|
+
storage.flush
|
64
|
+
|
65
|
+
@done = true
|
66
|
+
logger.info "Backup finished!"
|
67
|
+
end
|
68
|
+
|
69
|
+
def restore(location = '/tmp', file = nil)
|
70
|
+
files = file.nil? ? all_files : all_files.select{|f| f.path == file}
|
71
|
+
files.each do |file|
|
72
|
+
file.restore(location)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def to_yaml_properties
|
77
|
+
instance_variables.reject{|i| %w(@backups).include?(i) }.sort
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def update_backup_info
|
83
|
+
raise "Cannot save info twice!" if @done
|
84
|
+
|
85
|
+
@backups << self
|
86
|
+
save_backup_info(@backups)
|
87
|
+
end
|
88
|
+
|
89
|
+
def collect_files
|
90
|
+
files = begin
|
91
|
+
Dir.glob(File.join(@options['folder'], '**', '**')).select do |file|
|
92
|
+
if File.directory?(file) || File.symlink?(file)
|
93
|
+
false
|
94
|
+
else
|
95
|
+
if @options['exclude'].blank? || file !~ /#{@options['exclude']}/
|
96
|
+
if @full || File.mtime(file).to_i > @last_backup.date
|
97
|
+
true
|
98
|
+
else
|
99
|
+
false
|
100
|
+
end
|
101
|
+
else
|
102
|
+
false
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
files.collect{|f| FileInfo.new(self, f) }
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module Backs3
|
2
|
+
class FileInfo
|
3
|
+
include Backs3
|
4
|
+
|
5
|
+
attr_reader :backup_info
|
6
|
+
attr_reader :path
|
7
|
+
attr_reader :md5sum
|
8
|
+
|
9
|
+
def initialize(backup, path)
|
10
|
+
@backup_info = backup
|
11
|
+
@path = path
|
12
|
+
@md5sum = md5(@path)
|
13
|
+
@options = @backup_info.options
|
14
|
+
end
|
15
|
+
|
16
|
+
def storage
|
17
|
+
@backup_info.storage
|
18
|
+
end
|
19
|
+
|
20
|
+
def ==(other_obj)
|
21
|
+
other_obj.backup_info == self.backup_info && other_obj.path == self.path
|
22
|
+
end
|
23
|
+
|
24
|
+
def aws_filename
|
25
|
+
File.join(@backup_info.key, path)
|
26
|
+
end
|
27
|
+
|
28
|
+
def backup
|
29
|
+
logger.info "Backing up #{@path} to #{aws_filename}"
|
30
|
+
storage.store(aws_filename, open(@path))
|
31
|
+
end
|
32
|
+
|
33
|
+
def restore(location = '/tmp')
|
34
|
+
restore_path = File.join(location, @path)
|
35
|
+
|
36
|
+
if storage.exists?(aws_filename)
|
37
|
+
$stdout.write "Restoring file #{@path}"
|
38
|
+
FileUtils.mkdir_p File.dirname(restore_path)
|
39
|
+
File.open(restore_path, 'w') do |f|
|
40
|
+
storage.read(aws_filename) do |segment|
|
41
|
+
$stdout.write "."
|
42
|
+
f.write segment
|
43
|
+
end
|
44
|
+
end
|
45
|
+
$stdout.write "\n"
|
46
|
+
else
|
47
|
+
logger.info "Could not restore #{@path} because file data could not be found!"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/backs3'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module Backs3
|
5
|
+
class Restore
|
6
|
+
include Backs3
|
7
|
+
include Storage
|
8
|
+
|
9
|
+
def self.commands
|
10
|
+
%w(ls available restore cat info)
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(options = {})
|
14
|
+
@options = options
|
15
|
+
@options['prefix'] ||= ''
|
16
|
+
@backups = load_backup_info.sort{|a,b| a.date <=> b.date }
|
17
|
+
end
|
18
|
+
|
19
|
+
def available(backup_key = nil)
|
20
|
+
if backup_key.nil?
|
21
|
+
puts "Backups available: #{@backups.map{|b| b.date}.join(", ")}"
|
22
|
+
else
|
23
|
+
unless backup = @backups.detect{|b| b.date.to_s == backup_key.to_s }
|
24
|
+
raise "No backup #{backup_key} available"
|
25
|
+
end
|
26
|
+
|
27
|
+
files = backup.all_files
|
28
|
+
|
29
|
+
puts "Backup information for #{backup.date}"
|
30
|
+
files.each do |file|
|
31
|
+
puts "\tFile: #{file.path}, backed up #{Time.at(file.backup_info.date).to_s}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def info(file)
|
37
|
+
files = @backups.collect{|b| b.files}.flatten.select{|f| f.path == file}
|
38
|
+
|
39
|
+
if files.empty?
|
40
|
+
puts "No information found for file #{file}"
|
41
|
+
else
|
42
|
+
puts "Backup information for file #{file}"
|
43
|
+
|
44
|
+
files.each do |f|
|
45
|
+
puts "\tBacked up #{Time.at(f.backup_info.date).to_s} in #{f.backup_info.date} with md5sum #{f.md5sum}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def ls(backup)
|
51
|
+
storage.list(backup).each do |name|
|
52
|
+
puts name
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def cat(date, name)
|
57
|
+
backup = @backups.detect{|b| b.date.to_s == date.to_s}
|
58
|
+
raise "Cannot find backup #{date}" unless backup
|
59
|
+
file = backup.all_files.detect{|f| f.path == name}
|
60
|
+
raise "Cannot find file #{name}" unless file
|
61
|
+
puts storage.read(File.join(backup.date.to_s, name))
|
62
|
+
end
|
63
|
+
|
64
|
+
def restore(date, file = nil)
|
65
|
+
backup = @backups.detect{|b| b.date.to_s == date.to_s}
|
66
|
+
raise 'Cannot find backup %s' % date if backup.nil?
|
67
|
+
backup.restore('/tmp', file)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/../spec_helper"
|
2
|
+
require 'backs3/backs3'
|
3
|
+
require 'backs3/backup'
|
4
|
+
|
5
|
+
describe Backs3::Backup do
|
6
|
+
before(:each) do
|
7
|
+
@options = {'folder' => 'test', 'logger' => StringIO.new(''), 'storage' => 'test'}
|
8
|
+
@previous = []
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should set the date to Time.now.to_i' do
|
12
|
+
@mock_time = mock(:time)
|
13
|
+
@mock_time.should_receive(:to_i).and_return(1)
|
14
|
+
Time.should_receive(:now).and_return(@mock_time)
|
15
|
+
|
16
|
+
Backup.new(@previous, @options).date.should == 1
|
17
|
+
end
|
18
|
+
|
19
|
+
describe 'backup' do
|
20
|
+
before(:each) do
|
21
|
+
@file_info = mock(:file_info)
|
22
|
+
@backup = Backup.new(@previous, @options)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should call backup on each file in the backup folder' do
|
26
|
+
@backup.should_receive(:save_backup_info).once
|
27
|
+
@file_info.should_receive(:backup).exactly(3).times
|
28
|
+
|
29
|
+
(1..3).each do |f|
|
30
|
+
FileInfo.should_receive(:new).with(@backup, 'test/file_%d' % f).and_return(@file_info)
|
31
|
+
end
|
32
|
+
|
33
|
+
@backup.backup
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should exclude files based on the exclude option' do
|
37
|
+
@options['exclude'] = 'file_1'
|
38
|
+
@backup = Backup.new(@previous, @options)
|
39
|
+
@backup.should_receive(:save_backup_info).once
|
40
|
+
|
41
|
+
@file_info.should_receive(:backup).exactly(2).times
|
42
|
+
|
43
|
+
(2..3).each do |f|
|
44
|
+
FileInfo.should_receive(:new).with(@backup, 'test/file_%d' % f).and_return(@file_info)
|
45
|
+
end
|
46
|
+
|
47
|
+
@backup.backup
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe 'full_backup?' do
|
52
|
+
it 'should be false if there is a previous full backup' do
|
53
|
+
@previous << mock(:backup, :date => Time.now.to_i, :full => true)
|
54
|
+
Backup.new(@previous, @options).first_backup?.should == false
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should be true if there are no previous full backups' do
|
58
|
+
Backup.new(@previous, @options).first_backup?.should == true
|
59
|
+
@previous << mock(:backup, :date => Time.now.to_i, :full => false)
|
60
|
+
Backup.new(@previous, @options).first_backup?.should == true
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe 'all_files' do
|
65
|
+
it 'should just be the list of files in the backup if the backup is full' do
|
66
|
+
backup = Backup.new(@previous, @options)
|
67
|
+
backup.full.should == true
|
68
|
+
backup.all_files.should == backup.files
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'should include files from the last full backup if partial' do
|
72
|
+
last_full = Backup.new([], @options)
|
73
|
+
last_full.stub!(:full).and_return(true)
|
74
|
+
|
75
|
+
current = Backup.new([last_full], @options)
|
76
|
+
current.stub!(:full).and_return(false)
|
77
|
+
|
78
|
+
file_1 = mock(:file_info, :path => 'test/file_1', :backup_info => last_full)
|
79
|
+
file_2 = mock(:file_info, :path => 'test/file_2', :backup_info => current)
|
80
|
+
|
81
|
+
last_full.stub!(:files).and_return([file_1])
|
82
|
+
current.stub!(:files).and_return([file_2])
|
83
|
+
|
84
|
+
current.all_files.should == [file_1, file_2]
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
describe '@full' do
|
89
|
+
it 'should set full if there is no previous backup specified' do
|
90
|
+
Backup.new(@previous, @options).full.should == true
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'should always set full if there is no full previous backup' do
|
94
|
+
(1..10).each do |d|
|
95
|
+
@previous << mock(:backup, :date => Time.now.to_i - d.days, :full => false)
|
96
|
+
Backup.new(@previous, @options).full.should == true
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
it 'should set full if the previous backup happened more than 7 days ago' do
|
101
|
+
@previous << mock(:backup, :date => Time.now.to_i - 8.days, :full => true)
|
102
|
+
Backup.new(@previous, @options).full.should == true
|
103
|
+
end
|
104
|
+
|
105
|
+
it 'should set full to false if the previous backup happened less than 7 days ago' do
|
106
|
+
@previous << mock(:backup, :date => Time.now.to_i - 6.days, :full => true)
|
107
|
+
Backup.new(@previous, @options).full.should == false
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'should set full if the previous backup happened more than options full' do
|
111
|
+
@options['full'] = 5
|
112
|
+
@previous << mock(:backup, :date => Time.now.to_i - 6.days, :full => true)
|
113
|
+
Backup.new(@previous, @options).full.should == true
|
114
|
+
end
|
115
|
+
|
116
|
+
it 'should not set full if the previous backup happened less than options full days ago' do
|
117
|
+
@options['full'] = 5
|
118
|
+
@previous << mock(:backup, :date => Time.now.to_i - 4.days, :full => true)
|
119
|
+
Backup.new(@previous, @options).full.should == false
|
120
|
+
end
|
121
|
+
|
122
|
+
it 'should always set full if the force full backup options is passed' do
|
123
|
+
@options['force-full'] = true
|
124
|
+
Backup.new(@previous, @options).full.should == true
|
125
|
+
|
126
|
+
(1..10).each do |d|
|
127
|
+
@previous << mock(:backup, :date => Time.now.to_i - d.days, :full => true)
|
128
|
+
Backup.new(@previous, @options).full.should == true
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/../spec_helper"
|
2
|
+
require 'backs3/backs3'
|
3
|
+
require 'backs3/file_info'
|
4
|
+
|
5
|
+
include Backs3
|
6
|
+
|
7
|
+
describe Backs3::FileInfo do
|
8
|
+
before(:each) do
|
9
|
+
@options = {'folder' => 'test', 'logger' => StringIO.new(''), 'bucket' => 'test_bucket'}
|
10
|
+
@backup = mock(:backup)
|
11
|
+
@path = 'test/file_1'
|
12
|
+
|
13
|
+
@storage = mock(:storage)
|
14
|
+
@backup.stub!(:storage).and_return(@storage)
|
15
|
+
|
16
|
+
@backup.stub!(:options).and_return(@options)
|
17
|
+
end
|
18
|
+
|
19
|
+
describe 'aws_filename' do
|
20
|
+
it 'should return the full filename to be put on aws' do
|
21
|
+
@backup.should_receive(:key).and_return('12345')
|
22
|
+
FileInfo.new(@backup, @path).aws_filename.should == '12345/test/file_1'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe '==' do
|
27
|
+
it 'should be equal if files have same path and same backup info' do
|
28
|
+
file_1 = FileInfo.new(@backup, @path)
|
29
|
+
file_2 = FileInfo.new(@backup, @path)
|
30
|
+
file_1.should == file_2
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should be different if files have different path' do
|
34
|
+
file_1 = FileInfo.new(@backup, @path)
|
35
|
+
file_2 = FileInfo.new(@backup, @path + 'diff')
|
36
|
+
file_1.should_not == file_2
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'should be different if files have different backup' do
|
40
|
+
@backup2 = mock(:backup)
|
41
|
+
@backup2.stub!(:options).and_return(@options)
|
42
|
+
file_1 = FileInfo.new(@backup, @path)
|
43
|
+
file_2 = FileInfo.new(@backup2, @path)
|
44
|
+
file_1.should_not == file_2
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
describe 'backup' do
|
49
|
+
before(:each) do
|
50
|
+
@backup.stub!(:key).and_return('12345')
|
51
|
+
end
|
52
|
+
|
53
|
+
it 'should store the file' do
|
54
|
+
@storage.should_receive(:store).with('12345/test/file_1', anything())
|
55
|
+
|
56
|
+
@info = FileInfo.new(@backup, @path)
|
57
|
+
@info.backup
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/../spec_helper"
|
2
|
+
require 'backs3/restore'
|
3
|
+
|
4
|
+
describe Backs3::Restore do
|
5
|
+
before(:each) do
|
6
|
+
@restore = Restore.new('storage' => 'test', 'folder' => 'test')
|
7
|
+
|
8
|
+
@backup_mock1 = mock(:backup_info, :date => 12345, :full => true)
|
9
|
+
@backup_mock2 = mock(:backup_info, :date => 54321, :last_full_backup => @backup_mock1, :full => false)
|
10
|
+
|
11
|
+
@file_mock1 = mock(:file, :backup_info => @backup_mock1, :path => 'test/file_1')
|
12
|
+
@file_mock2 = mock(:file, :backup_info => @backup_mock1, :path => 'test/file_2')
|
13
|
+
@file_mock3 = mock(:file, :backup_info => @backup_mock1, :path => 'test/file_3')
|
14
|
+
@file_mock4 = mock(:file, :backup_info => @backup_mock2, :path => 'test/file_1')
|
15
|
+
|
16
|
+
@files_mock1 = [
|
17
|
+
@file_mock1, @file_mock2, @file_mock3
|
18
|
+
]
|
19
|
+
|
20
|
+
@files_mock2 = [
|
21
|
+
@file_mock4
|
22
|
+
]
|
23
|
+
|
24
|
+
@files_mock3 = [
|
25
|
+
@file_mock4, @file_mock2, @file_mock3
|
26
|
+
]
|
27
|
+
|
28
|
+
@backup_mock1.stub!(:files).and_return(@files_mock1)
|
29
|
+
@backup_mock2.stub!(:files).and_return(@files_mock2)
|
30
|
+
|
31
|
+
@backup_mock1.stub!(:all_files).and_return(@files_mock1)
|
32
|
+
@backup_mock2.stub!(:all_files).and_return(@files_mock3)
|
33
|
+
|
34
|
+
@backup_array = [@backup_mock1, @backup_mock2]
|
35
|
+
@restore.stub!(:load_backup_info).and_return(@backup_array)
|
36
|
+
@restore.instance_variable_set('@backups', @backup_array)
|
37
|
+
end
|
38
|
+
|
39
|
+
describe 'self.commands' do
|
40
|
+
it 'should return an array' do
|
41
|
+
Restore.commands.should be_a(Array)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
describe 'available' do
|
46
|
+
it 'should list all of the backups available' do
|
47
|
+
@restore.should_receive(:puts).with('Backups available: 12345, 54321')
|
48
|
+
@restore.available
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should list all of the available files in a full backup' do
|
52
|
+
@restore.should_receive(:puts).once.with('Backup information for 12345')
|
53
|
+
@restore.should_receive(:puts).once.with("\tFile: test/file_1, backed up #{Time.at(12345).to_s}")
|
54
|
+
@restore.should_receive(:puts).once.with("\tFile: test/file_2, backed up #{Time.at(12345).to_s}")
|
55
|
+
@restore.should_receive(:puts).once.with("\tFile: test/file_3, backed up #{Time.at(12345).to_s}")
|
56
|
+
@restore.available(12345)
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'should list all the files from the last full backup for a partial backup' do
|
60
|
+
@restore.should_receive(:puts).once.with('Backup information for 54321')
|
61
|
+
@restore.should_receive(:puts).once.with("\tFile: test/file_1, backed up #{Time.at(54321).to_s}")
|
62
|
+
@restore.should_receive(:puts).once.with("\tFile: test/file_2, backed up #{Time.at(12345).to_s}")
|
63
|
+
@restore.should_receive(:puts).once.with("\tFile: test/file_3, backed up #{Time.at(12345).to_s}")
|
64
|
+
|
65
|
+
@restore.available(54321)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
describe 'ls' do
|
70
|
+
it 'should list all of the files in a directory'
|
71
|
+
end
|
72
|
+
|
73
|
+
describe 'cat' do
|
74
|
+
it 'should show an error if the file specified does not exist'
|
75
|
+
it 'should output the contents of a file'
|
76
|
+
end
|
77
|
+
|
78
|
+
describe 'restore' do
|
79
|
+
it 'should restore a whole backup if no file is specified'
|
80
|
+
it 'should restore a file'
|
81
|
+
end
|
82
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
begin
|
2
|
+
require 'spec'
|
3
|
+
rescue LoadError
|
4
|
+
require 'rubygems'
|
5
|
+
gem 'rspec'
|
6
|
+
require 'spec'
|
7
|
+
end
|
8
|
+
|
9
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
10
|
+
require 'backs3'
|
11
|
+
|
12
|
+
include Spec::Matchers
|
13
|
+
|
14
|
+
module Kernel
|
15
|
+
def logger
|
16
|
+
@@__log_file__ ||= StringIO.new
|
17
|
+
@@__log__ = ActiveSupport::BufferedLogger.new @@__log_file__
|
18
|
+
end
|
19
|
+
end
|
data/tasks/rspec.rake
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
begin
|
2
|
+
require 'spec'
|
3
|
+
rescue LoadError
|
4
|
+
require 'rubygems'
|
5
|
+
require 'spec'
|
6
|
+
end
|
7
|
+
begin
|
8
|
+
require 'spec/rake/spectask'
|
9
|
+
rescue LoadError
|
10
|
+
puts <<-EOS
|
11
|
+
To use rspec for testing you must install rspec gem:
|
12
|
+
gem install rspec
|
13
|
+
EOS
|
14
|
+
exit(0)
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "Run the specs under spec"
|
18
|
+
Spec::Rake::SpecTask.new do |t|
|
19
|
+
t.spec_opts = ['--options', "spec/spec.opts"]
|
20
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
21
|
+
end
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: backs3
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.5
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jeremy Wells
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-10-16 00:00:00 +13:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: activesupport
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 2.0.2
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: aws-s3
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.5.1
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: hoe
|
37
|
+
type: :development
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 2.3.3
|
44
|
+
version:
|
45
|
+
description: S3 backup and restore program
|
46
|
+
email:
|
47
|
+
- jeremy@boost.co.nz
|
48
|
+
executables:
|
49
|
+
- backs3
|
50
|
+
- res3
|
51
|
+
extensions: []
|
52
|
+
|
53
|
+
extra_rdoc_files:
|
54
|
+
- History.txt
|
55
|
+
- Manifest.txt
|
56
|
+
files:
|
57
|
+
- History.txt
|
58
|
+
- Manifest.txt
|
59
|
+
- README.rdoc
|
60
|
+
- Rakefile
|
61
|
+
- example.conf
|
62
|
+
- bin/backs3
|
63
|
+
- bin/res3
|
64
|
+
- lib/backs3.rb
|
65
|
+
- lib/backs3/backs3.rb
|
66
|
+
- lib/backs3/backup.rb
|
67
|
+
- lib/backs3/restore.rb
|
68
|
+
- lib/backs3/version.rb
|
69
|
+
- lib/backs3/file_info.rb
|
70
|
+
- spec/spec.opts
|
71
|
+
- spec/spec_helper.rb
|
72
|
+
- spec/backs3/backup_spec.rb
|
73
|
+
- spec/backs3/restore_spec.rb
|
74
|
+
- spec/backs3/file_info_spec.rb
|
75
|
+
- tasks/rspec.rake
|
76
|
+
has_rdoc: true
|
77
|
+
homepage: http://github.com/jemmyw/backs3
|
78
|
+
licenses: []
|
79
|
+
|
80
|
+
post_install_message: PostInstall.txt
|
81
|
+
rdoc_options:
|
82
|
+
- --main
|
83
|
+
- README.rdoc
|
84
|
+
require_paths:
|
85
|
+
- lib
|
86
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: "0"
|
91
|
+
version:
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: "0"
|
97
|
+
version:
|
98
|
+
requirements: []
|
99
|
+
|
100
|
+
rubyforge_project: backs3
|
101
|
+
rubygems_version: 1.3.3
|
102
|
+
signing_key:
|
103
|
+
specification_version: 3
|
104
|
+
summary: A simple backup and restore program for S3
|
105
|
+
test_files: []
|
106
|
+
|