rehabilitate 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +9 -0
- data/Gemfile.lock +46 -0
- data/Manifest +14 -0
- data/README.mdown +33 -0
- data/Rakefile +14 -0
- data/bin/rehabilitate +179 -0
- data/lib/rehabilitate/plugin.rb +13 -0
- data/lib/rehabilitate/plugins/lzop.rb +25 -0
- data/lib/rehabilitate/plugins/postgresql.rb +30 -0
- data/lib/rehabilitate/plugins/s3.rb +105 -0
- data/lib/rehabilitate/plugins/scp.rb +35 -0
- data/lib/rehabilitate/plugins/splitter.rb +20 -0
- data/lib/rehabilitate.rb +39 -0
- data/rehabilitate.gemspec +32 -0
- metadata +94 -0
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
GEM
|
|
2
|
+
remote: http://rubygems.org/
|
|
3
|
+
specs:
|
|
4
|
+
builder (3.0.0)
|
|
5
|
+
commander (4.0.3)
|
|
6
|
+
highline (>= 1.5.0)
|
|
7
|
+
echoe (4.3.1)
|
|
8
|
+
gemcutter
|
|
9
|
+
rubyforge
|
|
10
|
+
excon (0.2.4)
|
|
11
|
+
fog (0.3.25)
|
|
12
|
+
builder
|
|
13
|
+
excon (>= 0.2.4)
|
|
14
|
+
formatador (>= 0.0.16)
|
|
15
|
+
json
|
|
16
|
+
mime-types
|
|
17
|
+
net-ssh (~> 2.0.23)
|
|
18
|
+
nokogiri (~> 1.4.3.1)
|
|
19
|
+
ruby-hmac
|
|
20
|
+
formatador (0.0.16)
|
|
21
|
+
gemcutter (0.6.1)
|
|
22
|
+
highline (1.6.1)
|
|
23
|
+
json (1.4.6)
|
|
24
|
+
json_pure (1.4.6)
|
|
25
|
+
log4r (1.1.8)
|
|
26
|
+
mime-types (1.16)
|
|
27
|
+
net-scp (1.0.4)
|
|
28
|
+
net-ssh (>= 1.99.1)
|
|
29
|
+
net-ssh (2.0.23)
|
|
30
|
+
nokogiri (1.4.3.1)
|
|
31
|
+
pluginfactory (1.0.7)
|
|
32
|
+
ruby-hmac (0.4.0)
|
|
33
|
+
rubyforge (2.0.4)
|
|
34
|
+
json_pure (>= 1.1.7)
|
|
35
|
+
|
|
36
|
+
PLATFORMS
|
|
37
|
+
ruby
|
|
38
|
+
|
|
39
|
+
DEPENDENCIES
|
|
40
|
+
commander (= 4.0.3)
|
|
41
|
+
echoe (= 4.3.1)
|
|
42
|
+
fog (= 0.3.25)
|
|
43
|
+
log4r (= 1.1.8)
|
|
44
|
+
net-scp (= 1.0.4)
|
|
45
|
+
net-ssh (= 2.0.23)
|
|
46
|
+
pluginfactory (= 1.0.7)
|
data/Manifest
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Gemfile
|
|
2
|
+
Gemfile.lock
|
|
3
|
+
Manifest
|
|
4
|
+
README.mdown
|
|
5
|
+
Rakefile
|
|
6
|
+
rehabilitate.gemspec
|
|
7
|
+
bin/rehabilitate
|
|
8
|
+
lib/rehabilitate.rb
|
|
9
|
+
lib/rehabilitate/plugin.rb
|
|
10
|
+
lib/rehabilitate/plugins/lzop.rb
|
|
11
|
+
lib/rehabilitate/plugins/postgresql.rb
|
|
12
|
+
lib/rehabilitate/plugins/s3.rb
|
|
13
|
+
lib/rehabilitate/plugins/scp.rb
|
|
14
|
+
lib/rehabilitate/plugins/splitter.rb
|
data/README.mdown
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Rehabilitate
|
|
2
|
+
============
|
|
3
|
+
|
|
4
|
+
More to come, here's the comnandline output:
|
|
5
|
+
|
|
6
|
+
NAME:
|
|
7
|
+
|
|
8
|
+
Backups
|
|
9
|
+
|
|
10
|
+
DESCRIPTION:
|
|
11
|
+
|
|
12
|
+
A backup system that can be given commands to be run before and after a backup or restore
|
|
13
|
+
|
|
14
|
+
COMMANDS:
|
|
15
|
+
|
|
16
|
+
help Display global or [command] help documentation.
|
|
17
|
+
list List backups from a location
|
|
18
|
+
restore Restore a backup from a location to a database or directory
|
|
19
|
+
save Save a backup from a database to a location
|
|
20
|
+
|
|
21
|
+
GLOBAL OPTIONS:
|
|
22
|
+
|
|
23
|
+
-d, --debug
|
|
24
|
+
More verbose logging and nothing is actually run
|
|
25
|
+
|
|
26
|
+
-h, --help
|
|
27
|
+
Display help documentation
|
|
28
|
+
|
|
29
|
+
-v, --version
|
|
30
|
+
Display version information
|
|
31
|
+
|
|
32
|
+
-t, --trace
|
|
33
|
+
Display backtrace when an error occurs
|
data/Rakefile
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'rake'
|
|
3
|
+
require 'echoe'
|
|
4
|
+
|
|
5
|
+
Echoe.new('rehabilitate', '0.3.2') do |p|
|
|
6
|
+
p.description = "Backup stuff"
|
|
7
|
+
p.url = "https://github.com/fearoffish/rehabilitate"
|
|
8
|
+
p.author = "Jamie van Dyke"
|
|
9
|
+
p.email = "jamie@fearoffish.com"
|
|
10
|
+
p.ignore_pattern = ["tmp/*", "script/*", "testing/*"]
|
|
11
|
+
p.development_dependencies = []
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
|
data/bin/rehabilitate
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
#
|
|
3
|
+
# Copyright (c) 2010 Jamie van Dyke
|
|
4
|
+
# You can redistribute it and/or modify it under the same terms as Ruby.
|
|
5
|
+
#
|
|
6
|
+
# The idea behind rehabilitate is that rather than worry about a million different
|
|
7
|
+
# ways of backing up, we output our backup to a file and do whatever we want
|
|
8
|
+
# with it, utilising existing tools to augment that.
|
|
9
|
+
# For example, to back up a postgresql database to a an ftp server we would
|
|
10
|
+
# give backup the command to run for backing up and then give it an after command
|
|
11
|
+
# to upload the file.
|
|
12
|
+
#
|
|
13
|
+
# example: rehabilitate save --driver postgresql --type scp --location user:pass@ftp.example.com
|
|
14
|
+
|
|
15
|
+
require "rubygems"
|
|
16
|
+
require "bundler/setup"
|
|
17
|
+
|
|
18
|
+
$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
|
|
19
|
+
$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib/rehabilitate"
|
|
20
|
+
|
|
21
|
+
require 'rehabilitate'
|
|
22
|
+
require 'commander/import'
|
|
23
|
+
require 'log4r'
|
|
24
|
+
require 'log4r/outputter/syslogoutputter'
|
|
25
|
+
require 'tmpdir'
|
|
26
|
+
require 'fileutils'
|
|
27
|
+
|
|
28
|
+
include Log4r
|
|
29
|
+
|
|
30
|
+
program :name, 'Rehabilitate'
|
|
31
|
+
program :version, '0.3.2'
|
|
32
|
+
program :description, 'A backup system that can be given commands to be run before and after a backup or restore'
|
|
33
|
+
|
|
34
|
+
global_option('-d', '--debug', 'More verbose logging and nothing is actually run') { $DEBUG = true }
|
|
35
|
+
|
|
36
|
+
$LOGGER = Log4r::Logger.new('rehabilitate')
|
|
37
|
+
$LOGGER.outputters = SyslogOutputter.new("rehabilitate")
|
|
38
|
+
$LOGGER.info "Starting backup"
|
|
39
|
+
|
|
40
|
+
def location_from_options(options)
|
|
41
|
+
ARGV[((options.__hash__.keys.size * 2) + 1)..-1]
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def setup_logger(quiet)
|
|
45
|
+
unless quiet
|
|
46
|
+
$LOGGER.outputters << Outputter.stdout
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def cleanup(files)
|
|
51
|
+
$LOGGER.info "Cleaning up temporary files..."
|
|
52
|
+
files.flatten.compact.each do |file|
|
|
53
|
+
$LOGGER.info "Deleting #{file}"
|
|
54
|
+
FileUtils.rm(file)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
begin
|
|
59
|
+
|
|
60
|
+
command :save do |c|
|
|
61
|
+
c.syntax = 'rehabilitate save --driver DRIVER --storage TYPE --location LOCATION'
|
|
62
|
+
c.description = 'Save a backup from a database to a location'
|
|
63
|
+
c.example "Backup a postgresql database to scp", "rehabilitate save --driver postgresql --storage scp --location \"user:pass@ftp.example.com\""
|
|
64
|
+
c.example "Backup a postgresql database to s3", "rehabilitate save --driver postgresql --storage s3 --location \"access_id:secret_key@bucket/file\""
|
|
65
|
+
c.example "Backup a postgresql database to a dir", "rehabilitate save --driver postgresql --storage dir --location \"/some/location\""
|
|
66
|
+
c.option '--driver STRING', String, 'Chooses a dbms type.'
|
|
67
|
+
c.option '--database STRING', String, 'Give the database name.'
|
|
68
|
+
c.option '--user STRING', String, 'Give the database username.'
|
|
69
|
+
c.option '--pass STRING', String, 'Give the database password.'
|
|
70
|
+
c.option '--host STRING', String, 'Give the database host.'
|
|
71
|
+
c.option '--storage STRING', [:scp, :dir, :s3], 'Choices are [scp, dir, s3].'
|
|
72
|
+
c.option '--location STRING', String, 'A location string which accompanies the type of backup e.g. "user:pass@ftp.example.com/dir".'
|
|
73
|
+
c.option '--compressor', String, "Compress the backup on the fly using a compression plugin."
|
|
74
|
+
c.option '--quiet', "Turn off outputting"
|
|
75
|
+
c.option '--skip-cleanup', "No deleting temporary files"
|
|
76
|
+
c.option '--tmp STRING', String, "Where to store temporary files. Default: #{Dir.tmpdir}"
|
|
77
|
+
c.option '--steps STRING', String, "A comma separated list of steps to perform"
|
|
78
|
+
c.option '--file FILE', String, "The filename to use when we skip backing up and just upload"
|
|
79
|
+
c.option '--split-size BYTES', Integer, "How many bytes should we split files by. Default: 4.5TB"
|
|
80
|
+
c.action do |args, options|
|
|
81
|
+
options.default :steps => %w{ backup compress upload },
|
|
82
|
+
:user => ENV['USER'],
|
|
83
|
+
:driver => 'postgresql',
|
|
84
|
+
:host => 'localhost',
|
|
85
|
+
:quiet => false,
|
|
86
|
+
:tmp => Dir.tmpdir,
|
|
87
|
+
:compressor => 'lzop',
|
|
88
|
+
:split_size => (4.5*1024*1024*1024*1024*10).to_i,
|
|
89
|
+
:_base_backup_name => "#{options.database}--#{Time.now.strftime("%Y-%m-%d_%H-%M")}",
|
|
90
|
+
:_backup_files => [],
|
|
91
|
+
:_failure => nil,
|
|
92
|
+
:_tmp_files => []
|
|
93
|
+
if options.file
|
|
94
|
+
options._backup_files = [options.file]
|
|
95
|
+
else
|
|
96
|
+
options._backup_files = ["#{options.tmp}/#{options._base_backup_name}"]
|
|
97
|
+
end
|
|
98
|
+
setup_logger(options.quiet)
|
|
99
|
+
(options.steps.is_a?(Array) ? options.steps : options.steps.split(",")).each do |cmd|
|
|
100
|
+
if options._failure
|
|
101
|
+
log "Quitting due to failure, use debug next time."
|
|
102
|
+
else
|
|
103
|
+
Backup.send(cmd, options)
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
cleanup(options._tmp_files) unless options.file or options.skip_cleanup
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
command :restore do |c|
|
|
111
|
+
c.syntax = 'rehabilitate restore --driver DRIVER --storage TYPE'
|
|
112
|
+
c.description = 'Restore a backup from a location to a database or directory'
|
|
113
|
+
c.example "List all the backups on a storage type and location", "rehabilitate restore --list --storage TYPE"
|
|
114
|
+
c.example "Restore a postgresql database from scp", "rehabilitate restore --driver postgresql --storage scp --location \"user:pass@ftp.example.com\""
|
|
115
|
+
c.example "Restore a postgresql database from s3", "rehabilitate restore --driver postgresql --storage s3 --location \"access_id:secret_key@bucket/file\""
|
|
116
|
+
c.example "Restore a postgresql database from a dir", "rehabilitate restore --driver postgresql --storage dir --location \"/some/location\""
|
|
117
|
+
c.option '--driver STRING', String, 'Chooses a dbms type.'
|
|
118
|
+
c.option '--database STRING', String, 'Give the database name.'
|
|
119
|
+
c.option '--user STRING', String, 'Give the database username.'
|
|
120
|
+
c.option '--pass STRING', String, 'Give the database password.'
|
|
121
|
+
c.option '--host STRING', String, 'Give the database host.'
|
|
122
|
+
c.option '--storage STRING', [:scp, :dir, :s3], 'Choices are [scp, dir, s3].'
|
|
123
|
+
c.option '--location STRING', String, 'A location string which accompanies the type of backup e.g. "user:pass@ftp.example.com/dir".'
|
|
124
|
+
c.option '--compressor', String, "Decompress the backup on the fly using a compression plugin."
|
|
125
|
+
c.option '--quiet', "Turn off outputting"
|
|
126
|
+
c.option '--skip-cleanup', "No deleting temporary files"
|
|
127
|
+
c.option '--steps STRING', String, "A comma separated list of steps to perform"
|
|
128
|
+
c.option '--file FILE', String, "The filename to use when we skip backing up and just upload"
|
|
129
|
+
c.option '--tmp STRING', String, "Where to store temporary files. Default: #{Dir.tmpdir}"
|
|
130
|
+
c.option '--number INTEGER', Integer, "The backup number to restore (taken from 'backup list')"
|
|
131
|
+
c.action do |args, options|
|
|
132
|
+
options.default :steps => %w{ download uncompress restore },
|
|
133
|
+
:user => ENV['USER'],
|
|
134
|
+
:driver => 'postgresql',
|
|
135
|
+
:host => 'localhost',
|
|
136
|
+
:quiet => false,
|
|
137
|
+
:tmp => Dir.tmpdir,
|
|
138
|
+
:compressor => 'lzop',
|
|
139
|
+
:_base_backup_name => "#{options.database}--#{Time.now.strftime("%Y-%m-%d_%H-%M")}",
|
|
140
|
+
:_backup_files => [],
|
|
141
|
+
:_failure => nil,
|
|
142
|
+
:_tmp_files => []
|
|
143
|
+
options._backup_files = ["#{options.tmp}/#{options._base_backup_name}"]
|
|
144
|
+
setup_logger(options.quiet)
|
|
145
|
+
(options.steps.is_a?(Array) ? options.steps : options.steps.split(",")).each do |cmd|
|
|
146
|
+
if options._failure
|
|
147
|
+
log "Quitting due to failure, use debug next time."
|
|
148
|
+
else
|
|
149
|
+
Backup.send(cmd, options)
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
cleanup(options._tmp_files) unless options.file or options.skip_cleanup
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
command :list do |c|
|
|
157
|
+
required_options = %w{ storage }
|
|
158
|
+
c.syntax = 'rehabilitate list --driver DRIVER --storage TYPE'
|
|
159
|
+
c.description = 'List backups from a location'
|
|
160
|
+
c.example "List all the backups on a storage type and location", "rehabilitate list --storage TYPE --location s3"
|
|
161
|
+
c.option '--list', 'Get a list of downloads on a storage device given'
|
|
162
|
+
c.option '--storage STRING', [:scp, :dir, :s3], 'Choices are [scp, dir, s3].'
|
|
163
|
+
c.option '--location STRING', String, 'A location string which accompanies the type of backup e.g. "user:pass@ftp.example.com/dir".'
|
|
164
|
+
c.action do |args, options|
|
|
165
|
+
options.default :driver => 'postgresql',
|
|
166
|
+
:_backup_files => [],
|
|
167
|
+
:_failure => nil
|
|
168
|
+
options._backup_files = ["#{options.tmp}/#{options._base_backup_name}"]
|
|
169
|
+
setup_logger(options.quiet)
|
|
170
|
+
Backup.list(options)
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
$LOGGER.info "Backup complete"
|
|
175
|
+
|
|
176
|
+
rescue => x
|
|
177
|
+
$LOGGER.debug "Backup failed with exception:"
|
|
178
|
+
$LOGGER.debug x.message
|
|
179
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
require 'plugin'
|
|
2
|
+
|
|
3
|
+
class Lzop < Plugin
|
|
4
|
+
def compress(options)
|
|
5
|
+
options._backup_files.collect! do |backup_file|
|
|
6
|
+
new_backup_name = "#{backup_file}.tar.lzop"
|
|
7
|
+
log "Compressing files..."
|
|
8
|
+
log "cd #{options.tmp} && tar --use-compress-program=lzop -cf #{new_backup_name} #{backup_file.split("/").last}"
|
|
9
|
+
log %x[cd #{options.tmp} && tar --use-compress-program=lzop -cf #{new_backup_name} #{backup_file.split("/").last}]
|
|
10
|
+
options._tmp_files << new_backup_name
|
|
11
|
+
new_backup_name
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def uncompress(options)
|
|
16
|
+
options._backup_files.collect! do |backup_file|
|
|
17
|
+
new_backup_name = "#{options.tmp}/#{File.basename(backup_file).gsub(".tar.lzop", "")}"
|
|
18
|
+
log "Uncompressing files to #{options.tmp}"
|
|
19
|
+
log %{ cd #{options.tmp} && lzop -dq < #{options._backup_files.first} | tar -xvf - }
|
|
20
|
+
log %x{ cd #{options.tmp} && lzop -dq < #{options._backup_files.first} | tar -xvf - }
|
|
21
|
+
options._tmp_files << new_backup_name
|
|
22
|
+
new_backup_name
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
require 'plugin'
|
|
2
|
+
|
|
3
|
+
class Postgresql < Plugin
|
|
4
|
+
def backup(options)
|
|
5
|
+
options._backup_files.collect! do |backup_file|
|
|
6
|
+
new_backup_name = "#{backup_file}.sql"
|
|
7
|
+
log "Backing up database #{options.database}"
|
|
8
|
+
log %[pg_dump -h #{options.host} -U #{options.user} #{options.database} > #{new_backup_name}]
|
|
9
|
+
log %x[pg_dump -h #{options.host} -U #{options.user} #{options.database} > #{new_backup_name}]
|
|
10
|
+
options._failure = true if $? == 256
|
|
11
|
+
options._tmp_files << new_backup_name
|
|
12
|
+
new_backup_name
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def restore(options)
|
|
17
|
+
options._backup_files.collect! do |backup_file|
|
|
18
|
+
log "Restoring database #{options.database}"
|
|
19
|
+
drop_table_sql = File.join(options.tmp, 'droptables.sql')
|
|
20
|
+
log %[psql -t -h #{options.host} -U #{options.user} -d #{options.database} -c "SELECT 'DROP TABLE ' || n.nspname || '.' || c.relname || ' CASCADE;' FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'pg_toast') AND pg_catalog.pg_table_is_visible(c.oid)" > #{drop_table_sql} ]
|
|
21
|
+
log %x[psql -t -h #{options.host} -U #{options.user} -d #{options.database} -c "SELECT 'DROP TABLE ' || n.nspname || '.' || c.relname || ' CASCADE;' FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'pg_toast') AND pg_catalog.pg_table_is_visible(c.oid)" > #{drop_table_sql} ]
|
|
22
|
+
log %[psql -h #{options.host} -U #{options.user} #{options.database} < #{drop_table_sql} ] unless $? == 256
|
|
23
|
+
log %x[psql -h #{options.host} -U #{options.user} #{options.database} < #{drop_table_sql} ] unless $? == 256
|
|
24
|
+
log %[psql -h #{options.host} -U #{options.user} #{options.database} < #{backup_file}] unless $? == 256
|
|
25
|
+
log %x[psql -h #{options.host} -U #{options.user} #{options.database} < #{backup_file}] unless $? == 256
|
|
26
|
+
options._tmp_files << drop_table_sql
|
|
27
|
+
options._failure = true if $? == 256
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
require 'plugin'
|
|
2
|
+
require 'fog'
|
|
3
|
+
require 'yaml'
|
|
4
|
+
|
|
5
|
+
class S3 < Plugin
|
|
6
|
+
def list(options)
|
|
7
|
+
location = parse_upload_string(options.location)
|
|
8
|
+
s3 = setup_fog(location)
|
|
9
|
+
|
|
10
|
+
log "Listing bucket contents for #{location[:bucket]}"
|
|
11
|
+
dir = s3.directories.get(location[:bucket])
|
|
12
|
+
backups = sorted_backups(dir.files)
|
|
13
|
+
backups.each_with_index do |backup, i|
|
|
14
|
+
say "#{i}: #{prettify(backup)}"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def upload(options)
|
|
19
|
+
location = parse_upload_string(options.location)
|
|
20
|
+
s3 = setup_fog(location)
|
|
21
|
+
location[:dir] = "#{location[:dir]}/#{options._base_backup_name}"
|
|
22
|
+
|
|
23
|
+
log "Creating bucket #{location[:bucket]} if it doesn't exist"
|
|
24
|
+
s3.directories.create(:key => location[:bucket])
|
|
25
|
+
options._backup_files.collect! do |local_file|
|
|
26
|
+
if File.size(local_file) > options.split_size
|
|
27
|
+
splitter = Splitter.new
|
|
28
|
+
local_file = splitter.split(local_file, options)
|
|
29
|
+
else
|
|
30
|
+
[local_file]
|
|
31
|
+
end
|
|
32
|
+
local_file.each do |local_file|
|
|
33
|
+
log "Uploading #{local_file}"
|
|
34
|
+
log " => #{local_file}"
|
|
35
|
+
log %x{ s3cmd --config /etc/s3cfg #{options.s3_options} put #{local_file} s3://#{location[:bucket]}/#{location[:dir]}/ }
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def download(options)
|
|
41
|
+
log "S3 downloading backup"
|
|
42
|
+
log "Number #{options.number}"
|
|
43
|
+
|
|
44
|
+
location = parse_upload_string(options.location)
|
|
45
|
+
s3 = setup_fog(location)
|
|
46
|
+
local_file = ""
|
|
47
|
+
dir = s3.directories.get(location[:bucket])
|
|
48
|
+
backups = sorted_backups(dir.files)
|
|
49
|
+
if backups[options.number]
|
|
50
|
+
log "Restoring #{backups[options.number]}"
|
|
51
|
+
restore_key = backups[options.number]
|
|
52
|
+
output = %x{ s3cmd --config /etc/s3cfg get --recursive --skip-existing s3://#{location[:bucket]}/#{backups[options.number]} #{options.tmp}}
|
|
53
|
+
output.split("\n").each {|f| log f}
|
|
54
|
+
else
|
|
55
|
+
log "Invalid number specified, use --list first to get the id you want to restore"
|
|
56
|
+
end
|
|
57
|
+
backup_files = Dir.glob("#{options.tmp}/#{backups[options.number].split("/").last}/*")
|
|
58
|
+
log "Joining #{backup_files.size} files..."
|
|
59
|
+
joiner = Splitter.new
|
|
60
|
+
joined = backup_files.size > 1 ? joiner.join(backup_files) : backup_files
|
|
61
|
+
options._tmp_files << joined
|
|
62
|
+
options._backup_files = joined.is_a?(Array) ? joined : [joined]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# access_id:secret_key@bucket/file
|
|
68
|
+
def parse_upload_string(upload_string="")
|
|
69
|
+
location = {}
|
|
70
|
+
|
|
71
|
+
if upload_string && !upload_string.empty?
|
|
72
|
+
location[:access_key], remaining = upload_string.split(":")
|
|
73
|
+
location[:secret_id], remaining = remaining.split("@")
|
|
74
|
+
location[:bucket] = remaining.split("/").first
|
|
75
|
+
location[:dir] = remaining.split("/")[1..-1].join("/")
|
|
76
|
+
else
|
|
77
|
+
# use a ~/.fog file
|
|
78
|
+
fog_config = YAML.load(File.read(File.expand_path(ENV['FOG_RC'] || "~/.fog")))
|
|
79
|
+
location[:access_key] = fog_config[:default][:aws_access_key_id]
|
|
80
|
+
location[:secret_id] = fog_config[:default][:aws_secret_access_key]
|
|
81
|
+
location[:bucket] = fog_config[:default][:bucket]
|
|
82
|
+
location[:dir] = fog_config[:default][:dir]
|
|
83
|
+
end
|
|
84
|
+
location
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def setup_fog(location)
|
|
88
|
+
Fog::AWS::Storage.new(
|
|
89
|
+
:aws_access_key_id => location[:access_key],
|
|
90
|
+
:aws_secret_access_key => location[:secret_id]
|
|
91
|
+
)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def sorted_backups(backups)
|
|
95
|
+
backups.collect do |f|
|
|
96
|
+
key = f.key
|
|
97
|
+
key.split("/")[0..-2].join("/")
|
|
98
|
+
end.sort.uniq
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def prettify(backup_name)
|
|
102
|
+
env, db, date = backup_name.scan(/(.*?)\/(.*?)--(.*)/).flatten
|
|
103
|
+
"#{env} => #{ DateTime.strptime(date, "%Y-%m-%d_%H-%M").strftime("%Y-%m-%d %H-%M UTC") }"
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
require 'plugin'
|
|
2
|
+
require 'uri'
|
|
3
|
+
require 'net/ssh'
|
|
4
|
+
require 'net/scp'
|
|
5
|
+
|
|
6
|
+
class Scp < Plugin
|
|
7
|
+
def upload(options)
|
|
8
|
+
location = parse_upload_string(options.location)
|
|
9
|
+
remote_dir = "#{location[:dir]}/#{options._base_backup_name}"
|
|
10
|
+
log "Connecting to #{location[:host]} with user: #{location[:user]}:#{location[:pass]}"
|
|
11
|
+
|
|
12
|
+
options._backup_files.collect! do |local_file|
|
|
13
|
+
remote_file = "#{remote_dir}/#{File.basename(local_file)}"
|
|
14
|
+
log "Uploading '#{local_file}' to '#{remote_file}'"
|
|
15
|
+
Net::SCP.upload!( location[:host],
|
|
16
|
+
location[:user],
|
|
17
|
+
local_file,
|
|
18
|
+
remote_file,
|
|
19
|
+
:password => location[:pass],
|
|
20
|
+
:recursive => true )
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
# e.g. user:pass@ftp.example.com/backup-dir
|
|
27
|
+
def parse_upload_string(upload_string)
|
|
28
|
+
location = {}
|
|
29
|
+
location[:user], remaining = upload_string.split(":")
|
|
30
|
+
location[:pass], remaining = remaining.split("@")
|
|
31
|
+
location[:host] = remaining.split("/").first
|
|
32
|
+
location[:dir] = remaining.split("/")[1..-1].join("/")
|
|
33
|
+
location
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
require 'plugin'
|
|
2
|
+
|
|
3
|
+
class Splitter < Plugin
|
|
4
|
+
MAX_FILE_SIZE = (4.5*1024*1024*1024*1024*10).to_i #4.5TB
|
|
5
|
+
|
|
6
|
+
def join(files)
|
|
7
|
+
log %[ cat #{files.join(" ")} > #{files[0].split("-")[0..-2].join("-")} ]
|
|
8
|
+
log %x[ cat #{files.join(" ")} > #{files[0].split("-")[0..-2].join("-")} ]
|
|
9
|
+
files[0].split("-")[0..-2].join("-")
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def split(file, options)
|
|
13
|
+
log "Splitting file (#{file})"
|
|
14
|
+
split_size = options.split_size || MAX_FILE_SIZE
|
|
15
|
+
log " => #{split_size} byte pieces"
|
|
16
|
+
|
|
17
|
+
log %x{ split -b #{split_size} #{file} #{file}- }
|
|
18
|
+
Dir.glob("#{file}-??")
|
|
19
|
+
end
|
|
20
|
+
end
|
data/lib/rehabilitate.rb
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
require 'rehabilitate/plugin'
|
|
2
|
+
require 'rehabilitate/plugins/splitter'
|
|
3
|
+
|
|
4
|
+
class Rehabilitate
|
|
5
|
+
def self.backup(options)
|
|
6
|
+
driver = Plugin::create( options.driver )
|
|
7
|
+
driver.backup(options)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def self.restore(options)
|
|
11
|
+
driver = Plugin::create( options.driver )
|
|
12
|
+
driver.restore(options)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.compress(options)
|
|
16
|
+
driver = Plugin::create( options.compressor )
|
|
17
|
+
driver.compress(options)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.uncompress(options)
|
|
21
|
+
driver = Plugin::create( options.compressor )
|
|
22
|
+
driver.uncompress(options)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.upload(options)
|
|
26
|
+
driver = Plugin::create( options.storage )
|
|
27
|
+
driver.upload(options)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.download(options)
|
|
31
|
+
driver = Plugin::create( options.storage )
|
|
32
|
+
driver.download(options)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.list(options)
|
|
36
|
+
driver = Plugin::create( options.storage )
|
|
37
|
+
driver.list(options)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
Gem::Specification.new do |s|
|
|
4
|
+
s.name = %q{rehabilitate}
|
|
5
|
+
s.version = "0.3.2"
|
|
6
|
+
|
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
|
8
|
+
s.authors = ["Jamie van Dyke"]
|
|
9
|
+
s.date = %q{2011-01-05}
|
|
10
|
+
s.default_executable = %q{rehabilitate}
|
|
11
|
+
s.description = %q{Backup stuff}
|
|
12
|
+
s.email = %q{jamie@fearoffish.com}
|
|
13
|
+
s.executables = ["rehabilitate"]
|
|
14
|
+
s.extra_rdoc_files = ["README.mdown", "bin/rehabilitate", "lib/rehabilitate.rb", "lib/rehabilitate/plugin.rb", "lib/rehabilitate/plugins/lzop.rb", "lib/rehabilitate/plugins/postgresql.rb", "lib/rehabilitate/plugins/s3.rb", "lib/rehabilitate/plugins/scp.rb", "lib/rehabilitate/plugins/splitter.rb"]
|
|
15
|
+
s.files = ["Gemfile", "Gemfile.lock", "Manifest", "README.mdown", "Rakefile", "rehabilitate.gemspec", "bin/rehabilitate", "lib/rehabilitate.rb", "lib/rehabilitate/plugin.rb", "lib/rehabilitate/plugins/lzop.rb", "lib/rehabilitate/plugins/postgresql.rb", "lib/rehabilitate/plugins/s3.rb", "lib/rehabilitate/plugins/scp.rb", "lib/rehabilitate/plugins/splitter.rb"]
|
|
16
|
+
s.homepage = %q{https://github.com/fearoffish/rehabilitate}
|
|
17
|
+
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Rehabilitate", "--main", "README.mdown"]
|
|
18
|
+
s.require_paths = ["lib"]
|
|
19
|
+
s.rubyforge_project = %q{rehabilitate}
|
|
20
|
+
s.rubygems_version = %q{1.3.7}
|
|
21
|
+
s.summary = %q{Backup stuff}
|
|
22
|
+
|
|
23
|
+
if s.respond_to? :specification_version then
|
|
24
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
|
25
|
+
s.specification_version = 3
|
|
26
|
+
|
|
27
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
|
28
|
+
else
|
|
29
|
+
end
|
|
30
|
+
else
|
|
31
|
+
end
|
|
32
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: rehabilitate
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
hash: 23
|
|
5
|
+
prerelease: false
|
|
6
|
+
segments:
|
|
7
|
+
- 0
|
|
8
|
+
- 3
|
|
9
|
+
- 2
|
|
10
|
+
version: 0.3.2
|
|
11
|
+
platform: ruby
|
|
12
|
+
authors:
|
|
13
|
+
- Jamie van Dyke
|
|
14
|
+
autorequire:
|
|
15
|
+
bindir: bin
|
|
16
|
+
cert_chain: []
|
|
17
|
+
|
|
18
|
+
date: 2011-01-05 00:00:00 +00:00
|
|
19
|
+
default_executable:
|
|
20
|
+
dependencies: []
|
|
21
|
+
|
|
22
|
+
description: Backup stuff
|
|
23
|
+
email: jamie@fearoffish.com
|
|
24
|
+
executables:
|
|
25
|
+
- rehabilitate
|
|
26
|
+
extensions: []
|
|
27
|
+
|
|
28
|
+
extra_rdoc_files:
|
|
29
|
+
- README.mdown
|
|
30
|
+
- bin/rehabilitate
|
|
31
|
+
- lib/rehabilitate.rb
|
|
32
|
+
- lib/rehabilitate/plugin.rb
|
|
33
|
+
- lib/rehabilitate/plugins/lzop.rb
|
|
34
|
+
- lib/rehabilitate/plugins/postgresql.rb
|
|
35
|
+
- lib/rehabilitate/plugins/s3.rb
|
|
36
|
+
- lib/rehabilitate/plugins/scp.rb
|
|
37
|
+
- lib/rehabilitate/plugins/splitter.rb
|
|
38
|
+
files:
|
|
39
|
+
- Gemfile
|
|
40
|
+
- Gemfile.lock
|
|
41
|
+
- Manifest
|
|
42
|
+
- README.mdown
|
|
43
|
+
- Rakefile
|
|
44
|
+
- rehabilitate.gemspec
|
|
45
|
+
- bin/rehabilitate
|
|
46
|
+
- lib/rehabilitate.rb
|
|
47
|
+
- lib/rehabilitate/plugin.rb
|
|
48
|
+
- lib/rehabilitate/plugins/lzop.rb
|
|
49
|
+
- lib/rehabilitate/plugins/postgresql.rb
|
|
50
|
+
- lib/rehabilitate/plugins/s3.rb
|
|
51
|
+
- lib/rehabilitate/plugins/scp.rb
|
|
52
|
+
- lib/rehabilitate/plugins/splitter.rb
|
|
53
|
+
has_rdoc: true
|
|
54
|
+
homepage: https://github.com/fearoffish/rehabilitate
|
|
55
|
+
licenses: []
|
|
56
|
+
|
|
57
|
+
post_install_message:
|
|
58
|
+
rdoc_options:
|
|
59
|
+
- --line-numbers
|
|
60
|
+
- --inline-source
|
|
61
|
+
- --title
|
|
62
|
+
- Rehabilitate
|
|
63
|
+
- --main
|
|
64
|
+
- README.mdown
|
|
65
|
+
require_paths:
|
|
66
|
+
- lib
|
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
68
|
+
none: false
|
|
69
|
+
requirements:
|
|
70
|
+
- - ">="
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
hash: 3
|
|
73
|
+
segments:
|
|
74
|
+
- 0
|
|
75
|
+
version: "0"
|
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
77
|
+
none: false
|
|
78
|
+
requirements:
|
|
79
|
+
- - ">="
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
hash: 11
|
|
82
|
+
segments:
|
|
83
|
+
- 1
|
|
84
|
+
- 2
|
|
85
|
+
version: "1.2"
|
|
86
|
+
requirements: []
|
|
87
|
+
|
|
88
|
+
rubyforge_project: rehabilitate
|
|
89
|
+
rubygems_version: 1.3.7
|
|
90
|
+
signing_key:
|
|
91
|
+
specification_version: 3
|
|
92
|
+
summary: Backup stuff
|
|
93
|
+
test_files: []
|
|
94
|
+
|