rehabilitate 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +9 -0
- data/Gemfile.lock +46 -0
- data/Manifest +14 -0
- data/README.mdown +33 -0
- data/Rakefile +14 -0
- data/bin/rehabilitate +179 -0
- data/lib/rehabilitate/plugin.rb +13 -0
- data/lib/rehabilitate/plugins/lzop.rb +25 -0
- data/lib/rehabilitate/plugins/postgresql.rb +30 -0
- data/lib/rehabilitate/plugins/s3.rb +105 -0
- data/lib/rehabilitate/plugins/scp.rb +35 -0
- data/lib/rehabilitate/plugins/splitter.rb +20 -0
- data/lib/rehabilitate.rb +39 -0
- data/rehabilitate.gemspec +32 -0
- metadata +94 -0
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
builder (3.0.0)
|
5
|
+
commander (4.0.3)
|
6
|
+
highline (>= 1.5.0)
|
7
|
+
echoe (4.3.1)
|
8
|
+
gemcutter
|
9
|
+
rubyforge
|
10
|
+
excon (0.2.4)
|
11
|
+
fog (0.3.25)
|
12
|
+
builder
|
13
|
+
excon (>= 0.2.4)
|
14
|
+
formatador (>= 0.0.16)
|
15
|
+
json
|
16
|
+
mime-types
|
17
|
+
net-ssh (~> 2.0.23)
|
18
|
+
nokogiri (~> 1.4.3.1)
|
19
|
+
ruby-hmac
|
20
|
+
formatador (0.0.16)
|
21
|
+
gemcutter (0.6.1)
|
22
|
+
highline (1.6.1)
|
23
|
+
json (1.4.6)
|
24
|
+
json_pure (1.4.6)
|
25
|
+
log4r (1.1.8)
|
26
|
+
mime-types (1.16)
|
27
|
+
net-scp (1.0.4)
|
28
|
+
net-ssh (>= 1.99.1)
|
29
|
+
net-ssh (2.0.23)
|
30
|
+
nokogiri (1.4.3.1)
|
31
|
+
pluginfactory (1.0.7)
|
32
|
+
ruby-hmac (0.4.0)
|
33
|
+
rubyforge (2.0.4)
|
34
|
+
json_pure (>= 1.1.7)
|
35
|
+
|
36
|
+
PLATFORMS
|
37
|
+
ruby
|
38
|
+
|
39
|
+
DEPENDENCIES
|
40
|
+
commander (= 4.0.3)
|
41
|
+
echoe (= 4.3.1)
|
42
|
+
fog (= 0.3.25)
|
43
|
+
log4r (= 1.1.8)
|
44
|
+
net-scp (= 1.0.4)
|
45
|
+
net-ssh (= 2.0.23)
|
46
|
+
pluginfactory (= 1.0.7)
|
data/Manifest
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Gemfile
|
2
|
+
Gemfile.lock
|
3
|
+
Manifest
|
4
|
+
README.mdown
|
5
|
+
Rakefile
|
6
|
+
rehabilitate.gemspec
|
7
|
+
bin/rehabilitate
|
8
|
+
lib/rehabilitate.rb
|
9
|
+
lib/rehabilitate/plugin.rb
|
10
|
+
lib/rehabilitate/plugins/lzop.rb
|
11
|
+
lib/rehabilitate/plugins/postgresql.rb
|
12
|
+
lib/rehabilitate/plugins/s3.rb
|
13
|
+
lib/rehabilitate/plugins/scp.rb
|
14
|
+
lib/rehabilitate/plugins/splitter.rb
|
data/README.mdown
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
Rehabilitate
|
2
|
+
============
|
3
|
+
|
4
|
+
More to come, here's the comnandline output:
|
5
|
+
|
6
|
+
NAME:
|
7
|
+
|
8
|
+
Backups
|
9
|
+
|
10
|
+
DESCRIPTION:
|
11
|
+
|
12
|
+
A backup system that can be given commands to be run before and after a backup or restore
|
13
|
+
|
14
|
+
COMMANDS:
|
15
|
+
|
16
|
+
help Display global or [command] help documentation.
|
17
|
+
list List backups from a location
|
18
|
+
restore Restore a backup from a location to a database or directory
|
19
|
+
save Save a backup from a database to a location
|
20
|
+
|
21
|
+
GLOBAL OPTIONS:
|
22
|
+
|
23
|
+
-d, --debug
|
24
|
+
More verbose logging and nothing is actually run
|
25
|
+
|
26
|
+
-h, --help
|
27
|
+
Display help documentation
|
28
|
+
|
29
|
+
-v, --version
|
30
|
+
Display version information
|
31
|
+
|
32
|
+
-t, --trace
|
33
|
+
Display backtrace when an error occurs
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'echoe'
|
4
|
+
|
5
|
+
Echoe.new('rehabilitate', '0.3.2') do |p|
|
6
|
+
p.description = "Backup stuff"
|
7
|
+
p.url = "https://github.com/fearoffish/rehabilitate"
|
8
|
+
p.author = "Jamie van Dyke"
|
9
|
+
p.email = "jamie@fearoffish.com"
|
10
|
+
p.ignore_pattern = ["tmp/*", "script/*", "testing/*"]
|
11
|
+
p.development_dependencies = []
|
12
|
+
end
|
13
|
+
|
14
|
+
Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
|
data/bin/rehabilitate
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (c) 2010 Jamie van Dyke
|
4
|
+
# You can redistribute it and/or modify it under the same terms as Ruby.
|
5
|
+
#
|
6
|
+
# The idea behind rehabilitate is that rather than worry about a million different
|
7
|
+
# ways of backing up, we output our backup to a file and do whatever we want
|
8
|
+
# with it, utilising existing tools to augment that.
|
9
|
+
# For example, to back up a postgresql database to a an ftp server we would
|
10
|
+
# give backup the command to run for backing up and then give it an after command
|
11
|
+
# to upload the file.
|
12
|
+
#
|
13
|
+
# example: rehabilitate save --driver postgresql --type scp --location user:pass@ftp.example.com
|
14
|
+
|
15
|
+
require "rubygems"
|
16
|
+
require "bundler/setup"
|
17
|
+
|
18
|
+
$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
|
19
|
+
$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib/rehabilitate"
|
20
|
+
|
21
|
+
require 'rehabilitate'
|
22
|
+
require 'commander/import'
|
23
|
+
require 'log4r'
|
24
|
+
require 'log4r/outputter/syslogoutputter'
|
25
|
+
require 'tmpdir'
|
26
|
+
require 'fileutils'
|
27
|
+
|
28
|
+
include Log4r
|
29
|
+
|
30
|
+
program :name, 'Rehabilitate'
|
31
|
+
program :version, '0.3.2'
|
32
|
+
program :description, 'A backup system that can be given commands to be run before and after a backup or restore'
|
33
|
+
|
34
|
+
global_option('-d', '--debug', 'More verbose logging and nothing is actually run') { $DEBUG = true }
|
35
|
+
|
36
|
+
$LOGGER = Log4r::Logger.new('rehabilitate')
|
37
|
+
$LOGGER.outputters = SyslogOutputter.new("rehabilitate")
|
38
|
+
$LOGGER.info "Starting backup"
|
39
|
+
|
40
|
+
def location_from_options(options)
|
41
|
+
ARGV[((options.__hash__.keys.size * 2) + 1)..-1]
|
42
|
+
end
|
43
|
+
|
44
|
+
def setup_logger(quiet)
|
45
|
+
unless quiet
|
46
|
+
$LOGGER.outputters << Outputter.stdout
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def cleanup(files)
|
51
|
+
$LOGGER.info "Cleaning up temporary files..."
|
52
|
+
files.flatten.compact.each do |file|
|
53
|
+
$LOGGER.info "Deleting #{file}"
|
54
|
+
FileUtils.rm(file)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
begin
|
59
|
+
|
60
|
+
command :save do |c|
|
61
|
+
c.syntax = 'rehabilitate save --driver DRIVER --storage TYPE --location LOCATION'
|
62
|
+
c.description = 'Save a backup from a database to a location'
|
63
|
+
c.example "Backup a postgresql database to scp", "rehabilitate save --driver postgresql --storage scp --location \"user:pass@ftp.example.com\""
|
64
|
+
c.example "Backup a postgresql database to s3", "rehabilitate save --driver postgresql --storage s3 --location \"access_id:secret_key@bucket/file\""
|
65
|
+
c.example "Backup a postgresql database to a dir", "rehabilitate save --driver postgresql --storage dir --location \"/some/location\""
|
66
|
+
c.option '--driver STRING', String, 'Chooses a dbms type.'
|
67
|
+
c.option '--database STRING', String, 'Give the database name.'
|
68
|
+
c.option '--user STRING', String, 'Give the database username.'
|
69
|
+
c.option '--pass STRING', String, 'Give the database password.'
|
70
|
+
c.option '--host STRING', String, 'Give the database host.'
|
71
|
+
c.option '--storage STRING', [:scp, :dir, :s3], 'Choices are [scp, dir, s3].'
|
72
|
+
c.option '--location STRING', String, 'A location string which accompanies the type of backup e.g. "user:pass@ftp.example.com/dir".'
|
73
|
+
c.option '--compressor', String, "Compress the backup on the fly using a compression plugin."
|
74
|
+
c.option '--quiet', "Turn off outputting"
|
75
|
+
c.option '--skip-cleanup', "No deleting temporary files"
|
76
|
+
c.option '--tmp STRING', String, "Where to store temporary files. Default: #{Dir.tmpdir}"
|
77
|
+
c.option '--steps STRING', String, "A comma separated list of steps to perform"
|
78
|
+
c.option '--file FILE', String, "The filename to use when we skip backing up and just upload"
|
79
|
+
c.option '--split-size BYTES', Integer, "How many bytes should we split files by. Default: 4.5TB"
|
80
|
+
c.action do |args, options|
|
81
|
+
options.default :steps => %w{ backup compress upload },
|
82
|
+
:user => ENV['USER'],
|
83
|
+
:driver => 'postgresql',
|
84
|
+
:host => 'localhost',
|
85
|
+
:quiet => false,
|
86
|
+
:tmp => Dir.tmpdir,
|
87
|
+
:compressor => 'lzop',
|
88
|
+
:split_size => (4.5*1024*1024*1024*1024*10).to_i,
|
89
|
+
:_base_backup_name => "#{options.database}--#{Time.now.strftime("%Y-%m-%d_%H-%M")}",
|
90
|
+
:_backup_files => [],
|
91
|
+
:_failure => nil,
|
92
|
+
:_tmp_files => []
|
93
|
+
if options.file
|
94
|
+
options._backup_files = [options.file]
|
95
|
+
else
|
96
|
+
options._backup_files = ["#{options.tmp}/#{options._base_backup_name}"]
|
97
|
+
end
|
98
|
+
setup_logger(options.quiet)
|
99
|
+
(options.steps.is_a?(Array) ? options.steps : options.steps.split(",")).each do |cmd|
|
100
|
+
if options._failure
|
101
|
+
log "Quitting due to failure, use debug next time."
|
102
|
+
else
|
103
|
+
Backup.send(cmd, options)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
cleanup(options._tmp_files) unless options.file or options.skip_cleanup
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
command :restore do |c|
|
111
|
+
c.syntax = 'rehabilitate restore --driver DRIVER --storage TYPE'
|
112
|
+
c.description = 'Restore a backup from a location to a database or directory'
|
113
|
+
c.example "List all the backups on a storage type and location", "rehabilitate restore --list --storage TYPE"
|
114
|
+
c.example "Restore a postgresql database from scp", "rehabilitate restore --driver postgresql --storage scp --location \"user:pass@ftp.example.com\""
|
115
|
+
c.example "Restore a postgresql database from s3", "rehabilitate restore --driver postgresql --storage s3 --location \"access_id:secret_key@bucket/file\""
|
116
|
+
c.example "Restore a postgresql database from a dir", "rehabilitate restore --driver postgresql --storage dir --location \"/some/location\""
|
117
|
+
c.option '--driver STRING', String, 'Chooses a dbms type.'
|
118
|
+
c.option '--database STRING', String, 'Give the database name.'
|
119
|
+
c.option '--user STRING', String, 'Give the database username.'
|
120
|
+
c.option '--pass STRING', String, 'Give the database password.'
|
121
|
+
c.option '--host STRING', String, 'Give the database host.'
|
122
|
+
c.option '--storage STRING', [:scp, :dir, :s3], 'Choices are [scp, dir, s3].'
|
123
|
+
c.option '--location STRING', String, 'A location string which accompanies the type of backup e.g. "user:pass@ftp.example.com/dir".'
|
124
|
+
c.option '--compressor', String, "Decompress the backup on the fly using a compression plugin."
|
125
|
+
c.option '--quiet', "Turn off outputting"
|
126
|
+
c.option '--skip-cleanup', "No deleting temporary files"
|
127
|
+
c.option '--steps STRING', String, "A comma separated list of steps to perform"
|
128
|
+
c.option '--file FILE', String, "The filename to use when we skip backing up and just upload"
|
129
|
+
c.option '--tmp STRING', String, "Where to store temporary files. Default: #{Dir.tmpdir}"
|
130
|
+
c.option '--number INTEGER', Integer, "The backup number to restore (taken from 'backup list')"
|
131
|
+
c.action do |args, options|
|
132
|
+
options.default :steps => %w{ download uncompress restore },
|
133
|
+
:user => ENV['USER'],
|
134
|
+
:driver => 'postgresql',
|
135
|
+
:host => 'localhost',
|
136
|
+
:quiet => false,
|
137
|
+
:tmp => Dir.tmpdir,
|
138
|
+
:compressor => 'lzop',
|
139
|
+
:_base_backup_name => "#{options.database}--#{Time.now.strftime("%Y-%m-%d_%H-%M")}",
|
140
|
+
:_backup_files => [],
|
141
|
+
:_failure => nil,
|
142
|
+
:_tmp_files => []
|
143
|
+
options._backup_files = ["#{options.tmp}/#{options._base_backup_name}"]
|
144
|
+
setup_logger(options.quiet)
|
145
|
+
(options.steps.is_a?(Array) ? options.steps : options.steps.split(",")).each do |cmd|
|
146
|
+
if options._failure
|
147
|
+
log "Quitting due to failure, use debug next time."
|
148
|
+
else
|
149
|
+
Backup.send(cmd, options)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
cleanup(options._tmp_files) unless options.file or options.skip_cleanup
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
command :list do |c|
|
157
|
+
required_options = %w{ storage }
|
158
|
+
c.syntax = 'rehabilitate list --driver DRIVER --storage TYPE'
|
159
|
+
c.description = 'List backups from a location'
|
160
|
+
c.example "List all the backups on a storage type and location", "rehabilitate list --storage TYPE --location s3"
|
161
|
+
c.option '--list', 'Get a list of downloads on a storage device given'
|
162
|
+
c.option '--storage STRING', [:scp, :dir, :s3], 'Choices are [scp, dir, s3].'
|
163
|
+
c.option '--location STRING', String, 'A location string which accompanies the type of backup e.g. "user:pass@ftp.example.com/dir".'
|
164
|
+
c.action do |args, options|
|
165
|
+
options.default :driver => 'postgresql',
|
166
|
+
:_backup_files => [],
|
167
|
+
:_failure => nil
|
168
|
+
options._backup_files = ["#{options.tmp}/#{options._base_backup_name}"]
|
169
|
+
setup_logger(options.quiet)
|
170
|
+
Backup.list(options)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
$LOGGER.info "Backup complete"
|
175
|
+
|
176
|
+
rescue => x
|
177
|
+
$LOGGER.debug "Backup failed with exception:"
|
178
|
+
$LOGGER.debug x.message
|
179
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'plugin'
|
2
|
+
|
3
|
+
class Lzop < Plugin
|
4
|
+
def compress(options)
|
5
|
+
options._backup_files.collect! do |backup_file|
|
6
|
+
new_backup_name = "#{backup_file}.tar.lzop"
|
7
|
+
log "Compressing files..."
|
8
|
+
log "cd #{options.tmp} && tar --use-compress-program=lzop -cf #{new_backup_name} #{backup_file.split("/").last}"
|
9
|
+
log %x[cd #{options.tmp} && tar --use-compress-program=lzop -cf #{new_backup_name} #{backup_file.split("/").last}]
|
10
|
+
options._tmp_files << new_backup_name
|
11
|
+
new_backup_name
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def uncompress(options)
|
16
|
+
options._backup_files.collect! do |backup_file|
|
17
|
+
new_backup_name = "#{options.tmp}/#{File.basename(backup_file).gsub(".tar.lzop", "")}"
|
18
|
+
log "Uncompressing files to #{options.tmp}"
|
19
|
+
log %{ cd #{options.tmp} && lzop -dq < #{options._backup_files.first} | tar -xvf - }
|
20
|
+
log %x{ cd #{options.tmp} && lzop -dq < #{options._backup_files.first} | tar -xvf - }
|
21
|
+
options._tmp_files << new_backup_name
|
22
|
+
new_backup_name
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'plugin'
|
2
|
+
|
3
|
+
class Postgresql < Plugin
|
4
|
+
def backup(options)
|
5
|
+
options._backup_files.collect! do |backup_file|
|
6
|
+
new_backup_name = "#{backup_file}.sql"
|
7
|
+
log "Backing up database #{options.database}"
|
8
|
+
log %[pg_dump -h #{options.host} -U #{options.user} #{options.database} > #{new_backup_name}]
|
9
|
+
log %x[pg_dump -h #{options.host} -U #{options.user} #{options.database} > #{new_backup_name}]
|
10
|
+
options._failure = true if $? == 256
|
11
|
+
options._tmp_files << new_backup_name
|
12
|
+
new_backup_name
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def restore(options)
|
17
|
+
options._backup_files.collect! do |backup_file|
|
18
|
+
log "Restoring database #{options.database}"
|
19
|
+
drop_table_sql = File.join(options.tmp, 'droptables.sql')
|
20
|
+
log %[psql -t -h #{options.host} -U #{options.user} -d #{options.database} -c "SELECT 'DROP TABLE ' || n.nspname || '.' || c.relname || ' CASCADE;' FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'pg_toast') AND pg_catalog.pg_table_is_visible(c.oid)" > #{drop_table_sql} ]
|
21
|
+
log %x[psql -t -h #{options.host} -U #{options.user} -d #{options.database} -c "SELECT 'DROP TABLE ' || n.nspname || '.' || c.relname || ' CASCADE;' FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'pg_toast') AND pg_catalog.pg_table_is_visible(c.oid)" > #{drop_table_sql} ]
|
22
|
+
log %[psql -h #{options.host} -U #{options.user} #{options.database} < #{drop_table_sql} ] unless $? == 256
|
23
|
+
log %x[psql -h #{options.host} -U #{options.user} #{options.database} < #{drop_table_sql} ] unless $? == 256
|
24
|
+
log %[psql -h #{options.host} -U #{options.user} #{options.database} < #{backup_file}] unless $? == 256
|
25
|
+
log %x[psql -h #{options.host} -U #{options.user} #{options.database} < #{backup_file}] unless $? == 256
|
26
|
+
options._tmp_files << drop_table_sql
|
27
|
+
options._failure = true if $? == 256
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'plugin'
|
2
|
+
require 'fog'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
class S3 < Plugin
|
6
|
+
def list(options)
|
7
|
+
location = parse_upload_string(options.location)
|
8
|
+
s3 = setup_fog(location)
|
9
|
+
|
10
|
+
log "Listing bucket contents for #{location[:bucket]}"
|
11
|
+
dir = s3.directories.get(location[:bucket])
|
12
|
+
backups = sorted_backups(dir.files)
|
13
|
+
backups.each_with_index do |backup, i|
|
14
|
+
say "#{i}: #{prettify(backup)}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def upload(options)
|
19
|
+
location = parse_upload_string(options.location)
|
20
|
+
s3 = setup_fog(location)
|
21
|
+
location[:dir] = "#{location[:dir]}/#{options._base_backup_name}"
|
22
|
+
|
23
|
+
log "Creating bucket #{location[:bucket]} if it doesn't exist"
|
24
|
+
s3.directories.create(:key => location[:bucket])
|
25
|
+
options._backup_files.collect! do |local_file|
|
26
|
+
if File.size(local_file) > options.split_size
|
27
|
+
splitter = Splitter.new
|
28
|
+
local_file = splitter.split(local_file, options)
|
29
|
+
else
|
30
|
+
[local_file]
|
31
|
+
end
|
32
|
+
local_file.each do |local_file|
|
33
|
+
log "Uploading #{local_file}"
|
34
|
+
log " => #{local_file}"
|
35
|
+
log %x{ s3cmd --config /etc/s3cfg #{options.s3_options} put #{local_file} s3://#{location[:bucket]}/#{location[:dir]}/ }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def download(options)
|
41
|
+
log "S3 downloading backup"
|
42
|
+
log "Number #{options.number}"
|
43
|
+
|
44
|
+
location = parse_upload_string(options.location)
|
45
|
+
s3 = setup_fog(location)
|
46
|
+
local_file = ""
|
47
|
+
dir = s3.directories.get(location[:bucket])
|
48
|
+
backups = sorted_backups(dir.files)
|
49
|
+
if backups[options.number]
|
50
|
+
log "Restoring #{backups[options.number]}"
|
51
|
+
restore_key = backups[options.number]
|
52
|
+
output = %x{ s3cmd --config /etc/s3cfg get --recursive --skip-existing s3://#{location[:bucket]}/#{backups[options.number]} #{options.tmp}}
|
53
|
+
output.split("\n").each {|f| log f}
|
54
|
+
else
|
55
|
+
log "Invalid number specified, use --list first to get the id you want to restore"
|
56
|
+
end
|
57
|
+
backup_files = Dir.glob("#{options.tmp}/#{backups[options.number].split("/").last}/*")
|
58
|
+
log "Joining #{backup_files.size} files..."
|
59
|
+
joiner = Splitter.new
|
60
|
+
joined = backup_files.size > 1 ? joiner.join(backup_files) : backup_files
|
61
|
+
options._tmp_files << joined
|
62
|
+
options._backup_files = joined.is_a?(Array) ? joined : [joined]
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
# access_id:secret_key@bucket/file
|
68
|
+
def parse_upload_string(upload_string="")
|
69
|
+
location = {}
|
70
|
+
|
71
|
+
if upload_string && !upload_string.empty?
|
72
|
+
location[:access_key], remaining = upload_string.split(":")
|
73
|
+
location[:secret_id], remaining = remaining.split("@")
|
74
|
+
location[:bucket] = remaining.split("/").first
|
75
|
+
location[:dir] = remaining.split("/")[1..-1].join("/")
|
76
|
+
else
|
77
|
+
# use a ~/.fog file
|
78
|
+
fog_config = YAML.load(File.read(File.expand_path(ENV['FOG_RC'] || "~/.fog")))
|
79
|
+
location[:access_key] = fog_config[:default][:aws_access_key_id]
|
80
|
+
location[:secret_id] = fog_config[:default][:aws_secret_access_key]
|
81
|
+
location[:bucket] = fog_config[:default][:bucket]
|
82
|
+
location[:dir] = fog_config[:default][:dir]
|
83
|
+
end
|
84
|
+
location
|
85
|
+
end
|
86
|
+
|
87
|
+
def setup_fog(location)
|
88
|
+
Fog::AWS::Storage.new(
|
89
|
+
:aws_access_key_id => location[:access_key],
|
90
|
+
:aws_secret_access_key => location[:secret_id]
|
91
|
+
)
|
92
|
+
end
|
93
|
+
|
94
|
+
def sorted_backups(backups)
|
95
|
+
backups.collect do |f|
|
96
|
+
key = f.key
|
97
|
+
key.split("/")[0..-2].join("/")
|
98
|
+
end.sort.uniq
|
99
|
+
end
|
100
|
+
|
101
|
+
def prettify(backup_name)
|
102
|
+
env, db, date = backup_name.scan(/(.*?)\/(.*?)--(.*)/).flatten
|
103
|
+
"#{env} => #{ DateTime.strptime(date, "%Y-%m-%d_%H-%M").strftime("%Y-%m-%d %H-%M UTC") }"
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'plugin'
|
2
|
+
require 'uri'
|
3
|
+
require 'net/ssh'
|
4
|
+
require 'net/scp'
|
5
|
+
|
6
|
+
class Scp < Plugin
|
7
|
+
def upload(options)
|
8
|
+
location = parse_upload_string(options.location)
|
9
|
+
remote_dir = "#{location[:dir]}/#{options._base_backup_name}"
|
10
|
+
log "Connecting to #{location[:host]} with user: #{location[:user]}:#{location[:pass]}"
|
11
|
+
|
12
|
+
options._backup_files.collect! do |local_file|
|
13
|
+
remote_file = "#{remote_dir}/#{File.basename(local_file)}"
|
14
|
+
log "Uploading '#{local_file}' to '#{remote_file}'"
|
15
|
+
Net::SCP.upload!( location[:host],
|
16
|
+
location[:user],
|
17
|
+
local_file,
|
18
|
+
remote_file,
|
19
|
+
:password => location[:pass],
|
20
|
+
:recursive => true )
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# e.g. user:pass@ftp.example.com/backup-dir
|
27
|
+
def parse_upload_string(upload_string)
|
28
|
+
location = {}
|
29
|
+
location[:user], remaining = upload_string.split(":")
|
30
|
+
location[:pass], remaining = remaining.split("@")
|
31
|
+
location[:host] = remaining.split("/").first
|
32
|
+
location[:dir] = remaining.split("/")[1..-1].join("/")
|
33
|
+
location
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'plugin'
|
2
|
+
|
3
|
+
class Splitter < Plugin
|
4
|
+
MAX_FILE_SIZE = (4.5*1024*1024*1024*1024*10).to_i #4.5TB
|
5
|
+
|
6
|
+
def join(files)
|
7
|
+
log %[ cat #{files.join(" ")} > #{files[0].split("-")[0..-2].join("-")} ]
|
8
|
+
log %x[ cat #{files.join(" ")} > #{files[0].split("-")[0..-2].join("-")} ]
|
9
|
+
files[0].split("-")[0..-2].join("-")
|
10
|
+
end
|
11
|
+
|
12
|
+
def split(file, options)
|
13
|
+
log "Splitting file (#{file})"
|
14
|
+
split_size = options.split_size || MAX_FILE_SIZE
|
15
|
+
log " => #{split_size} byte pieces"
|
16
|
+
|
17
|
+
log %x{ split -b #{split_size} #{file} #{file}- }
|
18
|
+
Dir.glob("#{file}-??")
|
19
|
+
end
|
20
|
+
end
|
data/lib/rehabilitate.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'rehabilitate/plugin'
|
2
|
+
require 'rehabilitate/plugins/splitter'
|
3
|
+
|
4
|
+
class Rehabilitate
|
5
|
+
def self.backup(options)
|
6
|
+
driver = Plugin::create( options.driver )
|
7
|
+
driver.backup(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.restore(options)
|
11
|
+
driver = Plugin::create( options.driver )
|
12
|
+
driver.restore(options)
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.compress(options)
|
16
|
+
driver = Plugin::create( options.compressor )
|
17
|
+
driver.compress(options)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.uncompress(options)
|
21
|
+
driver = Plugin::create( options.compressor )
|
22
|
+
driver.uncompress(options)
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.upload(options)
|
26
|
+
driver = Plugin::create( options.storage )
|
27
|
+
driver.upload(options)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.download(options)
|
31
|
+
driver = Plugin::create( options.storage )
|
32
|
+
driver.download(options)
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.list(options)
|
36
|
+
driver = Plugin::create( options.storage )
|
37
|
+
driver.list(options)
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{rehabilitate}
|
5
|
+
s.version = "0.3.2"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Jamie van Dyke"]
|
9
|
+
s.date = %q{2011-01-05}
|
10
|
+
s.default_executable = %q{rehabilitate}
|
11
|
+
s.description = %q{Backup stuff}
|
12
|
+
s.email = %q{jamie@fearoffish.com}
|
13
|
+
s.executables = ["rehabilitate"]
|
14
|
+
s.extra_rdoc_files = ["README.mdown", "bin/rehabilitate", "lib/rehabilitate.rb", "lib/rehabilitate/plugin.rb", "lib/rehabilitate/plugins/lzop.rb", "lib/rehabilitate/plugins/postgresql.rb", "lib/rehabilitate/plugins/s3.rb", "lib/rehabilitate/plugins/scp.rb", "lib/rehabilitate/plugins/splitter.rb"]
|
15
|
+
s.files = ["Gemfile", "Gemfile.lock", "Manifest", "README.mdown", "Rakefile", "rehabilitate.gemspec", "bin/rehabilitate", "lib/rehabilitate.rb", "lib/rehabilitate/plugin.rb", "lib/rehabilitate/plugins/lzop.rb", "lib/rehabilitate/plugins/postgresql.rb", "lib/rehabilitate/plugins/s3.rb", "lib/rehabilitate/plugins/scp.rb", "lib/rehabilitate/plugins/splitter.rb"]
|
16
|
+
s.homepage = %q{https://github.com/fearoffish/rehabilitate}
|
17
|
+
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Rehabilitate", "--main", "README.mdown"]
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
s.rubyforge_project = %q{rehabilitate}
|
20
|
+
s.rubygems_version = %q{1.3.7}
|
21
|
+
s.summary = %q{Backup stuff}
|
22
|
+
|
23
|
+
if s.respond_to? :specification_version then
|
24
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
25
|
+
s.specification_version = 3
|
26
|
+
|
27
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
28
|
+
else
|
29
|
+
end
|
30
|
+
else
|
31
|
+
end
|
32
|
+
end
|
metadata
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rehabilitate
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 3
|
9
|
+
- 2
|
10
|
+
version: 0.3.2
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Jamie van Dyke
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-01-05 00:00:00 +00:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description: Backup stuff
|
23
|
+
email: jamie@fearoffish.com
|
24
|
+
executables:
|
25
|
+
- rehabilitate
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files:
|
29
|
+
- README.mdown
|
30
|
+
- bin/rehabilitate
|
31
|
+
- lib/rehabilitate.rb
|
32
|
+
- lib/rehabilitate/plugin.rb
|
33
|
+
- lib/rehabilitate/plugins/lzop.rb
|
34
|
+
- lib/rehabilitate/plugins/postgresql.rb
|
35
|
+
- lib/rehabilitate/plugins/s3.rb
|
36
|
+
- lib/rehabilitate/plugins/scp.rb
|
37
|
+
- lib/rehabilitate/plugins/splitter.rb
|
38
|
+
files:
|
39
|
+
- Gemfile
|
40
|
+
- Gemfile.lock
|
41
|
+
- Manifest
|
42
|
+
- README.mdown
|
43
|
+
- Rakefile
|
44
|
+
- rehabilitate.gemspec
|
45
|
+
- bin/rehabilitate
|
46
|
+
- lib/rehabilitate.rb
|
47
|
+
- lib/rehabilitate/plugin.rb
|
48
|
+
- lib/rehabilitate/plugins/lzop.rb
|
49
|
+
- lib/rehabilitate/plugins/postgresql.rb
|
50
|
+
- lib/rehabilitate/plugins/s3.rb
|
51
|
+
- lib/rehabilitate/plugins/scp.rb
|
52
|
+
- lib/rehabilitate/plugins/splitter.rb
|
53
|
+
has_rdoc: true
|
54
|
+
homepage: https://github.com/fearoffish/rehabilitate
|
55
|
+
licenses: []
|
56
|
+
|
57
|
+
post_install_message:
|
58
|
+
rdoc_options:
|
59
|
+
- --line-numbers
|
60
|
+
- --inline-source
|
61
|
+
- --title
|
62
|
+
- Rehabilitate
|
63
|
+
- --main
|
64
|
+
- README.mdown
|
65
|
+
require_paths:
|
66
|
+
- lib
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
68
|
+
none: false
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
hash: 3
|
73
|
+
segments:
|
74
|
+
- 0
|
75
|
+
version: "0"
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
|
+
none: false
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
hash: 11
|
82
|
+
segments:
|
83
|
+
- 1
|
84
|
+
- 2
|
85
|
+
version: "1.2"
|
86
|
+
requirements: []
|
87
|
+
|
88
|
+
rubyforge_project: rehabilitate
|
89
|
+
rubygems_version: 1.3.7
|
90
|
+
signing_key:
|
91
|
+
specification_version: 3
|
92
|
+
summary: Backup stuff
|
93
|
+
test_files: []
|
94
|
+
|