backupgem 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +4 -0
- data/TODO +7 -0
- data/examples/s3.rb +35 -0
- data/lib/backup/actor.rb +1 -0
- data/lib/backup/recipes/standard.rb +5 -0
- data/lib/backup/rotator.rb +18 -6
- data/lib/backup/s3_helpers.rb +95 -0
- data/lib/backup/ssh_helpers.rb +3 -2
- data/lib/backup.rb +9 -0
- data/tests/s3_test.rb +40 -0
- metadata +8 -2
data/CHANGELOG
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
= Backup Changelog
|
2
2
|
|
3
|
+
== Version 0.0.5
|
4
|
+
* Added Amazon s3 support. Thanks to Jason Perry for this patch.
|
5
|
+
* Added an option for ssh port. Thanks to Frank Oxener for this patch.
|
6
|
+
|
3
7
|
== Version 0.0.4
|
4
8
|
|
5
9
|
* Fixed a bug in 'scp' in the standard recipe that ignored the user when using
|
data/TODO
ADDED
data/examples/s3.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
#------------------------------------------------------------------------------
|
2
|
+
# Example S3 Backup script
|
3
|
+
# @author: Jason L. Perry <jasper@ambethia.com>
|
4
|
+
#------------------------------------------------------------------------------
|
5
|
+
|
6
|
+
# Set the name of the s3 bucket you want to store your backups in.
|
7
|
+
# Your Access ID is prepended to this to avoid naming conflicts.
|
8
|
+
set :backup_path, "database_backup"
|
9
|
+
|
10
|
+
# You can specify your keys here, or set them as environment variables:
|
11
|
+
# AMAZON_ACCESS_KEY_ID
|
12
|
+
# AMAZON_SECRET_ACCESS_KEY
|
13
|
+
set :aws_access, '123'
|
14
|
+
set :aws_secret, 'ABC'
|
15
|
+
|
16
|
+
# S3 does not support renaming objects, so rotation data is stored in an
|
17
|
+
# index. You can specify a different key for index here, if you need to.
|
18
|
+
#
|
19
|
+
# set :rotation_object_key, 'backup_rotation_index.yml'
|
20
|
+
|
21
|
+
action(:content) do
|
22
|
+
dump = c[:tmp_dir] + "/databases.sql"
|
23
|
+
sh "mysqldump -uroot --all-databases > #{dump}"
|
24
|
+
dump
|
25
|
+
end
|
26
|
+
|
27
|
+
action :deliver, :method => :s3
|
28
|
+
action :rotate, :method => :via_s3
|
29
|
+
|
30
|
+
set :son_promoted_on, :fri
|
31
|
+
set :father_promoted_on, :last_fri_of_the_month
|
32
|
+
|
33
|
+
set :sons_to_keep, 7
|
34
|
+
set :fathers_to_keep, 5
|
35
|
+
set :grandfathers_to_keep, 12
|
data/lib/backup/actor.rb
CHANGED
@@ -113,6 +113,7 @@ module Backup
|
|
113
113
|
def via_mv; rotator.rotate_via_mv(last_result); end
|
114
114
|
def via_ssh; rotator.rotate_via_ssh(last_result); end
|
115
115
|
def via_ftp; rotator.rotate_via_ftp(last_result); end
|
116
|
+
def via_s3; rotator.rotate_via_s3(last_result); end
|
116
117
|
|
117
118
|
# By default, +:content+ can perform one of three actions
|
118
119
|
# * +:is_file+
|
data/lib/backup/rotator.rb
CHANGED
@@ -59,6 +59,15 @@ module Backup
|
|
59
59
|
# ftp.close
|
60
60
|
end
|
61
61
|
|
62
|
+
def rotate_via_s3(last_result)
|
63
|
+
s3 = Backup::S3Actor.new(c)
|
64
|
+
s3.verify_rotation_hierarchy_exists(hierarchy)
|
65
|
+
index = s3.rotation
|
66
|
+
index[todays_generation] << last_result
|
67
|
+
s3.rotation = index
|
68
|
+
s3.cleanup(todays_generation, how_many_to_keep_today)
|
69
|
+
end
|
70
|
+
|
62
71
|
def create_sons_today?; is_today_a? :son_created_on; end
|
63
72
|
def promote_sons_today?; is_today_a? :son_promoted_on; end
|
64
73
|
def promote_fathers_today?; is_today_a? :father_promoted_on; end
|
@@ -110,18 +119,21 @@ module Backup
|
|
110
119
|
promote_sons_today? ? "fathers" : "sons"
|
111
120
|
end
|
112
121
|
|
122
|
+
def self.timestamped_prefix(name)
|
123
|
+
newname = Time.now.strftime("%Y-%m-%d-%H-%M-%S_") + File.basename(name)
|
124
|
+
end
|
125
|
+
|
126
|
+
# Given +name+ returns a timestamped version of name.
|
127
|
+
def timestamped_prefix(name)
|
128
|
+
Backup::Rotator.timestamped_prefix(name)
|
129
|
+
end
|
130
|
+
|
113
131
|
private
|
114
132
|
def place_in
|
115
133
|
goes_in = todays_generation
|
116
134
|
place_in = c[:backup_path] + "/" + goes_in
|
117
135
|
end
|
118
136
|
|
119
|
-
|
120
|
-
# Given +name+ returns a timestamped version of name.
|
121
|
-
def timestamped_prefix(name)
|
122
|
-
newname = Time.now.strftime("%Y-%m-%d-%H-%M-%S_") + File.basename(name)
|
123
|
-
end
|
124
|
-
|
125
137
|
# Returns the number of sons to keep. Looks for config values +:sons_to_keep+,
|
126
138
|
# +:son_promoted_on+. Default +14+.
|
127
139
|
def sons_to_keep
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'YAML'
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
class S3Actor
|
5
|
+
include AWS::S3
|
6
|
+
|
7
|
+
attr_accessor :rotation
|
8
|
+
|
9
|
+
attr_reader :config
|
10
|
+
alias_method :c, :config
|
11
|
+
|
12
|
+
def initialize(config)
|
13
|
+
@config = config
|
14
|
+
@rotation_key = c[:rotation_object_key] ||= 'backup_rotation_index.yml'
|
15
|
+
@access_key = c[:aws_access] ||= ENV['AMAZON_ACCESS_KEY_ID']
|
16
|
+
@secret_key = c[:aws_secret] ||= ENV['AMAZON_SECRET_ACCESS_KEY']
|
17
|
+
@bucket_key = "#{@access_key}.#{c[:backup_path]}"
|
18
|
+
Base.establish_connection!(
|
19
|
+
:access_key_id => @access_key,
|
20
|
+
:secret_access_key => @secret_key
|
21
|
+
)
|
22
|
+
begin
|
23
|
+
# Look for our bucket, if it's not there, try to create it.
|
24
|
+
@bucket = Bucket.find @bucket_key
|
25
|
+
rescue NoSuchBucket
|
26
|
+
@bucket = Bucket.create @bucket_key
|
27
|
+
@bucket = Bucket.find @bucket_key
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def rotation
|
32
|
+
object = S3Object.find(@rotation_key, @bucket.name)
|
33
|
+
index = YAML::load(object.value)
|
34
|
+
end
|
35
|
+
|
36
|
+
def rotation=(index)
|
37
|
+
object = S3Object.store(@rotation_key, index.to_yaml, @bucket.name)
|
38
|
+
index
|
39
|
+
end
|
40
|
+
|
41
|
+
# Send a file to s3
|
42
|
+
def put(last_result)
|
43
|
+
puts last_result
|
44
|
+
object_key = Rotator.timestamped_prefix(last_result)
|
45
|
+
S3Object.store object_key,
|
46
|
+
open(last_result),
|
47
|
+
@bucket.name
|
48
|
+
object_key
|
49
|
+
end
|
50
|
+
|
51
|
+
# Remove a file from s3
|
52
|
+
def delete(object_key)
|
53
|
+
S3Object.delete object_key, @bucket.name
|
54
|
+
end
|
55
|
+
|
56
|
+
# Make sure our rotation index exists and contains the hierarchy we're using.
|
57
|
+
# Create it if it does not exist
|
58
|
+
def verify_rotation_hierarchy_exists(hierarchy)
|
59
|
+
begin
|
60
|
+
index = rotation
|
61
|
+
verified_index = index.merge(init_rotation_index(hierarchy)) { |m,x,y| x ||= y }
|
62
|
+
unless (verified_index == index)
|
63
|
+
self.rotation = verified_index
|
64
|
+
end
|
65
|
+
rescue NoSuchKey
|
66
|
+
self.rotation = init_rotation_index(hierarchy)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Expire old objects
|
71
|
+
def cleanup(generation, keep)
|
72
|
+
puts "Cleaning up"
|
73
|
+
|
74
|
+
keys = self.rotation[generation]
|
75
|
+
diff = keys.size - keep
|
76
|
+
|
77
|
+
1.upto( diff ) do
|
78
|
+
extra_key = keys.shift
|
79
|
+
delete extra_key
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
# Create a new index representing our backup hierarchy
|
86
|
+
def init_rotation_index(hierarchy)
|
87
|
+
hash = {}
|
88
|
+
hierarchy.each do |m|
|
89
|
+
hash[m] = Array.new
|
90
|
+
end
|
91
|
+
hash
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
data/lib/backup/ssh_helpers.rb
CHANGED
@@ -96,8 +96,9 @@ module Backup
|
|
96
96
|
def connect
|
97
97
|
c[:servers].each do |server| # todo, make this actually work
|
98
98
|
@session = Net::SSH.start(
|
99
|
-
server,
|
100
|
-
c[:
|
99
|
+
server,
|
100
|
+
:port => c[:port],
|
101
|
+
:username => c[:ssh_user],
|
101
102
|
:host_key => "ssh-rsa",
|
102
103
|
:keys => [ c[:identity_key] ],
|
103
104
|
:auth_methods => %w{ publickey } )
|
data/lib/backup.rb
CHANGED
@@ -5,3 +5,12 @@ require 'backup/configuration'
|
|
5
5
|
require 'backup/extensions'
|
6
6
|
require 'backup/ssh_helpers'
|
7
7
|
require 'backup/date_parser'
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'aws/s3'
|
11
|
+
require 'backup/s3_helpers'
|
12
|
+
rescue LoadError
|
13
|
+
# If AWS::S3 is not installed, no worries, we just
|
14
|
+
# wont have access to s3 methods. It's worth noting
|
15
|
+
# at least version 1.8.4 of ruby is required for s3.
|
16
|
+
end
|
data/tests/s3_test.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.dirname(__FILE__) + "/tests_helper"
|
2
|
+
|
3
|
+
class S3Test < Test::Unit::TestCase
|
4
|
+
|
5
|
+
# These tests require actual S3 access keys set as environment variables
|
6
|
+
|
7
|
+
def setup
|
8
|
+
@config = Backup::Configuration.new
|
9
|
+
@config.load "standard"
|
10
|
+
@config.set :backup_path, 'test_backup'
|
11
|
+
@config.action :deliver, :method => :s3
|
12
|
+
@actor = Backup::S3Actor.new(@config)
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_exists
|
16
|
+
assert @config
|
17
|
+
assert @actor
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_on_s3
|
21
|
+
dir = create_tmp_files
|
22
|
+
config = <<-END
|
23
|
+
action :content, :is_folder => "#{dir}"
|
24
|
+
action :rotate, :method => :via_s3
|
25
|
+
END
|
26
|
+
@config.load :string => config
|
27
|
+
assert result = @config.actor.content
|
28
|
+
assert File.exists?(result)
|
29
|
+
@config.actor.start_process!
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def create_tmp_files
|
35
|
+
newtmp = @config[:tmp_dir] + "/test_#{rand}_" + Time.now.strftime("%Y%m%d%H%M%S")
|
36
|
+
sh "mkdir #{newtmp}"
|
37
|
+
0.upto(5) { |i| sh "touch #{newtmp}/#{i}" }
|
38
|
+
newtmp
|
39
|
+
end
|
40
|
+
end
|
metadata
CHANGED
@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
|
|
3
3
|
specification_version: 1
|
4
4
|
name: backupgem
|
5
5
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 0.0.
|
7
|
-
date:
|
6
|
+
version: 0.0.5
|
7
|
+
date: 2007-01-26 00:00:00 -08:00
|
8
8
|
summary: Beginning-to-end solution for backups and rotation.
|
9
9
|
require_paths:
|
10
10
|
- lib
|
@@ -40,29 +40,35 @@ files:
|
|
40
40
|
- lib/backup/extensions.rb
|
41
41
|
- lib/backup/recipes
|
42
42
|
- lib/backup/rotator.rb
|
43
|
+
- lib/backup/s3_helpers.rb
|
43
44
|
- lib/backup/ssh_helpers.rb
|
44
45
|
- lib/backup/recipes/standard.rb
|
45
46
|
- tests/actor_test.rb
|
46
47
|
- tests/cleanup.sh
|
47
48
|
- tests/rotation_test.rb
|
49
|
+
- tests/s3_test.rb
|
48
50
|
- tests/ssh_test.rb
|
49
51
|
- tests/tests_helper.rb
|
50
52
|
- examples/global.rb
|
51
53
|
- examples/mediawiki.rb
|
54
|
+
- examples/s3.rb
|
52
55
|
- doc/index.html
|
53
56
|
- doc/LICENSE-GPL
|
54
57
|
- doc/styles.css
|
55
58
|
- README
|
56
59
|
- CHANGELOG
|
60
|
+
- TODO
|
57
61
|
test_files:
|
58
62
|
- tests/actor_test.rb
|
59
63
|
- tests/rotation_test.rb
|
64
|
+
- tests/s3_test.rb
|
60
65
|
- tests/ssh_test.rb
|
61
66
|
rdoc_options: []
|
62
67
|
|
63
68
|
extra_rdoc_files:
|
64
69
|
- README
|
65
70
|
- CHANGELOG
|
71
|
+
- TODO
|
66
72
|
executables: []
|
67
73
|
|
68
74
|
extensions: []
|