dreamback 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/dreamback/backup.rb +88 -14
- data/lib/dreamback/exclusions.txt +2 -1
- data/lib/dreamback/initializer.rb +35 -2
- data/lib/dreamback/version.rb +1 -1
- data/test/dreamback_test.rb +74 -1
- metadata +4 -4
data/lib/dreamback/backup.rb
CHANGED
@@ -39,20 +39,24 @@ module Dreamback
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
-
# Get
|
42
|
+
# Get yesterday's folder to link against
|
43
43
|
backup_to_link = backup_folders.first[0]
|
44
|
+
# If this would link us to the same folder, don't do that. Try yesterday's instead.
|
45
|
+
if backup_to_link.eql? "dreamback." + Date.today.strftime("%Y%m%d")
|
46
|
+
backup_to_link = "dreamback." + (Date.today - 1).strftime("%Y%m%d")
|
47
|
+
end
|
44
48
|
|
45
49
|
# Delete any folders older than our limit
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
folders_to_delete =
|
52
|
-
folders_to_delete.map! {|f| f[0]}
|
53
|
-
rsync_delete(folders_to_delete)
|
50
|
+
if Dreamback.settings[:days_to_keep]
|
51
|
+
folders_to_delete = rotate_daily(backup_folders)
|
52
|
+
elsif Dreamback.settings[:keep_time_machine]
|
53
|
+
folders_to_delete = rotate_time_machine(backup_folders)[:delete]
|
54
|
+
else
|
55
|
+
folders_to_delete = nil
|
54
56
|
end
|
57
|
+
rsync_delete(folders_to_delete)
|
55
58
|
end
|
59
|
+
|
56
60
|
backup_to_link
|
57
61
|
end
|
58
62
|
|
@@ -60,6 +64,7 @@ module Dreamback
|
|
60
64
|
# We do this because sftp has no recursive delete method
|
61
65
|
# @params [Array[String]] list of backup directories
|
62
66
|
def self.rsync_delete(directories)
|
67
|
+
return if directories.nil? || directories.empty?
|
63
68
|
empty_dir_path = File.expand_path("../.dreamback_empty_dir", __FILE__)
|
64
69
|
empty_dir = Dir.mkdir(empty_dir_path) unless File.exists?(empty_dir_path)
|
65
70
|
begin
|
@@ -78,6 +83,8 @@ module Dreamback
|
|
78
83
|
# @param [String] name of the most recent backup folder prior to starting this run to link against
|
79
84
|
def self.rsync_backup(link_dir)
|
80
85
|
tmp_dir_path = "~/.dreamback_tmp"
|
86
|
+
tmp_dir = File.expand_path(tmp_dir_path)
|
87
|
+
Dir.mkdir(tmp_dir) unless File.exists?(tmp_dir)
|
81
88
|
user_exclusions_path = File.expand_path("~/.dreamback_exclusions")
|
82
89
|
default_exclusions_path = File.expand_path("../exclusions.txt", __FILE__)
|
83
90
|
exclusions_path = File.exists?(user_exclusions_path) ? user_exclusions_path : default_exclusions_path
|
@@ -90,17 +97,84 @@ module Dreamback
|
|
90
97
|
user = dreamhost[:user]
|
91
98
|
server = dreamhost[:server]
|
92
99
|
# rsync won't do remote<->remote syncing, so we stage everything here first
|
93
|
-
tmp_dir = File.expand_path(tmp_dir_path)
|
94
|
-
Dir.mkdir(tmp_dir) unless File.exists?(tmp_dir)
|
95
100
|
`rsync -e ssh -av --keep-dirlinks --exclude-from #{exclusions_path} --copy-links #{user}@#{server}:~/ #{tmp_dir}/#{user}@#{server}`
|
96
|
-
# Now we can sync local to remote. Only use link-dest if a previous folder to link to exists.
|
97
|
-
link_dest = link_dir.nil? ? "" : "--link-dest=~#{BACKUP_FOLDER.gsub(".", "")}/#{link_dir}"
|
98
|
-
`rsync -e ssh -av --delete --copy-links --keep-dirlinks #{link_dest} #{tmp_dir}/ #{backup_server_user}@#{backup_server}:#{BACKUP_FOLDER}/dreamback.#{today}`
|
99
101
|
end
|
102
|
+
# Now we can sync local to remote. Only use link-dest if a previous folder to link to exists.
|
103
|
+
link_dest = link_dir.nil? ? "" : "--link-dest=~#{BACKUP_FOLDER.gsub(".", "")}/#{link_dir}"
|
104
|
+
`rsync -e ssh -av --delete --copy-links --keep-dirlinks #{link_dest} #{tmp_dir}/ #{backup_server_user}@#{backup_server}:#{BACKUP_FOLDER}/dreamback.#{today}`
|
100
105
|
ensure
|
101
106
|
# Remove the staging directory
|
102
107
|
`rm -rf #{File.expand_path(tmp_dir_path)}`
|
103
108
|
end
|
104
109
|
end
|
110
|
+
|
111
|
+
# Rotate folders based on the number of days to keep
|
112
|
+
# @param [Array] folders to rotate in Dreamback format (dreamback.20120521)
|
113
|
+
# @return [Array] list of folders to delete
|
114
|
+
def self.rotate_daily(backup_folders)
|
115
|
+
# Subtract one to account for the folder we're about to create
|
116
|
+
# Normally we remove a folder so that our count is one less than the "days to keep"
|
117
|
+
# However, if today's folder already exists then there's no need
|
118
|
+
offset = backup_folders.include?("dreamback.#{Time.now.strftime("%Y%m%d")}") ? 0 : 1
|
119
|
+
if backup_folders.length >= Dreamback.settings[:days_to_keep] - offset
|
120
|
+
folders_to_delete = backup_folders.slice(Dreamback.settings[:days_to_keep] - offset, backup_folders.length)
|
121
|
+
folders_to_delete.map! {|f| f[0]}
|
122
|
+
end
|
123
|
+
folders_to_delete ||= nil
|
124
|
+
end
|
125
|
+
|
126
|
+
# Use a time machine-like algorithm to rotate backups. This will keep daily backups for seven days, weeklies for three months, and monthlies forever
|
127
|
+
# @param [Date] the day to count as "today", where the rotation will start
|
128
|
+
# @param [Array] folders to rotate in Dreamback format (dreamback.20120521)
|
129
|
+
# @return [Hash] :keep => list of folders to retain, :delete => folders marked for deletion
|
130
|
+
def self.rotate_time_machine(folders, today = nil)
|
131
|
+
today ||= Date.today
|
132
|
+
ymd = "%Y%m%d"
|
133
|
+
|
134
|
+
folder_data = {}
|
135
|
+
folders.each do |f|
|
136
|
+
folder_data[f] = {
|
137
|
+
:date => Date.strptime(f[1].to_s, ymd)
|
138
|
+
}
|
139
|
+
end
|
140
|
+
|
141
|
+
keep = []
|
142
|
+
one_week_ago = today - 7
|
143
|
+
three_months_ago = today << 3
|
144
|
+
|
145
|
+
week_hash = {}
|
146
|
+
month_hash = {}
|
147
|
+
folder_data.each do |k, v|
|
148
|
+
if v[:date] >= one_week_ago
|
149
|
+
keep << k
|
150
|
+
elsif v[:date] >= three_months_ago
|
151
|
+
week_hash[v[:date].cweek] = [] if week_hash[v[:date].cweek].nil?
|
152
|
+
week_hash[v[:date].cweek] << [k, v]
|
153
|
+
else
|
154
|
+
ym = v[:date].strftime("%Y%m").to_i
|
155
|
+
month_hash[ym] = [] if month_hash[ym].nil?
|
156
|
+
month_hash[ym] << [k, v]
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
week_hash.each do |week, dates|
|
161
|
+
dates.sort! {|a, b| a[1][:date] <=> b[1][:date]}
|
162
|
+
end
|
163
|
+
|
164
|
+
week_hash.each do |week, dates|
|
165
|
+
keep << dates.shift[0] unless dates.empty?
|
166
|
+
end
|
167
|
+
|
168
|
+
month_hash.each do |month, dates|
|
169
|
+
dates.sort! {|a, b| a[1][:date] <=> b[1][:date]}
|
170
|
+
end
|
171
|
+
|
172
|
+
month_hash.each do |month, dates|
|
173
|
+
keep << dates.shift[0] unless dates.empty?
|
174
|
+
end
|
175
|
+
|
176
|
+
keep.sort!.compact!
|
177
|
+
{ :keep => keep, :delete => folders - keep}
|
178
|
+
end
|
105
179
|
end
|
106
180
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'net/sftp'
|
3
3
|
require 'cronedit'
|
4
|
+
require 'highline/import'
|
4
5
|
|
5
6
|
module Dreamback
|
6
7
|
@settings
|
@@ -35,7 +36,20 @@ module Dreamback
|
|
35
36
|
save_settings(SETTINGS_LOCATION)
|
36
37
|
else
|
37
38
|
if direct
|
38
|
-
say(bold("You have already setup Dreamback.
|
39
|
+
say(bold("You have already setup Dreamback."))
|
40
|
+
|
41
|
+
choose("From here you can: \n") do |menu|
|
42
|
+
menu.prompt = "Select an option: "
|
43
|
+
|
44
|
+
menu.choice("Run setup again") {
|
45
|
+
create_new_settings
|
46
|
+
save_settings(SETTINGS_LOCATION)
|
47
|
+
}
|
48
|
+
|
49
|
+
menu.choice("Run a backup right now") {
|
50
|
+
Dreamback::Backup.start
|
51
|
+
}
|
52
|
+
end
|
39
53
|
end
|
40
54
|
end
|
41
55
|
|
@@ -125,6 +139,7 @@ module Dreamback
|
|
125
139
|
def self.create_new_settings
|
126
140
|
settings = {}
|
127
141
|
say("#{bold("Server Where We Should Store Your Backup")}\nYour dreamhost backup-specific account will work best, but any POSIX server with rsync should work\n<%= color('Note:', BOLD)%> dreamhost does not allow you to store non-webhosted data except your BACKUP-SPECIFIC account")
|
142
|
+
say("You can configure your Dreamhost backup user here: https://panel.dreamhost.com/index.cgi?tree=users.backup")
|
128
143
|
settings[:backup_server] = ask("Server name: ")
|
129
144
|
settings[:backup_server_user] = ask(bold("Username for the backup server: "))
|
130
145
|
settings[:dreamhost_users] = []
|
@@ -136,7 +151,25 @@ module Dreamback
|
|
136
151
|
settings[:dreamhost_users] << dreamhost
|
137
152
|
another_user = agree(bold("Add another dreamhost account? [y/n]"))
|
138
153
|
end
|
139
|
-
|
154
|
+
|
155
|
+
choose("What method should we use to keep backups? \n") do |menu|
|
156
|
+
menu.prompt = "Select an option "
|
157
|
+
|
158
|
+
menu.choice("Number of days") {
|
159
|
+
settings[:days_to_keep] = ask(bold("How many days of backups do you want to keep [1-9999]? "), Integer) {|q| q.in = 1..9999}
|
160
|
+
say("#{bold("We will keep #{settings[:days_to_keep]} days of backups")}")
|
161
|
+
}
|
162
|
+
|
163
|
+
menu.choice("Time machine-inspired\n (daily backups for seven days, weekly backups for three months, monthly backups forever)") {
|
164
|
+
settings[:keep_time_machine] = true
|
165
|
+
say("#{bold("We will keep backups based on a time machine-inspired method")}")
|
166
|
+
}
|
167
|
+
|
168
|
+
menu.choice("Keep everything") {
|
169
|
+
say("#{bold("We will keep everything. This may take a lot of space.")}")
|
170
|
+
}
|
171
|
+
end
|
172
|
+
|
140
173
|
@settings = settings
|
141
174
|
end
|
142
175
|
|
data/lib/dreamback/version.rb
CHANGED
data/test/dreamback_test.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
require "rubygems"
|
2
|
+
require "json"
|
2
3
|
require "test/unit"
|
3
4
|
require File.expand_path('../../lib/dreamback/initializer', __FILE__)
|
4
|
-
require
|
5
|
+
require File.expand_path('../../lib/dreamback/backup', __FILE__)
|
5
6
|
|
6
7
|
# Used for testing private methods
|
7
8
|
class Class
|
@@ -55,4 +56,76 @@ class DreambackTest < Test::Unit::TestCase
|
|
55
56
|
assert_equal settings_file, JSON.parse(settings_test, :symbolize_names => true)
|
56
57
|
end
|
57
58
|
|
59
|
+
def test_time_machine_rotate
|
60
|
+
folders = [
|
61
|
+
"dreamback.20120514",
|
62
|
+
"dreamback.20120513",
|
63
|
+
"dreamback.20120512",
|
64
|
+
"dreamback.20120511",
|
65
|
+
"dreamback.20120510",
|
66
|
+
"dreamback.20120509",
|
67
|
+
"dreamback.20120508",
|
68
|
+
"dreamback.20120501",
|
69
|
+
"dreamback.20120424",
|
70
|
+
"dreamback.20120413",
|
71
|
+
"dreamback.20120417",
|
72
|
+
"dreamback.20120411",
|
73
|
+
"dreamback.20120410",
|
74
|
+
"dreamback.20120403",
|
75
|
+
"dreamback.20120327",
|
76
|
+
"dreamback.20120325",
|
77
|
+
"dreamback.20120320",
|
78
|
+
"dreamback.20120321",
|
79
|
+
"dreamback.20120313",
|
80
|
+
"dreamback.20120312",
|
81
|
+
"dreamback.20120309",
|
82
|
+
"dreamback.20120308",
|
83
|
+
"dreamback.20120301",
|
84
|
+
"dreamback.20120227",
|
85
|
+
"dreamback.20120215",
|
86
|
+
"dreamback.20120207",
|
87
|
+
"dreamback.20120208"
|
88
|
+
]
|
89
|
+
|
90
|
+
folders_sorted = {
|
91
|
+
:keep=>
|
92
|
+
[
|
93
|
+
"dreamback.20120207",
|
94
|
+
"dreamback.20120215",
|
95
|
+
"dreamback.20120227",
|
96
|
+
"dreamback.20120308",
|
97
|
+
"dreamback.20120312",
|
98
|
+
"dreamback.20120320",
|
99
|
+
"dreamback.20120327",
|
100
|
+
"dreamback.20120403",
|
101
|
+
"dreamback.20120410",
|
102
|
+
"dreamback.20120417",
|
103
|
+
"dreamback.20120424",
|
104
|
+
"dreamback.20120501",
|
105
|
+
"dreamback.20120508",
|
106
|
+
"dreamback.20120509",
|
107
|
+
"dreamback.20120510",
|
108
|
+
"dreamback.20120511",
|
109
|
+
"dreamback.20120512",
|
110
|
+
"dreamback.20120513",
|
111
|
+
"dreamback.20120514"
|
112
|
+
],
|
113
|
+
:delete=>
|
114
|
+
[
|
115
|
+
"dreamback.20120413",
|
116
|
+
"dreamback.20120411",
|
117
|
+
"dreamback.20120325",
|
118
|
+
"dreamback.20120321",
|
119
|
+
"dreamback.20120313",
|
120
|
+
"dreamback.20120309",
|
121
|
+
"dreamback.20120301",
|
122
|
+
"dreamback.20120208"
|
123
|
+
]
|
124
|
+
}
|
125
|
+
|
126
|
+
today = Date.strptime("20120514", "%Y%m%d")
|
127
|
+
|
128
|
+
assert folders_sorted == Dreamback::Backup.rotate_time_machine(today, folders)
|
129
|
+
end
|
130
|
+
|
58
131
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dreamback
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 21
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 5
|
10
|
+
version: 0.0.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Paul R Alexander
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-
|
18
|
+
date: 2012-06-25 00:00:00 -07:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|