zfs_mgmt 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.travis.yml +7 -0
- data/COPYING +674 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +41 -0
- data/README.md +191 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/readsnaps +4 -0
- data/bin/setup +8 -0
- data/bin/zfs-list-snapshots +10 -0
- data/bin/zfsfuncs +126 -0
- data/bin/zfsmgr +40 -0
- data/bin/zfsrecvman +154 -0
- data/bin/zfssendman +252 -0
- data/bin/zfssnapman +69 -0
- data/lib/zfs_mgmt/version.rb +3 -0
- data/lib/zfs_mgmt.rb +314 -0
- data/zfs_mgmt.gemspec +46 -0
- metadata +156 -0
data/bin/zfssendman
ADDED
@@ -0,0 +1,252 @@
|
|
1
|
+
#! /bin/bash
|
2
|
+
|
3
|
+
export PATH=$PATH:/sbin
|
4
|
+
|
5
|
+
FILTER='.'
|
6
|
+
USER='root'
|
7
|
+
SEND='send'
|
8
|
+
RECV='recv -e -F'
|
9
|
+
FLOCK='/usr/bin/flock -w 60 -n 9'
|
10
|
+
PORT='1337'
|
11
|
+
MBUFFER='-s 128k -m 1G -4'
|
12
|
+
USE_MBUFFER='no'
|
13
|
+
LOCK_DIR='/var/run/'$(basename $0)
|
14
|
+
test -d $LOCK_DIR || mkdir $LOCK_DIR
|
15
|
+
TEST=0
|
16
|
+
VERB=0
|
17
|
+
|
18
|
+
test -f $HOME/.keychain/$HOSTNAME-sh && . $HOME/.keychain/$HOSTNAME-sh
|
19
|
+
|
20
|
+
function zfssendrecv {
|
21
|
+
local OPTIND OPTARG opt
|
22
|
+
local zfs
|
23
|
+
local snap
|
24
|
+
local dest
|
25
|
+
local inc=''
|
26
|
+
local hold=0
|
27
|
+
while getopts "z:s:d:i:I:h" opt; do
|
28
|
+
case $opt in
|
29
|
+
z)
|
30
|
+
zfs=$OPTARG
|
31
|
+
;;
|
32
|
+
s)
|
33
|
+
snap=$OPTARG
|
34
|
+
;;
|
35
|
+
d)
|
36
|
+
dest=$OPTARG
|
37
|
+
;;
|
38
|
+
i)
|
39
|
+
inc="-i ${OPTARG}"
|
40
|
+
;;
|
41
|
+
I)
|
42
|
+
inc="-I ${OPTARG}"
|
43
|
+
;;
|
44
|
+
h)
|
45
|
+
hold=1
|
46
|
+
;;
|
47
|
+
esac
|
48
|
+
done
|
49
|
+
shift $((OPTIND-1))
|
50
|
+
local zfs_normal=$( echo $zfs|sed 's/[\:\|\/\\ ]/_/g' )
|
51
|
+
local lock="${LOCK_DIR}/${zfs_normal}.lock"
|
52
|
+
local zfs_recv_status
|
53
|
+
local zfs_send_status
|
54
|
+
local pipe_status
|
55
|
+
(
|
56
|
+
if ! $FLOCK; then
|
57
|
+
$ulog "unable to lock ${lock}"
|
58
|
+
return -2
|
59
|
+
fi
|
60
|
+
if [[ $TEST == 0 && $hold == 1 ]]; then
|
61
|
+
zfs hold -r zfsrecvman $snap 2>&1 | $ulog
|
62
|
+
local hold_status="${PIPESTATUS[0]}"
|
63
|
+
if [[ $hold_status != 0 ]]; then
|
64
|
+
$ulog "unable to place a hold on our snapshots: ${snap}"
|
65
|
+
return -3
|
66
|
+
fi
|
67
|
+
fi
|
68
|
+
$ulog "estimating size of sending ${snap}"
|
69
|
+
local size=$( zfs $SEND -v -n $inc $snap 2>&1 | tail -1 | cut -d" " -f 5 )
|
70
|
+
# could be 0 or 400 or 4K or 9.3g, etc.
|
71
|
+
local suf=$( echo $size | sed -E 's/[0-9]+\.?[0-9]*//' | tr '[:lower:]' '[:upper:]' )
|
72
|
+
size=$( echo $size | sed -E 's/[pPtTgGmMkKB]$//' ) # remove known suffixes
|
73
|
+
if [[ $suf != 'B' ]]; then
|
74
|
+
size=$( echo "${size} * 1024" | bc | sed -E 's/\.[0-9]+//' ) # use bc to multiply decimals, sed to make ceil()
|
75
|
+
fi
|
76
|
+
case $suf in
|
77
|
+
B)
|
78
|
+
suf=''
|
79
|
+
;;
|
80
|
+
K)
|
81
|
+
suf=''
|
82
|
+
;;
|
83
|
+
M)
|
84
|
+
suf='K'
|
85
|
+
;;
|
86
|
+
G)
|
87
|
+
suf='M'
|
88
|
+
;;
|
89
|
+
T)
|
90
|
+
suf='G'
|
91
|
+
;;
|
92
|
+
P)
|
93
|
+
suf='T'
|
94
|
+
;;
|
95
|
+
esac
|
96
|
+
$ulog "estimated size of sending ${snap} is ${size}${suf}"
|
97
|
+
local pv_more="-s ${size}${suf}"
|
98
|
+
if [[ $USE_MBUFFER == 'yes' ]]; then
|
99
|
+
ssh "${USER}@${REMOTE}" "mbuffer ${MBUFFER} -q -I ${PORT} | zfs ${RECV} ${dest}" 2>&1 | $ulog &
|
100
|
+
sleep 5
|
101
|
+
zfs $SEND $inc $snap 2> >($ulog)|
|
102
|
+
mbuffer $MBUFFER $MBUFFER_SEND_OPTS -O ${REMOTE}:${PORT}
|
103
|
+
zfs_send_status="${PIPESTATUS[0]}"
|
104
|
+
$ulog "zfs send exited with status: ${zfs_send_status}"
|
105
|
+
$ulog "about to wait on zfs send (this may take a while and appear to have hung)"
|
106
|
+
wait
|
107
|
+
zfs_recv_status="${PIPESTATUS[0]}"
|
108
|
+
$ulog "zfs recv exited with status: ${zfs_recv_status}"
|
109
|
+
else
|
110
|
+
zfs $SEND $inc $snap 2> >($ulog) | pv $PV_OPTS $pv_more | ssh "${USER}@${REMOTE}" "zfs ${RECV} ${dest}" 2>&1 | $ulog
|
111
|
+
pipe_status=("${PIPESTATUS[@]}")
|
112
|
+
zfs_send_status="${pipe_status[0]}"
|
113
|
+
zfs_recv_status="${pipe_status[2]}"
|
114
|
+
$ulog "zfs send exited with status: ${zfs_send_status}"
|
115
|
+
$ulog "zfs recv exited with status: ${zfs_recv_status}"
|
116
|
+
fi
|
117
|
+
if [[ $zfs_send_status != 0 ]]; then
|
118
|
+
return $zfs_send_status
|
119
|
+
elif [[ $zfs_recv_status != 0 ]]; then
|
120
|
+
return $zfs_recv_status
|
121
|
+
else
|
122
|
+
# both must be zero
|
123
|
+
return 0
|
124
|
+
fi
|
125
|
+
) 9>$lock
|
126
|
+
}
|
127
|
+
|
128
|
+
function terminal_options {
|
129
|
+
if [ -t 1 ]; then
|
130
|
+
ISTERM=1
|
131
|
+
LOGGER_EXTRA='-s' # enable logger output to stderr
|
132
|
+
PV_OPTS='-perb' # enable all the magic output from pv
|
133
|
+
MBUFFER_SEND_OPTS='' # don't do quiet mode, we have a term
|
134
|
+
ulog="logger ${LOGGER_EXTRA} -p user.notice -t "$(basename $0 2>/dev/null)"[${$}]"
|
135
|
+
else
|
136
|
+
ISTERM=0
|
137
|
+
LOGGER_EXTRA='' # don't enable stderr output
|
138
|
+
PV_OPTS='-q' # make pv quiet
|
139
|
+
MBUFFER_SEND_OPTS='-q' # enable send side -q, no terminal
|
140
|
+
ulog="logger ${LOGGER_EXTRA} -p user.notice -t "$(basename $0 2>/dev/null)"[${$}]"
|
141
|
+
fi
|
142
|
+
}
|
143
|
+
|
144
|
+
terminal_options
|
145
|
+
|
146
|
+
while getopts "p:f:L:mnvr:u:d:" opt; do
|
147
|
+
case $opt in
|
148
|
+
p)
|
149
|
+
PORT=$OPTARG
|
150
|
+
;;
|
151
|
+
f)
|
152
|
+
FILTER=$OPTARG
|
153
|
+
;;
|
154
|
+
L)
|
155
|
+
PV_OPTS="${PV_OPTS} -L ${OPTARG}"
|
156
|
+
;;
|
157
|
+
m)
|
158
|
+
USE_MBUFFER='yes'
|
159
|
+
;;
|
160
|
+
n)
|
161
|
+
RECV="${RECV} -n"
|
162
|
+
TEST=1
|
163
|
+
VERB=1
|
164
|
+
PV_OPTS='-q' # make pv quiet
|
165
|
+
MBUFFER_SEND_OPTS='-q' # enable send side -q, no terminal
|
166
|
+
;;
|
167
|
+
v)
|
168
|
+
VERB=1
|
169
|
+
;;
|
170
|
+
r)
|
171
|
+
REMOTE=$OPTARG
|
172
|
+
;;
|
173
|
+
u)
|
174
|
+
USER=$OPTARG
|
175
|
+
;;
|
176
|
+
d)
|
177
|
+
DEST=$OPTARG
|
178
|
+
;;
|
179
|
+
|
180
|
+
esac
|
181
|
+
done
|
182
|
+
if [[ "${REMOTE}Z" == 'Z' ]]; then
|
183
|
+
echo 'must set remote with -r option'
|
184
|
+
exit 1;
|
185
|
+
fi
|
186
|
+
if [[ "${DEST}Z" == 'Z' ]]; then
|
187
|
+
echo 'must set dest with -d option'
|
188
|
+
exit 1;
|
189
|
+
fi
|
190
|
+
|
191
|
+
if [[ $VERB == 1 ]]; then
|
192
|
+
echo $RECV | grep -q -- -v || RECV="${RECV} -v"
|
193
|
+
fi
|
194
|
+
|
195
|
+
function set_options {
|
196
|
+
local zfs=$1
|
197
|
+
local target=$2
|
198
|
+
local compress=$( zfs get -H -o value compression $zfs )
|
199
|
+
local destroy=$( zfs get -H -o value zfssnapman:destroy $zfs )
|
200
|
+
ssh "${USER}@${REMOTE}" "zfs set readonly=on ${target} ; zfs set compression=${compress} ${target} ; zfs set zfssnapman:destroy=${destroy} ${target}"
|
201
|
+
}
|
202
|
+
|
203
|
+
# zfs is the filesystem to be sent, including source pool
|
204
|
+
for zfs in $( zfs list -H -t filesystem,volume -o name,zfssendman:send | grep true | egrep "${FILTER}" | awk -F\\t '{if($2 == "true") {print $1;}}' | sort ); do
|
205
|
+
# base is the zfs filesystem to send, minus the pool
|
206
|
+
target="${DEST}/${zfs}"
|
207
|
+
target_dir=$( dirname $target )
|
208
|
+
# recv_last is the last snapshot on the recv side of this zfs
|
209
|
+
if ! ssh "${USER}@${REMOTE}" zfs get written $target >/dev/null 2>/dev/null; then
|
210
|
+
$ulog sending initial snapshot of $zfs to $target_dir
|
211
|
+
zfssendrecv -z $zfs \
|
212
|
+
-s $( zfs list -t snapshot -o name -s creation -d 1 -H $zfs | head -1 ) \
|
213
|
+
-d $target_dir
|
214
|
+
if [[ $TEST == 0 ]]; then
|
215
|
+
set_options $zfs $target
|
216
|
+
fi
|
217
|
+
sleep 5
|
218
|
+
fi
|
219
|
+
|
220
|
+
recv_last=$( ssh "${USER}@${REMOTE}" zfs list -t snapshot -o name -s creation -d 1 -H $target | tail -1 ) || break;
|
221
|
+
if ! echo $recv_last | grep -q "${zfs}@"; then
|
222
|
+
$ulog "no snapshot on distant end, you must destroy the filesystem: $zfs"
|
223
|
+
break;
|
224
|
+
fi
|
225
|
+
# send is the snapshot on the recv side after stripping off the DEST
|
226
|
+
send=$( echo $recv_last|sed "s|$DEST/||" )
|
227
|
+
|
228
|
+
zfs list -t snapshot -o name -d 1 -H $zfs | grep -q $send
|
229
|
+
if [[ $? == 0 ]]; then
|
230
|
+
# most recent snapshot on the send side
|
231
|
+
current=$( zfs list -t snapshot -o name -s creation -d 1 -H $zfs | tail -1 )
|
232
|
+
if [[ $send == $current ]]; then
|
233
|
+
$ulog "${zfs} is in sync on source and destination (${target})"
|
234
|
+
else
|
235
|
+
$ulog sending $send through $current to $target
|
236
|
+
zfssendrecv -z $zfs \
|
237
|
+
-I $send \
|
238
|
+
-s $current \
|
239
|
+
-d $target_dir
|
240
|
+
if [[ $? == 0 ]]; then
|
241
|
+
$ulog "$zfs is in sync on source and destination"
|
242
|
+
if [[ $TEST == 0 ]]; then
|
243
|
+
set_options $zfs $target
|
244
|
+
fi
|
245
|
+
else
|
246
|
+
$ulog zfs exited with $? while sending $send through $current to $target
|
247
|
+
fi
|
248
|
+
fi
|
249
|
+
else
|
250
|
+
$ulog "the most recent snapshot ($recv_last) on the recv side does not exist on the send side ($send)"
|
251
|
+
fi
|
252
|
+
done
|
data/bin/zfssnapman
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
#! /usr/bin/perl
|
2
|
+
|
3
|
+
use strict;
|
4
|
+
use warnings;
|
5
|
+
|
6
|
+
use Getopt::Long;
|
7
|
+
use POSIX qw( strftime mktime );
|
8
|
+
|
9
|
+
my %months=(Jan => 0,
|
10
|
+
Feb => 1,
|
11
|
+
Mar => 2,
|
12
|
+
Apr => 3,
|
13
|
+
May => 4,
|
14
|
+
Jun => 5,
|
15
|
+
Jul => 6,
|
16
|
+
Aug => 7,
|
17
|
+
Sep => 8,
|
18
|
+
Oct => 9,
|
19
|
+
Nov => 10,
|
20
|
+
Dec => 11,
|
21
|
+
);
|
22
|
+
|
23
|
+
my $DESTROY=0;
|
24
|
+
my $SNAP=1;
|
25
|
+
my $VERBOSE=0;
|
26
|
+
|
27
|
+
GetOptions('snap!' => \$SNAP,
|
28
|
+
'destroy!' => \$DESTROY,
|
29
|
+
'v|verbose'=> \$VERBOSE,
|
30
|
+
);
|
31
|
+
|
32
|
+
my $time=strftime('%F%T',localtime(time()));
|
33
|
+
$time =~ s/[\:\-]//g;
|
34
|
+
|
35
|
+
foreach(`/sbin/zfs list -t filesystem,volume -o name,zfssnapman:snap,zfssnapman:destroy -H`) {
|
36
|
+
chomp;
|
37
|
+
my ($zfs,$snap,$days)=split(/\t/);
|
38
|
+
if($SNAP and ($snap eq 'true' or $snap eq 'on')) {
|
39
|
+
my $com="/sbin/zfs snapshot $zfs\@zfssnapman-$time";
|
40
|
+
$VERBOSE and print $com."\n";
|
41
|
+
system($com);
|
42
|
+
if($? != 0) {
|
43
|
+
warn "unable to create zfs snapshot for $zfs at $time";
|
44
|
+
warn "failed command: $com";
|
45
|
+
}
|
46
|
+
}
|
47
|
+
if($DESTROY and $days =~ /\d+/) {
|
48
|
+
foreach(`/sbin/zfs list -t snapshot -o name,creation -s creation -r -H $zfs`) {
|
49
|
+
chomp;
|
50
|
+
my ($snap,$creation)=split(/\t/);
|
51
|
+
next unless $snap =~ /^$zfs\@/;
|
52
|
+
unless($creation =~ /\s+(\w\w\w)\s+(\d+)\s+(\d+)\:(\d+)\s+(\d\d\d\d)/) {
|
53
|
+
die "unable to parse the date: $creation";
|
54
|
+
}
|
55
|
+
my $age=mktime(0,$4,$3,$2,$months{$1},($5-1900));
|
56
|
+
#printf("%s\t%s\t%s\t%d\t%s\n",strftime('%F %T',localtime(time())),$creation,strftime('%F %T',localtime($age)),$days,strftime('%F %T',localtime($age + ($days * 24 * 60 * 60 ))));
|
57
|
+
if(time() > ($age + ($days * 24 * 60 * 60 ))) {
|
58
|
+
my $comment=sprintf("removing: %s created at %s\n",$snap,strftime('%F %T',localtime($age)));
|
59
|
+
my $command="/sbin/zfs destroy $snap";
|
60
|
+
$VERBOSE and print $comment;
|
61
|
+
system($command);
|
62
|
+
if($? != 0) {
|
63
|
+
warn 'failed '.$comment;
|
64
|
+
warn "failed command: $command";
|
65
|
+
}
|
66
|
+
}
|
67
|
+
}
|
68
|
+
}
|
69
|
+
}
|
data/lib/zfs_mgmt.rb
ADDED
@@ -0,0 +1,314 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require "zfs_mgmt/version"
|
3
|
+
require 'pp'
|
4
|
+
require 'date'
|
5
|
+
require 'logger'
|
6
|
+
require 'text-table'
|
7
|
+
require 'open3'
|
8
|
+
require 'filesize'
|
9
|
+
|
10
|
+
$logger = Logger.new(STDERR)
|
11
|
+
|
12
|
+
$date_patterns = {
|
13
|
+
'hourly' => '%F Hour %H',
|
14
|
+
'daily' => '%F',
|
15
|
+
'weekly' => '%Y Week %U', # week, starting on sunday
|
16
|
+
'monthly' => '%Y-%m',
|
17
|
+
'yearly' => '%Y',
|
18
|
+
}
|
19
|
+
|
20
|
+
$time_pattern_map = {}
|
21
|
+
$date_patterns.keys.each do |tf|
|
22
|
+
$time_pattern_map[tf[0]] = tf
|
23
|
+
end
|
24
|
+
|
25
|
+
$time_specs = {
|
26
|
+
's' => 1,
|
27
|
+
'm' => 60,
|
28
|
+
'h' => 60*60,
|
29
|
+
'd' => 24*60*60,
|
30
|
+
'w' => 7*24*60*60,
|
31
|
+
}
|
32
|
+
|
33
|
+
$properties_xlate = {
|
34
|
+
'userrefs' => ->(x) { x.to_i },
|
35
|
+
'creation' => ->(x) { x.to_i },
|
36
|
+
}
|
37
|
+
|
38
|
+
module ZfsMgmt
|
39
|
+
def self.custom_properties()
|
40
|
+
return [
|
41
|
+
'policy',
|
42
|
+
'manage',
|
43
|
+
'strategy',
|
44
|
+
'minage',
|
45
|
+
'matchsnaps',
|
46
|
+
'ignoresnaps',
|
47
|
+
'snapshot',
|
48
|
+
'snap_prefix',
|
49
|
+
'snap_timestamp',
|
50
|
+
].map do |p|
|
51
|
+
['zfsmgmt',p].join(':')
|
52
|
+
end
|
53
|
+
end
|
54
|
+
def self.timespec_to_seconds(spec)
|
55
|
+
md = /^(\d+)([smhdw]?)/i.match(spec)
|
56
|
+
unless md.length == 3
|
57
|
+
raise 'SpecParseError'
|
58
|
+
end
|
59
|
+
if md[2] and md[2].length > 0
|
60
|
+
return md[1].to_i * $time_specs[md[2].downcase]
|
61
|
+
else
|
62
|
+
return md[1].to_i
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.zfsget(properties: ['name'],types: ['filesystem','volume'],zfs: '')
|
67
|
+
results={}
|
68
|
+
com = ['zfs', 'get', '-Hp', properties.join(','), '-t', types.join(','), zfs]
|
69
|
+
so,se,status = Open3.capture3(com.join(' '))
|
70
|
+
if status.signaled?
|
71
|
+
$logger.error("process was signalled \"#{com.join(' ')}\", termsig #{status.termsig}")
|
72
|
+
raise 'ZfsGetError'
|
73
|
+
end
|
74
|
+
unless status.success?
|
75
|
+
$logger.error("failed to execute \"#{com.join(' ')}\", exit status #{status.exitstatus}")
|
76
|
+
so.split("\n").each { |l| $logger.debug("stdout: #{l}") }
|
77
|
+
se.split("\n").each { |l| $logger.error("stderr: #{l}") }
|
78
|
+
raise 'ZfsGetError'
|
79
|
+
end
|
80
|
+
so.split("\n").each do |line|
|
81
|
+
params = line.split("\t")
|
82
|
+
unless results.has_key?(params[0])
|
83
|
+
results[params[0]] = {}
|
84
|
+
end
|
85
|
+
if params[2] != '-'
|
86
|
+
if $properties_xlate.has_key?(params[1])
|
87
|
+
results[params[0]][params[1]] = $properties_xlate[params[1]].call(params[2])
|
88
|
+
else
|
89
|
+
results[params[0]][params[1]] = params[2]
|
90
|
+
end
|
91
|
+
end
|
92
|
+
if params[3] != '-'
|
93
|
+
results[params[0]]["#{params[1]}@source"] = params[3]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
return results
|
97
|
+
end
|
98
|
+
def self.local_epoch_to_datetime(e)
|
99
|
+
return Time.at(e).to_datetime
|
100
|
+
end
|
101
|
+
def self.find_saved_reason(saved,snap)
|
102
|
+
results = {}
|
103
|
+
$date_patterns.each do |d,dk|
|
104
|
+
if saved.has_key?(d)
|
105
|
+
saved[d].each do |k,s|
|
106
|
+
if snap == s
|
107
|
+
results[d]=k
|
108
|
+
break
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
return [results['hourly'],results['daily'],results['weekly'],results['monthly'],results['yearly']]
|
114
|
+
end
|
115
|
+
def self.snapshot_destroy_policy(zfs,props,snaps)
|
116
|
+
minage = 0
|
117
|
+
if props.has_key?('zfsmgmt:minage')
|
118
|
+
minage = timespec_to_seconds(props['zfsmgmt:minage'])
|
119
|
+
end
|
120
|
+
strategy = 'youngest'
|
121
|
+
if props.has_key?('zfsmgmt:strategy') and props['zfsmgmt:strategy'] == 'oldest'
|
122
|
+
strategy = 'oldest'
|
123
|
+
end
|
124
|
+
sorted = snaps.keys.sort { |a,b| snaps[b]['creation'] <=> snaps[a]['creation'] }
|
125
|
+
# never consider the latest snapshot for anything
|
126
|
+
newest_snapshot_name = sorted.shift
|
127
|
+
|
128
|
+
counters = policy_parser(props['zfsmgmt:policy'])
|
129
|
+
$logger.debug(counters)
|
130
|
+
saved = {}
|
131
|
+
|
132
|
+
# set the counters variable to track the number of saved daily/hourly/etc. snapshots
|
133
|
+
$date_patterns.each do |d,p|
|
134
|
+
saved[d] = {}
|
135
|
+
end
|
136
|
+
|
137
|
+
sorted.each do |snap_name|
|
138
|
+
if props.has_key?('zfsmgmt:ignoresnaps') and /#{props['zfsmgmt:ignoresnaps']}/ =~ snap_name.split('@')[1]
|
139
|
+
$logger.debug("skipping #{snap_name} because it matches ignoresnaps pattern: #{props['zfsmgmt:ignoresnaps']}")
|
140
|
+
next
|
141
|
+
end
|
142
|
+
if props.has_key?('zfsmgmt:matchsnaps') and not /#{props['zfsmgmt:matchsnaps']}/ =~ snap_name.split('@')[1]
|
143
|
+
$logger.debug("skipping #{snap_name} because it does not match matchsnaps pattern: #{props['zfsmgmt:matchsnaps']}")
|
144
|
+
next
|
145
|
+
end
|
146
|
+
snaptime = local_epoch_to_datetime(snaps[snap_name]['creation'])
|
147
|
+
$date_patterns.each do |d,p|
|
148
|
+
pat = snaptime.strftime(p)
|
149
|
+
if saved[d].has_key?(pat)
|
150
|
+
if strategy == 'youngest'
|
151
|
+
# update the existing current save snapshot for this timeframe
|
152
|
+
$logger.debug("updating the saved snapshot for \"#{pat}\" to #{snap_name} at #{snaptime}")
|
153
|
+
saved[d][pat] = snap_name
|
154
|
+
else
|
155
|
+
$logger.debug("not updating the saved snapshot for \"#{pat}\" to #{snap_name} at #{snaptime}, we have an older snap")
|
156
|
+
end
|
157
|
+
elsif counters[d] > 0
|
158
|
+
# new pattern, and we want to save more snaps of this type
|
159
|
+
$logger.debug("new pattern \"#{pat}\" n#{counters[d]} #{d} snapshot, saving #{snap_name} at #{snaptime}")
|
160
|
+
counters[d] -= 1
|
161
|
+
saved[d][pat] = snap_name
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
# create a list of unique saved snap shots
|
167
|
+
saved_snaps = []
|
168
|
+
saved.each do |d,saved|
|
169
|
+
saved_snaps += saved.values()
|
170
|
+
end
|
171
|
+
saved_snaps = saved_snaps.sort.uniq
|
172
|
+
|
173
|
+
# delete everything not in the list of saved snapshots
|
174
|
+
deleteme = sorted - saved_snaps
|
175
|
+
deleteme = deleteme.select { |snap|
|
176
|
+
if props.has_key?('zfsmgmt:ignoresnaps') and /#{props['zfsmgmt:ignoresnaps']}/ =~ snap
|
177
|
+
$logger.debug("skipping #{snap} because it matches ignoresnaps pattern: #{props['zfsmgmt:ignoresnaps']}")
|
178
|
+
false
|
179
|
+
elsif minage > 0 and Time.at(snaps[snap]['creation'] + minage) > Time.now()
|
180
|
+
$logger.debug("skipping due to minage: #{snap} #{local_epoch_to_datetime(snaps[snap]['creation']).strftime('%F %T')}")
|
181
|
+
false
|
182
|
+
else
|
183
|
+
true
|
184
|
+
end
|
185
|
+
}
|
186
|
+
return saved,saved_snaps,deleteme
|
187
|
+
end
|
188
|
+
def self.zfs_managed_list(filter: '.+')
|
189
|
+
zfss = [] # array of arrays
|
190
|
+
zfsget(properties: custom_properties()).each do |zfs,props|
|
191
|
+
unless /#{filter}/ =~ zfs
|
192
|
+
next
|
193
|
+
end
|
194
|
+
unless props.has_key?('zfsmgmt:manage') and props['zfsmgmt:manage'] == 'true'
|
195
|
+
next
|
196
|
+
end
|
197
|
+
snaps = self.zfsget(properties: ['name','creation','userrefs','used','written','referenced'],types: ['snapshot'], zfs: zfs)
|
198
|
+
if snaps.length == 0
|
199
|
+
$logger.warn("unable to process this zfs, no snapshots at all: #{zfs}")
|
200
|
+
next
|
201
|
+
end
|
202
|
+
unless props.has_key?('zfsmgmt:policy') and policy = policy_parser(props['zfsmgmt:policy'])
|
203
|
+
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
|
204
|
+
next # zfs
|
205
|
+
end
|
206
|
+
zfss.push([zfs,props,snaps])
|
207
|
+
end
|
208
|
+
return zfss
|
209
|
+
end
|
210
|
+
def self.snapshot_policy(verbopt: false, debugopt: false, filter: '.+')
|
211
|
+
if debugopt
|
212
|
+
$logger.level = Logger::DEBUG
|
213
|
+
else
|
214
|
+
$logger.level = Logger::INFO
|
215
|
+
end
|
216
|
+
zfs_managed_list(filter: filter).each do |zdata|
|
217
|
+
(zfs,props,snaps) = zdata
|
218
|
+
# call the function that decides who to save and who to delete
|
219
|
+
(saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
|
220
|
+
|
221
|
+
if saved_snaps.length == 0
|
222
|
+
$logger.info("no snapshots marked as saved by policy for #{zfs}")
|
223
|
+
next
|
224
|
+
end
|
225
|
+
# print a table of saved snapshots with the reasons it is being saved
|
226
|
+
table = Text::Table.new
|
227
|
+
table.head = ['snap','creation','hourly','daily','weekly','monthly','yearly']
|
228
|
+
table.rows = []
|
229
|
+
saved_snaps.sort { |a,b| snaps[b]['creation'] <=> snaps[a]['creation'] }.each do |snap|
|
230
|
+
table.rows << [snap,local_epoch_to_datetime(snaps[snap]['creation'])] + find_saved_reason(saved,snap)
|
231
|
+
end
|
232
|
+
print table.to_s
|
233
|
+
end
|
234
|
+
end
|
235
|
+
def self.snapshot_destroy(noop: false, verbopt: false, debugopt: false, filter: '.+')
|
236
|
+
if debugopt
|
237
|
+
$logger.level = Logger::DEBUG
|
238
|
+
else
|
239
|
+
$logger.level = Logger::INFO
|
240
|
+
end
|
241
|
+
zfs_managed_list(filter: filter).each do |zdata|
|
242
|
+
(zfs,props,snaps) = zdata
|
243
|
+
# call the function that decides who to save and who to delete
|
244
|
+
(saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
|
245
|
+
|
246
|
+
$logger.info("deleting #{deleteme.length} snapshots for #{zfs}")
|
247
|
+
com_base = "zfs destroy -p"
|
248
|
+
if noop
|
249
|
+
com_base = "#{com_base}n"
|
250
|
+
end
|
251
|
+
if verbopt
|
252
|
+
com_base = "#{com_base}v"
|
253
|
+
end
|
254
|
+
deleteme.reverse! # oldest first for removal
|
255
|
+
deleteme.each do |snap_name|
|
256
|
+
$logger.debug("delete: #{snap_name} #{local_epoch_to_datetime(snaps[snap_name]['creation']).strftime('%F %T')}")
|
257
|
+
end
|
258
|
+
while deleteme.length > 0
|
259
|
+
for i in 0..(deleteme.length - 1) do
|
260
|
+
max = deleteme.length - 1 - i
|
261
|
+
$logger.debug("attempting to remove snaps 0 through #{max} out of #{deleteme.length} snapshots")
|
262
|
+
bigarg = "#{zfs}@#{deleteme[0..max].map { |s| s.split('@')[1] }.join(',')}"
|
263
|
+
com = "#{com_base} #{bigarg}"
|
264
|
+
$logger.debug("size of bigarg: #{bigarg.length} size of com: #{com.length}")
|
265
|
+
if bigarg.length >= 131072 or com.length >= (2097152-10000)
|
266
|
+
next
|
267
|
+
end
|
268
|
+
$logger.info(com)
|
269
|
+
deleteme = deleteme - deleteme[0..max]
|
270
|
+
system(com)
|
271
|
+
break
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
276
|
+
# parse a policy string into a hash of integers
|
277
|
+
def self.policy_parser(str)
|
278
|
+
res = {}
|
279
|
+
$date_patterns.keys.each do |tf|
|
280
|
+
res[tf]=0
|
281
|
+
end
|
282
|
+
p = str.scan(/\d+[#{$time_pattern_map.keys.join('')}]/i)
|
283
|
+
unless p.length > 0
|
284
|
+
raise "unable to parse the policy configuration #{str}"
|
285
|
+
end
|
286
|
+
p.each do |pi|
|
287
|
+
scn = /(\d+)([#{$time_pattern_map.keys.join('')}])/i.match(pi)
|
288
|
+
res[$time_pattern_map[scn[2].downcase]] = scn[1].to_i
|
289
|
+
end
|
290
|
+
res
|
291
|
+
end
|
292
|
+
def self.snapshot_create(noop: false, verbopt: false, debugopt: false, filter: '.+')
|
293
|
+
if debugopt
|
294
|
+
$logger.level = Logger::DEBUG
|
295
|
+
else
|
296
|
+
$logger.level = Logger::INFO
|
297
|
+
end
|
298
|
+
dt = DateTime.now
|
299
|
+
zfsget(properties: custom_properties()).each do |zfs,props|
|
300
|
+
# zfs must have snapshot set to true or recursive
|
301
|
+
if props.has_key?('zfsmgmt:snapshot') and props['zfsmgmt:snapshot'] == 'true' or ( props['zfsmgmt:snapshot'] == 'recursive' and props['zfsmgmt:snapshot@source'] == 'local' )
|
302
|
+
prefix = ( props.has_key?('zfsmgmt:snap_prefix') ? props['zfsmgmt:snap_prefix'] : 'zfsmgmt' )
|
303
|
+
ts = ( props.has_key?('zfsmgmt:snap_timestamp') ? props['zfsmgmt:snap_timestamp'] : '%FT%T%z' )
|
304
|
+
com = ['zfs','snapshot']
|
305
|
+
if props['zfsmgmt:snapshot'] == 'recursive' and props['zfsmgmt:snapshot@source'] == 'local'
|
306
|
+
com.push('-r')
|
307
|
+
end
|
308
|
+
com.push("#{zfs}@#{[prefix,dt.strftime(ts)].join('-')}")
|
309
|
+
$logger.info(com)
|
310
|
+
system(com.join(' '))
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
data/zfs_mgmt.gemspec
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "zfs_mgmt/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "zfs_mgmt"
|
8
|
+
spec.version = ZfsMgmt::VERSION
|
9
|
+
spec.licenses = ['GPL-3.0-or-later']
|
10
|
+
spec.authors = ["Aran Cox"]
|
11
|
+
spec.email = ["arancox@gmail.com"]
|
12
|
+
|
13
|
+
spec.summary = %q{Misc. helpers regarding snapshots and send/recv.}
|
14
|
+
#spec.description = %q{TODO: Write a longer description or delete this line.}
|
15
|
+
spec.homepage = 'https://github.com/aranc23/zfs_mgmt'
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
if spec.respond_to?(:metadata)
|
20
|
+
#spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
21
|
+
|
22
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
23
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
24
|
+
spec.metadata["changelog_uri"] = spec.homepage << '/commits/'
|
25
|
+
else
|
26
|
+
raise "RubyGems 2.0 or newer is required to protect against " \
|
27
|
+
"public gem pushes."
|
28
|
+
end
|
29
|
+
|
30
|
+
# Specify which files should be added to the gem when it is released.
|
31
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
32
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
33
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
34
|
+
end
|
35
|
+
spec.bindir = "bin"
|
36
|
+
spec.executables = ['readsnaps','zfssendman','zfssnapman','zfsrecvman','zfs-list-snapshots','zfsmgr']
|
37
|
+
spec.require_paths = ["lib"]
|
38
|
+
|
39
|
+
spec.add_development_dependency "bundler", "~> 1.16"
|
40
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
41
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
42
|
+
spec.add_development_dependency "thor", "~> 1.0.1"
|
43
|
+
spec.add_development_dependency "text-table", "~> 1.2.4"
|
44
|
+
spec.add_development_dependency "filesize", "~> 0.2.0"
|
45
|
+
|
46
|
+
end
|