zfs_mgmt 0.3.10 → 0.4.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/bin/zfsmgr +11 -1
- data/lib/zfs_mgmt.rb +371 -74
- data/lib/zfs_mgmt/restic.rb +4 -5
- data/lib/zfs_mgmt/version.rb +1 -1
- data/lib/zfs_mgmt/zfs_mgr.rb +1 -0
- data/lib/zfs_mgmt/zfs_mgr/list.rb +36 -4
- data/lib/zfs_mgmt/zfs_mgr/restic.rb +3 -0
- data/lib/zfs_mgmt/zfs_mgr/send.rb +40 -0
- data/lib/zfs_mgmt/zfs_mgr/snapshot.rb +12 -9
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 558d14c8bb3e9617b2d355308c85cd1da639035fb67662409a6445e1769c6e55
|
4
|
+
data.tar.gz: ef0acb4c148281bb81e0df51f08efa9c2ab87302a14c3700cce3214e9ab9e515
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 84f9fa0ba043d4c7a8e4b4625642bd5c14260591438da6851ecf6704a7747baedd4f1e397c3d55a648e55e6588c5bd6efd31e56a243a918bb9cb40de5db26c14
|
7
|
+
data.tar.gz: 1d77d1167f829dfa24236c3bdbda28093b15c5f325a55c02c26f9cd8eba7e9d73d1e2db3d8e68c6d9370ecc5e70ec676f2dba9682ab4a05d80bce273572f6206
|
data/Gemfile.lock
CHANGED
data/bin/zfsmgr
CHANGED
@@ -8,11 +8,19 @@ class ZfsMgr < Thor
|
|
8
8
|
class_option :zfs_binary, :type => :string, :default => 'zfs',
|
9
9
|
:desc => 'zfs binary'
|
10
10
|
class_option :zpool_binary, :type => :string, :default => 'zpool',
|
11
|
-
:desc => '
|
11
|
+
:desc => 'zpool binary'
|
12
|
+
class_option :mbuffer_binary, :type => :string, :default => 'mbuffer',
|
13
|
+
:desc => 'mbuffer binary'
|
14
|
+
class_option :pv_binary, :type => :string, :default => 'pv',
|
15
|
+
:desc => 'pv binary'
|
16
|
+
class_option :loglevel, :type => :string, :default => ( $stdout.isatty ? 'info' : 'warn' ),
|
17
|
+
:enum => ['debug','error','fatal','info','warn'],
|
18
|
+
:desc => 'set logging level to specified severity'
|
12
19
|
desc "zfsget [ZFS]", "execute zfs get for the given properties and types and parse the output into a nested hash"
|
13
20
|
method_option :properties, :type => :array, :default => ['name'], :desc => "List of properties passed to zfs get"
|
14
21
|
method_option :types, :type => :array, :default => ['filesystem','volume'], enum: ['filesystem','volume','snapshot'], :desc => "list of types"
|
15
22
|
def zfsget(zfs)
|
23
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
16
24
|
ZfsMgmt.global_options = options
|
17
25
|
pp ZfsMgmt.zfsget(properties: options[:properties],
|
18
26
|
types: options[:types],
|
@@ -24,6 +32,8 @@ class ZfsMgr < Thor
|
|
24
32
|
subcommand "list", ZfsMgmt::ZfsMgr::List
|
25
33
|
desc "restic SUBCOMMAND ...ARGS", "backup zfs to restic"
|
26
34
|
subcommand "restic", ZfsMgmt::ZfsMgr::Restic
|
35
|
+
desc "send SUBCOMMAND ...ARGS", "send zfs"
|
36
|
+
subcommand "send", ZfsMgmt::ZfsMgr::Send
|
27
37
|
end
|
28
38
|
|
29
39
|
ZfsMgr.start(ARGV)
|
data/lib/zfs_mgmt.rb
CHANGED
@@ -9,7 +9,7 @@ require 'text-table'
|
|
9
9
|
require 'open3'
|
10
10
|
require 'filesize'
|
11
11
|
|
12
|
-
$logger = Logger.new(STDERR)
|
12
|
+
$logger = Logger.new(STDERR, progname: 'zfs_mgmt')
|
13
13
|
|
14
14
|
$date_patterns = {
|
15
15
|
'hourly' => '%F Hour %H',
|
@@ -41,6 +41,8 @@ module ZfsMgmt
|
|
41
41
|
class << self
|
42
42
|
attr_accessor :global_options
|
43
43
|
end
|
44
|
+
class ZfsGetError < StandardError
|
45
|
+
end
|
44
46
|
def self.custom_properties()
|
45
47
|
return [
|
46
48
|
'policy',
|
@@ -53,6 +55,9 @@ module ZfsMgmt
|
|
53
55
|
'snapshot',
|
54
56
|
'snap_prefix',
|
55
57
|
'snap_timestamp',
|
58
|
+
'send',
|
59
|
+
'remote',
|
60
|
+
'destination',
|
56
61
|
].map do |p|
|
57
62
|
['zfsmgmt',p].join(':')
|
58
63
|
end
|
@@ -87,8 +92,7 @@ module ZfsMgmt
|
|
87
92
|
|
88
93
|
def self.zfs_hold(hold,snapshot)
|
89
94
|
com = [global_options['zfs_binary'], 'hold', hold, snapshot]
|
90
|
-
|
91
|
-
system(com.join(' '))
|
95
|
+
system_com(com)
|
92
96
|
unless $?.success?
|
93
97
|
errstr = "unable to set hold: #{hold} for snapshot: #{snapshot}"
|
94
98
|
$logger.error(errstr)
|
@@ -98,8 +102,7 @@ module ZfsMgmt
|
|
98
102
|
|
99
103
|
def self.zfs_release(hold,snapshot)
|
100
104
|
com = [@global_options['zfs_binary'], 'release', hold, snapshot]
|
101
|
-
|
102
|
-
system(com.join(' '))
|
105
|
+
system_com(com)
|
103
106
|
unless $?.success?
|
104
107
|
errstr = "unable to release hold: #{hold} for snapshot: #{snapshot}"
|
105
108
|
$logger.error(errstr)
|
@@ -107,20 +110,20 @@ module ZfsMgmt
|
|
107
110
|
end
|
108
111
|
end
|
109
112
|
|
110
|
-
def self.zfsget(properties: ['
|
113
|
+
def self.zfsget(properties: ['all'],types: ['filesystem','volume'],zfs: '', command_prefix: [])
|
111
114
|
results={}
|
112
115
|
com = [ZfsMgmt.global_options[:zfs_binary], 'get', '-Hp', properties.join(','), '-t', types.join(','), zfs]
|
113
|
-
$logger.debug(com.join(' '))
|
114
|
-
so,se,status = Open3.capture3(com.join(' '))
|
116
|
+
$logger.debug((command_prefix+com).join(' '))
|
117
|
+
so,se,status = Open3.capture3((command_prefix+com).join(' '))
|
115
118
|
if status.signaled?
|
116
119
|
$logger.error("process was signalled \"#{com.join(' ')}\", termsig #{status.termsig}")
|
117
|
-
raise
|
120
|
+
raise ZfsGetError, "process was signalled \"#{com.join(' ')}\", termsig #{status.termsig}"
|
118
121
|
end
|
119
122
|
unless status.success?
|
120
123
|
$logger.error("failed to execute \"#{com.join(' ')}\", exit status #{status.exitstatus}")
|
121
124
|
so.split("\n").each { |l| $logger.debug("stdout: #{l}") }
|
122
125
|
se.split("\n").each { |l| $logger.error("stderr: #{l}") }
|
123
|
-
raise
|
126
|
+
raise ZfsGetError, "failed to execute \"#{com.join(' ')}\", exit status #{status.exitstatus}"
|
124
127
|
end
|
125
128
|
so.split("\n").each do |line|
|
126
129
|
params = line.split("\t")
|
@@ -235,7 +238,7 @@ module ZfsMgmt
|
|
235
238
|
}
|
236
239
|
return saved,saved_snaps,deleteme
|
237
240
|
end
|
238
|
-
def self.zfs_managed_list(filter: '.+', properties:
|
241
|
+
def self.zfs_managed_list(filter: '.+', properties: ['all'], property_match: { 'zfsmgmt:manage' => method(:prop_on?) } )
|
239
242
|
zfss = [] # array of arrays
|
240
243
|
zfsget(properties: properties).each do |zfs,props|
|
241
244
|
unless /#{filter}/ =~ zfs
|
@@ -243,7 +246,7 @@ module ZfsMgmt
|
|
243
246
|
end
|
244
247
|
managed = true
|
245
248
|
property_match.each do |k,v|
|
246
|
-
unless
|
249
|
+
unless key_comp?(props,k,v)
|
247
250
|
managed = false
|
248
251
|
break
|
249
252
|
end
|
@@ -258,20 +261,20 @@ module ZfsMgmt
|
|
258
261
|
end
|
259
262
|
return zfss
|
260
263
|
end
|
261
|
-
def self.snapshot_policy(
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
$logger.level = Logger::INFO
|
266
|
-
end
|
267
|
-
zfs_managed_list(filter: filter).each do |zdata|
|
268
|
-
(zfs,props,snaps) = zdata
|
269
|
-
unless props.has_key?('zfsmgmt:policy') and policy_parser(props['zfsmgmt:policy'])
|
270
|
-
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
|
264
|
+
def self.snapshot_policy(filter: '.+')
|
265
|
+
zfs_managed_list(filter: filter).each do |zfs,props,snaps|
|
266
|
+
unless props.has_key?('zfsmgmt:policy')
|
267
|
+
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no policy configuration in zfsmgmt:policy, skipping")
|
271
268
|
next # zfs
|
272
269
|
end
|
273
|
-
|
274
|
-
|
270
|
+
|
271
|
+
begin
|
272
|
+
# call the function that decides who to save and who to delete
|
273
|
+
(saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
|
274
|
+
rescue ArgumentError
|
275
|
+
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
|
276
|
+
next
|
277
|
+
end
|
275
278
|
|
276
279
|
if saved_snaps.length == 0
|
277
280
|
$logger.info("no snapshots marked as saved by policy for #{zfs}")
|
@@ -287,21 +290,20 @@ module ZfsMgmt
|
|
287
290
|
print table.to_s
|
288
291
|
end
|
289
292
|
end
|
290
|
-
def self.snapshot_destroy(noop: false,
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
$logger.level = Logger::INFO
|
295
|
-
end
|
296
|
-
zfs_managed_list(filter: filter).each do |zdata|
|
297
|
-
(zfs,props,snaps) = zdata
|
298
|
-
unless props.has_key?('zfsmgmt:policy') and policy_parser(props['zfsmgmt:policy'])
|
299
|
-
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
|
293
|
+
def self.snapshot_destroy(noop: false, verbose: false, filter: '.+')
|
294
|
+
zfs_managed_list(filter: filter).each do |zfs,props,snaps|
|
295
|
+
unless props.has_key?('zfsmgmt:policy')
|
296
|
+
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no policy configuration in zfsmgmt:policy, skipping")
|
300
297
|
next # zfs
|
301
298
|
end
|
302
299
|
|
303
|
-
|
304
|
-
|
300
|
+
begin
|
301
|
+
# call the function that decides who to save and who to delete
|
302
|
+
(saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
|
303
|
+
rescue ArgumentError
|
304
|
+
$logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
|
305
|
+
next
|
306
|
+
end
|
305
307
|
|
306
308
|
$logger.info("deleting #{deleteme.length} snapshots for #{zfs}")
|
307
309
|
deleteme.reverse! # oldest first for removal
|
@@ -309,32 +311,22 @@ module ZfsMgmt
|
|
309
311
|
$logger.debug("delete: #{snap_name} #{local_epoch_to_datetime(snaps[snap_name]['creation']).strftime('%F %T')}")
|
310
312
|
end
|
311
313
|
|
312
|
-
com_base =
|
313
|
-
if deleteme.length > 0
|
314
|
-
|
315
|
-
|
316
|
-
if noop
|
317
|
-
com_base = "#{com_base}n"
|
318
|
-
end
|
319
|
-
if verbopt
|
320
|
-
com_base = "#{com_base}v"
|
321
|
-
end
|
314
|
+
com_base = ['zfs', 'destroy']
|
315
|
+
com_base.push('-d') if deleteme.length > 0 # why?
|
316
|
+
com_base.push('-n') if noop
|
317
|
+
com_base.push('-v') if verbose
|
322
318
|
while deleteme.length > 0
|
323
319
|
for i in 0..(deleteme.length - 1) do
|
324
320
|
max = deleteme.length - 1 - i
|
325
321
|
$logger.debug("attempting to remove snaps 0 through #{max} out of #{deleteme.length} snapshots")
|
326
322
|
bigarg = "#{zfs}@#{deleteme[0..max].map { |s| s.split('@')[1] }.join(',')}"
|
327
|
-
com =
|
323
|
+
com = com_base + [bigarg]
|
328
324
|
$logger.debug("size of bigarg: #{bigarg.length} size of com: #{com.length}")
|
329
325
|
if bigarg.length >= 131072 or com.length >= (2097152-10000)
|
330
326
|
next
|
331
327
|
end
|
332
|
-
$logger.info(com)
|
333
328
|
deleteme = deleteme - deleteme[0..max]
|
334
|
-
|
335
|
-
if $?.exitstatus != 0
|
336
|
-
$logger.error("zfs exited with non-zero status: #{$?.exitstatus}")
|
337
|
-
end
|
329
|
+
system_com(com) # pass -n, always run the command though
|
338
330
|
break
|
339
331
|
end
|
340
332
|
end
|
@@ -348,7 +340,7 @@ module ZfsMgmt
|
|
348
340
|
end
|
349
341
|
p = str.scan(/\d+[#{$time_pattern_map.keys.join('')}]/i)
|
350
342
|
unless p.length > 0
|
351
|
-
raise "unable to parse the policy configuration #{str}"
|
343
|
+
raise ArgumentError.new("unable to parse the policy configuration #{str}")
|
352
344
|
end
|
353
345
|
p.each do |pi|
|
354
346
|
scn = /(\d+)([#{$time_pattern_map.keys.join('')}])/i.match(pi)
|
@@ -356,33 +348,338 @@ module ZfsMgmt
|
|
356
348
|
end
|
357
349
|
res
|
358
350
|
end
|
359
|
-
|
360
|
-
|
361
|
-
|
351
|
+
# snapshot all filesystems configured for snapshotting
|
352
|
+
def self.snapshot_create(noop: false, filter: '.+')
|
353
|
+
dt = DateTime.now
|
354
|
+
zfsget.select { |zfs,props|
|
355
|
+
# must match filter
|
356
|
+
match_filter?(zfs, filter) and
|
357
|
+
# snapshot must be on or true
|
358
|
+
(
|
359
|
+
key_comp?(props,'zfsmgmt:snapshot') or
|
360
|
+
# or snapshot can be recursive and local, but only if the source is local or received
|
361
|
+
( key_comp?(props,'zfsmgmt:snapshot',['recursive','local']) and key_comp?(props,'zfsmgmt:snapshot@source',['local','received']) )
|
362
|
+
)
|
363
|
+
}.each do |zfs,props|
|
364
|
+
prefix = ( props.has_key?('zfsmgmt:snap_prefix') ? props['zfsmgmt:snap_prefix'] : 'zfsmgmt' )
|
365
|
+
ts = ( props.has_key?('zfsmgmt:snap_timestamp') ? props['zfsmgmt:snap_timestamp'] : '%FT%T%z' )
|
366
|
+
com = [global_options['zfs_binary'],'snapshot']
|
367
|
+
if key_comp?(props,'zfsmgmt:snapshot','recursive') and key_comp?(props,'zfsmgmt:snapshot@source',['local','received'])
|
368
|
+
com.push('-r')
|
369
|
+
end
|
370
|
+
com.push("#{zfs}@#{[prefix,dt.strftime(ts)].join('-')}")
|
371
|
+
system_com(com,noop)
|
372
|
+
end
|
373
|
+
end
|
374
|
+
def self.system_com(com, noop = false)
|
375
|
+
comstr = com.join(' ')
|
376
|
+
$logger.info(comstr)
|
377
|
+
unless noop
|
378
|
+
system(comstr)
|
379
|
+
unless $?.success?
|
380
|
+
$logger.error("command failed: #{$?.exitstatus}")
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|
384
|
+
def self.zfs_send(options,zfs,props,snaps)
|
385
|
+
sorted = snaps.keys.sort { |a,b| snaps[a]['creation'] <=> snaps[b]['creation'] }
|
386
|
+
# compute the zfs "path"
|
387
|
+
# ternary operator 4eva
|
388
|
+
destination_path = ( options[:destination] ? options[:destination] : props['zfsmgmt:destination'] )
|
389
|
+
if props['zfsmgmt:destination@source'] == 'local'
|
390
|
+
destination_path = File.join( destination_path,
|
391
|
+
File.basename(zfs)
|
392
|
+
)
|
393
|
+
elsif m = /inherited from (.+)/.match(props['zfsmgmt:destination@source'])
|
394
|
+
destination_path = File.join( destination_path,
|
395
|
+
File.basename(m[1]),
|
396
|
+
zfs.sub(m[1],'')
|
397
|
+
)
|
398
|
+
else
|
399
|
+
$logger.error("fatal error: #{props['zfsmgmt:destination']} source: #{props['zfsmgmt:destination@source']}")
|
400
|
+
exit(1)
|
401
|
+
end
|
402
|
+
# does the destination zfs already exist?
|
403
|
+
remote_zfs_state = ''
|
404
|
+
begin
|
405
|
+
recv_zfs = zfsget(zfs: destination_path,
|
406
|
+
command_prefix: recv_command_prefix(options,props),
|
407
|
+
#properties: ['receive_resume_token'],
|
408
|
+
)
|
409
|
+
rescue ZfsGetError
|
410
|
+
$logger.debug("recv filesystem doesn't exist: #{destination_path}")
|
411
|
+
remote_zfs_state = 'missing'
|
412
|
+
else
|
413
|
+
if recv_zfs[destination_path].has_key?('receive_resume_token')
|
414
|
+
remote_zfs_state = recv_zfs[destination_path]['receive_resume_token']
|
415
|
+
else
|
416
|
+
remote_zfs_state = 'present'
|
417
|
+
end
|
418
|
+
end
|
419
|
+
|
420
|
+
if remote_zfs_state == 'missing'
|
421
|
+
# the zfs does not exist, send initial (oldest?) snapshot
|
422
|
+
com = []
|
423
|
+
source = sorted[0]
|
424
|
+
if options[:initial_snapshot] == 'newest' or
|
425
|
+
key_comp?(options, 'replicate') or
|
426
|
+
key_comp?(props, 'zfsmgmt:send_replcate')
|
427
|
+
source = sorted[-1]
|
428
|
+
end
|
429
|
+
com += zfs_send_com(options,
|
430
|
+
props,
|
431
|
+
[],
|
432
|
+
source,
|
433
|
+
)
|
434
|
+
e = zfs_send_estimate(com) if options[:verbose] == 'pv'
|
435
|
+
com += mbuffer_command(options) if options[:mbuffer]
|
436
|
+
com += pv_command(options,e) if options[:verbose] == 'pv'
|
437
|
+
com += zfs_recv_com(options,[],props,destination_path)
|
438
|
+
|
439
|
+
system_com(com)
|
440
|
+
unless $?.success?
|
441
|
+
return
|
442
|
+
end
|
443
|
+
|
444
|
+
elsif remote_zfs_state != 'present'
|
445
|
+
# should be resumable!
|
446
|
+
com = [ ]
|
447
|
+
com.push( ZfsMgmt.global_options[:zfs_binary], 'send', '-t', remote_zfs_state )
|
448
|
+
com.push('-v','-P') if key_comp?(options, 'verbose', 'send')
|
449
|
+
com.push('|')
|
450
|
+
e = zfs_send_estimate(com) if options[:verbose] == 'pv'
|
451
|
+
com += mbuffer_command(options) if options[:mbuffer]
|
452
|
+
com += pv_command(options,e) if options[:verbose] == 'pv'
|
453
|
+
|
454
|
+
recv = [ ZfsMgmt.global_options[:zfs_binary], 'recv', '-s' ]
|
455
|
+
recv.push('-n') if options[:noop]
|
456
|
+
recv.push('-u') if options[:unmount]
|
457
|
+
recv.push('-v') if options[:verbose] and ( options[:verbose] == 'receive' or options[:verbose] == 'recv' )
|
458
|
+
recv.push(dq(destination_path))
|
459
|
+
|
460
|
+
if options[:remote] or props['zfsmgmt:remote']
|
461
|
+
if options[:mbuffer]
|
462
|
+
recv = mbuffer_command(options) + recv
|
463
|
+
end
|
464
|
+
recv = recv_command_prefix(options,props) + [ sq(recv.join(' ')) ]
|
465
|
+
end
|
466
|
+
|
467
|
+
com += recv
|
468
|
+
|
469
|
+
system_com(com)
|
470
|
+
unless $?.success?
|
471
|
+
return
|
472
|
+
end
|
473
|
+
end
|
474
|
+
|
475
|
+
# the zfs already exists, so update with incremental?
|
476
|
+
begin
|
477
|
+
remote_snaps = zfsget(zfs: destination_path,
|
478
|
+
types: ['snapshot'],
|
479
|
+
command_prefix: recv_command_prefix(options,props),
|
480
|
+
properties: ['creation','userrefs'],
|
481
|
+
)
|
482
|
+
rescue ZfsGetError
|
483
|
+
$logger.error("unable to get remote snapshot information for #{destination_path}")
|
484
|
+
return
|
485
|
+
end
|
486
|
+
unless remote_snaps and remote_snaps.keys.length > 0
|
487
|
+
$logger.error("receiving filesystem has NO snapshots, it must be destroyed: #{destination_path}")
|
488
|
+
return
|
489
|
+
end
|
490
|
+
if remote_snaps.has_key?(sorted[-1].sub(zfs,destination_path))
|
491
|
+
$logger.info("the most recent local snapshot (#{sorted[-1]}) already exists on the remote side (#{sorted[-1].sub(zfs,destination_path)})")
|
492
|
+
return
|
493
|
+
end
|
494
|
+
remote_snaps.sort_by { |k,v| -v['creation'] }.each do |rsnap,v|
|
495
|
+
# oldest first
|
496
|
+
#pp rsnap,rsnap.sub(destination_path,zfs)
|
497
|
+
#pp snaps
|
498
|
+
if snaps.has_key?(rsnap.sub(destination_path,zfs))
|
499
|
+
$logger.debug("process #{rsnap} to #{sorted[-1]}")
|
500
|
+
com = []
|
501
|
+
i_opt = '-i'
|
502
|
+
# allow the command line option for intermediary to override the property
|
503
|
+
if key_comp?(options,'intermediary',[true,false])
|
504
|
+
i_opt = '-I' if key_comp?(options, 'intermediary', true)
|
505
|
+
elsif key_comp?(props, 'zfsmgmt:send_intermediary')
|
506
|
+
i_opt = '-I'
|
507
|
+
end
|
508
|
+
|
509
|
+
com += zfs_send_com(options,props,[i_opt, dq('@' + rsnap.split('@')[1])], sorted[-1])
|
510
|
+
e = zfs_send_estimate(com) if options[:verbose] == 'pv'
|
511
|
+
com += mbuffer_command(options) if options[:mbuffer]
|
512
|
+
com += pv_command(options,e) if options[:verbose] == 'pv'
|
513
|
+
com += zfs_recv_com(options,[],props,destination_path)
|
514
|
+
|
515
|
+
system_com(com)
|
516
|
+
return
|
517
|
+
end
|
518
|
+
$logger.debug("skipping remote snapshot #{rsnap} because the same snapshot doesn't exist locally #{rsnap.sub(destination_path,zfs)}")
|
519
|
+
end
|
520
|
+
$logger.error("receiving filesystem has no snapshots that still exists on the sending side, it must be destroyed: #{destination_path}")
|
521
|
+
|
522
|
+
end
|
523
|
+
def self.mbuffer_command(options)
|
524
|
+
mbuffer_command = [ ZfsMgmt.global_options[:mbuffer_binary] ]
|
525
|
+
mbuffer_command.push('-q') unless options[:verbose] == 'mbuffer'
|
526
|
+
mbuffer_command.push('-m',options[:mbuffer_size]) if options[:mbuffer_size]
|
527
|
+
mbuffer_command.push('|')
|
528
|
+
mbuffer_command
|
529
|
+
end
|
530
|
+
def self.zfs_send_com(options,props,extra_opts,target)
|
531
|
+
zfs_send_com = [ ZfsMgmt.global_options[:zfs_binary], 'send' ]
|
532
|
+
zfs_send_com.push('-v','-P') if key_comp?(options,'verbose','send')
|
533
|
+
send_opts = {
|
534
|
+
'backup' => '-b',
|
535
|
+
'compressed' => '-c',
|
536
|
+
'embed' => '-e',
|
537
|
+
'holds' => '-h',
|
538
|
+
'large_block' => '-L',
|
539
|
+
'props' => '-p',
|
540
|
+
'raw' => '-w',
|
541
|
+
'replicate' => '-R',
|
542
|
+
}
|
543
|
+
send_opts.each do |p,o|
|
544
|
+
# allow the command line options to override the properties value
|
545
|
+
if key_comp?(options,p,[true,false])
|
546
|
+
zfs_send_com.push(o) if key_comp?(options,p,true)
|
547
|
+
elsif key_comp?(props,"zfsmgmt:send_#{p}")
|
548
|
+
zfs_send_com.push(o)
|
549
|
+
end
|
550
|
+
end
|
551
|
+
zfs_send_com + extra_opts + [dq(target),'|']
|
552
|
+
end
|
553
|
+
def self.zfs_recv_com(options,extra_opts,props,target)
|
554
|
+
zfs_recv_com = [ ZfsMgmt.global_options[:zfs_binary], 'recv', '-F', '-s' ]
|
555
|
+
recv_opts = {
|
556
|
+
'noop' => '-n',
|
557
|
+
'drop_holds' => '-h',
|
558
|
+
'unmount' => '-u',
|
559
|
+
#'discard_last' => '-e',
|
560
|
+
#'discard_first' => '-d',
|
561
|
+
}
|
562
|
+
recv_opts.each do |p,o|
|
563
|
+
# allow the command line options to override the properties value
|
564
|
+
if key_comp?(options,p,[true,false])
|
565
|
+
zfs_recv_com.push(o) if key_comp?(options,p,true)
|
566
|
+
elsif key_comp?(props,"zfsmgmt:recv_#{p}")
|
567
|
+
zfs_recv_com.push(o)
|
568
|
+
end
|
569
|
+
end
|
570
|
+
zfs_recv_com.push('-v') if key_comp?(options, 'verbose', ['receive', 'recv'])
|
571
|
+
if options[:exclude]
|
572
|
+
options[:exclude].each do |x|
|
573
|
+
zfs_recv_com.push('-x',x)
|
574
|
+
end
|
575
|
+
end
|
576
|
+
if options[:option]
|
577
|
+
options[:option].each do |x|
|
578
|
+
zfs_recv_com.push('-o',x)
|
579
|
+
end
|
580
|
+
end
|
581
|
+
zfs_recv_com += extra_opts
|
582
|
+
zfs_recv_com.push(dq(target))
|
583
|
+
|
584
|
+
if options[:remote] or props['zfsmgmt:remote']
|
585
|
+
if options[:mbuffer]
|
586
|
+
zfs_recv_com = mbuffer_command(options) + zfs_recv_com
|
587
|
+
end
|
588
|
+
zfs_recv_com = recv_command_prefix(options,props) + [ sq(zfs_recv_com.join(' ')) ]
|
589
|
+
end
|
590
|
+
zfs_recv_com
|
591
|
+
end
|
592
|
+
def self.recv_command_prefix(options,props)
|
593
|
+
( (options[:remote] or props['zfsmgmt:remote']) ?
|
594
|
+
[ 'ssh', ( options[:remote] ? options[:remote] : props['zfsmgmt:remote'] ) ] :
|
595
|
+
[] )
|
596
|
+
end
|
597
|
+
def self.zfs_send_estimate(com)
|
598
|
+
lcom = com.dup
|
599
|
+
lcom.pop() # remove the pipe symbol
|
600
|
+
precom = [ lcom.shift, lcom.shift ]
|
601
|
+
lcom.unshift('-P') unless lcom.include?('-P')
|
602
|
+
lcom.unshift('-n')
|
603
|
+
lcom.push('2>&1')
|
604
|
+
lcom = precom + lcom
|
605
|
+
$logger.debug(lcom.join(' '))
|
606
|
+
%x[#{lcom.join(' ')}].each_line do |l|
|
607
|
+
if m = /(incremental|size).*\s+(\d+)$/.match(l)
|
608
|
+
return m[2].to_i
|
609
|
+
end
|
610
|
+
end
|
611
|
+
$logger.error("no estimate available")
|
612
|
+
return nil
|
613
|
+
end
|
614
|
+
def self.pv_command(options,estimate)
|
615
|
+
a = []
|
616
|
+
a += [options[:pv_binary], '-prb' ]
|
617
|
+
if estimate
|
618
|
+
a += ['-e', '-s', estimate ]
|
619
|
+
end
|
620
|
+
a.push('|')
|
621
|
+
a
|
622
|
+
end
|
623
|
+
|
624
|
+
def self.sq(s)
|
625
|
+
"'#{s}'"
|
626
|
+
end
|
627
|
+
def self.dq(s)
|
628
|
+
"\"#{s}\""
|
629
|
+
end
|
630
|
+
def self.prop_on?(v)
|
631
|
+
['true','on'].include?(v)
|
632
|
+
end
|
633
|
+
def self.match_filter?(zfs, filter)
|
634
|
+
/#{filter}/ =~ zfs
|
635
|
+
end
|
636
|
+
def self.key_comp?(h,p,v = method(:prop_on?))
|
637
|
+
#$logger.debug("p:#{p}\th[p]:#{h[p]}\tv:#{v}")
|
638
|
+
return false unless h.has_key?(p)
|
639
|
+
if v.kind_of?(Array)
|
640
|
+
return v.include?(h[p])
|
641
|
+
elsif v.kind_of?(Hash)
|
642
|
+
return v.keys.include?(h[p])
|
643
|
+
elsif v.kind_of?(Method)
|
644
|
+
return v.call(h[p])
|
645
|
+
elsif v.kind_of?(Regexp)
|
646
|
+
return v =~ h[p]
|
362
647
|
else
|
648
|
+
# string, boolean, numbers?
|
649
|
+
return h[p] == v
|
650
|
+
end
|
651
|
+
end
|
652
|
+
def self.set_log_level(sev)
|
653
|
+
case sev
|
654
|
+
when 'debug'
|
655
|
+
$logger.level = Logger::DEBUG
|
656
|
+
when 'info'
|
363
657
|
$logger.level = Logger::INFO
|
658
|
+
when 'warn'
|
659
|
+
$logger.level = Logger::WARN
|
660
|
+
when 'error'
|
661
|
+
$logger.level = Logger::ERROR
|
662
|
+
when 'fatal'
|
663
|
+
$logger.level = Logger::FATAL
|
364
664
|
end
|
365
|
-
|
366
|
-
|
367
|
-
|
665
|
+
end
|
666
|
+
def self.zfs_send_all(options)
|
667
|
+
zfs_managed_list(filter: options[:filter],
|
668
|
+
property_match: { 'zfsmgmt:send' => method(:prop_on?) }).each do |zfs,props,snaps|
|
669
|
+
|
670
|
+
if props['zfsmgmt:send@source'] == 'received'
|
671
|
+
$logger.debug("skipping received filesystem: #{zfs}")
|
368
672
|
next
|
369
673
|
end
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
ts = ( props.has_key?('zfsmgmt:snap_timestamp') ? props['zfsmgmt:snap_timestamp'] : '%FT%T%z' )
|
378
|
-
com = [global_options['zfs_binary'],'snapshot']
|
379
|
-
if props['zfsmgmt:snapshot'] == 'recursive' and props['zfsmgmt:snapshot@source'] == 'local'
|
380
|
-
com.push('-r')
|
381
|
-
end
|
382
|
-
com.push("#{zfs}@#{[prefix,dt.strftime(ts)].join('-')}")
|
383
|
-
$logger.info(com)
|
384
|
-
system(com.join(' '))
|
674
|
+
if key_comp?(props,'zfsmgmt:send_replicate') and props['zfsmgmt:send_replicate@source'] != 'local'
|
675
|
+
$logger.debug("skipping descendant of replicated filesystems: #{zfs}")
|
676
|
+
next
|
677
|
+
end
|
678
|
+
unless props['zfsmgmt:destination']
|
679
|
+
$logger.error("#{zfs}: you must specify a destination zfs path via the user property zfsmgmt:destination, even if using --destination on the command line, skipping")
|
680
|
+
next
|
385
681
|
end
|
682
|
+
zfs_send(options,zfs,props,snaps)
|
386
683
|
end
|
387
684
|
end
|
388
685
|
end
|
data/lib/zfs_mgmt/restic.rb
CHANGED
@@ -18,7 +18,7 @@ module ZfsMgmt::Restic
|
|
18
18
|
com.push( '--repo', props['zfsmgmt:restic_repository'] )
|
19
19
|
end
|
20
20
|
|
21
|
-
$logger.
|
21
|
+
$logger.info("#{com.join(' ')}")
|
22
22
|
restic_output = %x(#{com.join(' ')})
|
23
23
|
unless $?.success?
|
24
24
|
$logger.error("unable to query the restic database")
|
@@ -70,7 +70,7 @@ module ZfsMgmt::Restic
|
|
70
70
|
'zfsmgmt:restic_repository',
|
71
71
|
'userrefs',
|
72
72
|
],
|
73
|
-
property_match: { 'zfsmgmt:restic_backup' => 'true' }).each do |blob|
|
73
|
+
property_match: { 'zfsmgmt:restic_backup' => ['on','true'] }).each do |blob|
|
74
74
|
zfs,props,zfs_snapshots = blob
|
75
75
|
last_zfs_snapshot = zfs_snapshots.keys.sort { |a,b| zfs_snapshots[a]['creation'] <=> zfs_snapshots[b]['creation'] }.last
|
76
76
|
zfs_snap_time = Time.at(zfs_snapshots[last_zfs_snapshot]['creation'])
|
@@ -107,7 +107,7 @@ module ZfsMgmt::Restic
|
|
107
107
|
"zfsmgmt:snapshot=#{last_zfs_snapshot}",
|
108
108
|
"zfsmgmt:zfs=#{zfs}",
|
109
109
|
"zfsmgmt:level=#{level}" ]
|
110
|
-
com = [ ZfsMgmt.global_options['zfs_binary'], 'send', '-
|
110
|
+
com = [ ZfsMgmt.global_options['zfs_binary'], 'send', '-w', '-h', '-p' ]
|
111
111
|
if level > 0
|
112
112
|
if options[:intermediary]
|
113
113
|
com.push('-I')
|
@@ -143,8 +143,7 @@ module ZfsMgmt::Restic
|
|
143
143
|
unless ZfsMgmt.zfs_holds(last_zfs_snapshot).include?('zfsmgmt_restic')
|
144
144
|
ZfsMgmt.zfs_hold('zfsmgmt_restic',last_zfs_snapshot)
|
145
145
|
end
|
146
|
-
|
147
|
-
system(com.join(' '))
|
146
|
+
ZfsMgmt.system_com(com)
|
148
147
|
chain_snaps = chain.map do |rsnap|
|
149
148
|
rsnap['zfsmgmt:snapshot']
|
150
149
|
end
|
data/lib/zfs_mgmt/version.rb
CHANGED
data/lib/zfs_mgmt/zfs_mgr.rb
CHANGED
@@ -5,21 +5,53 @@ class ZfsMgmt::ZfsMgr::List < Thor
|
|
5
5
|
:desc => 'only act on zfs matching this regexp'
|
6
6
|
desc "stale", "list all zfs with stale snapshots"
|
7
7
|
method_option :age, :desc => "timeframe outside of which the zfs will be considered stale", :default => '1d'
|
8
|
+
method_option :format, :desc => "output format", :type => :string, :enum => ['table','tab'], :default => 'table'
|
8
9
|
def stale()
|
10
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
9
11
|
ZfsMgmt.global_options = options
|
10
12
|
cutoff = Time.at(Time.now.to_i - ZfsMgmt.timespec_to_seconds(options[:age]))
|
11
13
|
table = Text::Table.new
|
12
14
|
table.head = ['zfs','snapshot','age']
|
13
15
|
table.rows = []
|
14
|
-
ZfsMgmt.zfs_managed_list(filter: options[:filter]).each do |
|
15
|
-
zfs,props,snaps = blob
|
16
|
+
ZfsMgmt.zfs_managed_list(filter: options[:filter]).each do |zfs,props,snaps|
|
16
17
|
last = snaps.keys.sort { |a,b| snaps[a]['creation'] <=> snaps[b]['creation'] }.last
|
17
18
|
snap_time = Time.at(snaps[last]['creation'])
|
18
19
|
if snap_time < cutoff
|
19
|
-
|
20
|
+
line = [zfs,last.split('@')[1],snap_time]
|
21
|
+
table.rows << line
|
22
|
+
if options[:format] == 'tab'
|
23
|
+
print line.join("\t"),"\n"
|
24
|
+
end
|
20
25
|
end
|
21
26
|
end
|
22
|
-
if table.rows.count > 0
|
27
|
+
if options[:format] == 'table' and table.rows.count > 0
|
28
|
+
print table.to_s
|
29
|
+
end
|
30
|
+
end
|
31
|
+
desc "holds", "list all holds on snapshots"
|
32
|
+
method_option :format, :desc => "output format", :type => :string, :enum => ['table','tab','release'], :default => 'table'
|
33
|
+
def holds()
|
34
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
35
|
+
ZfsMgmt.global_options = options
|
36
|
+
table = Text::Table.new
|
37
|
+
table.head = ['snapshot','userrefs','holds']
|
38
|
+
table.rows = []
|
39
|
+
ZfsMgmt.zfs_managed_list(filter: options[:filter], property_match: {} ).each do |zfs,props,snaps|
|
40
|
+
snaps.sort_by { |x,y| y['creation'] }.each do |snap,d|
|
41
|
+
if d['userrefs'] > 0
|
42
|
+
line = [snap,d['userrefs'].to_s,ZfsMgmt.zfs_holds(snap).join(',')]
|
43
|
+
table.rows << line
|
44
|
+
if options[:format] == 'tab'
|
45
|
+
print line.join("\t"),"\n"
|
46
|
+
elsif options[:format] == 'release'
|
47
|
+
ZfsMgmt.zfs_holds(snap).each do |hold|
|
48
|
+
print "zfs release #{hold} #{snap}\n"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
if options[:format] == 'table' and table.rows.count > 0
|
23
55
|
print table.to_s
|
24
56
|
end
|
25
57
|
end
|
@@ -19,17 +19,20 @@ class ZfsMgmt::ZfsMgr::Backup < Thor
|
|
19
19
|
method_option :level, :desc => "backup level in integer form", :default => 2, :type => :numeric
|
20
20
|
method_option :intermediary, :alias => '-I', :desc => "pass -I (intermediary) option to zfs send", :default => false, :type => :boolean
|
21
21
|
def incremental()
|
22
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
22
23
|
ZfsMgmt.global_options = options
|
23
24
|
ZfsMgmt::Restic.backup(backup_level: options[:level], options: options)
|
24
25
|
end
|
25
26
|
desc "differential", "perform differential backup"
|
26
27
|
method_option :intermediary, :alias => '-I', :desc => "pass -I (intermediary) option to zfs send", :default => false, :type => :boolean
|
27
28
|
def differential()
|
29
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
28
30
|
ZfsMgmt.global_options = options
|
29
31
|
ZfsMgmt::Restic.backup(backup_level: 1, options: options)
|
30
32
|
end
|
31
33
|
desc "full", "perform full backup"
|
32
34
|
def full()
|
35
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
33
36
|
ZfsMgmt.global_options = options
|
34
37
|
ZfsMgmt::Restic.backup(backup_level: 0, options: options)
|
35
38
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# zfs send stuff
|
2
|
+
|
3
|
+
class ZfsMgmt::ZfsMgr::Send < Thor
|
4
|
+
class_option :filter, :type => :string, :default => '.+',
|
5
|
+
:desc => 'only act on zfs matching this regexp'
|
6
|
+
desc "all", "send all zfs configured via user properties"
|
7
|
+
method_option :remote, :type => :string,
|
8
|
+
:desc => 'remote specification like root@otherhost or localhost'
|
9
|
+
method_option :destination, :type => :string,
|
10
|
+
:desc => 'destination path like otherpool/ourpool'
|
11
|
+
method_option :verbose, :type => :string, :aliases => :'-v', :enum => ['send','receive','recv','mbuffer','pv'],
|
12
|
+
:desc => 'enable verbose output on the specified element of the pipe'
|
13
|
+
method_option :initial_snapshot, :type => :string, :enum => ['oldest','newest'], :default => 'oldest',
|
14
|
+
:desc => 'when sending the initial snapshot use the oldest or most recent snapshot'
|
15
|
+
|
16
|
+
method_option :intermediary, :aliases => :'-I', :desc => "pass -I option to zfs send", :type => :boolean
|
17
|
+
method_option :backup, :aliases => :'-p', :desc => "pass -b (--backup) option to zfs send", :type => :boolean
|
18
|
+
method_option :compressed, :aliases => :'-c', :desc => "pass -c (compressed) option to zfs send", :type => :boolean
|
19
|
+
method_option :embed, :aliases => :'-e', :desc => "pass -e (--embed) option to zfs send", :type => :boolean
|
20
|
+
method_option :holds, :aliases => :'-h', :desc => "pass the -h (--holds) option to zfs send", :type => :boolean
|
21
|
+
method_option :large_block, :aliases => :'-L', :desc => "pass -L (--large-block) option to zfs send", :type => :boolean
|
22
|
+
method_option :props, :aliases => :'-p', :desc => "pass -p (--props) option to zfs send", :type => :boolean
|
23
|
+
method_option :raw, :aliases => :'-w', :desc => "pass -w (--raw) option to zfs send", :type => :boolean
|
24
|
+
method_option :replicate, :aliases => :'-R', :desc => "pass -R (--replicate) option to zfs send", :type => :boolean
|
25
|
+
|
26
|
+
method_option :noop, :aliases => :'-n', :desc => "pass -n (noop) option to zfs send", :type => :boolean
|
27
|
+
method_option :unmount, :aliases => :'-u', :desc => "pass -u (unmount) option to zfs receive", :type => :boolean
|
28
|
+
method_option :exclude, :aliases => :'-x', :desc => "passed to -x option of receive side", :type => :array
|
29
|
+
method_option :option, :aliases => :'-o', :desc => "passed to -o option of receive side", :type => :array
|
30
|
+
method_option :drop_holds, :desc => "pass the -h option to zfs recv, indicating holds should be ignored", :type => :boolean
|
31
|
+
|
32
|
+
method_option :mbuffer, :desc => "insert mbuffer between send and recv", :default => true, :type => :boolean
|
33
|
+
method_option :mbuffer_size, :desc => "passed to mbuffer -s option", :type => :string
|
34
|
+
def all()
|
35
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
36
|
+
ZfsMgmt.global_options = options
|
37
|
+
|
38
|
+
ZfsMgmt.zfs_send_all(options)
|
39
|
+
end
|
40
|
+
end
|
@@ -1,27 +1,30 @@
|
|
1
1
|
# implement snapshot management
|
2
2
|
|
3
3
|
class ZfsMgmt::ZfsMgr::Snapshot < Thor
|
4
|
-
class_option :noop, :type => :boolean, :default => false,
|
5
|
-
:desc => 'pass -n option to zfs commands'
|
6
|
-
class_option :verbose, :type => :boolean, :default => false,
|
7
|
-
:desc => 'pass -v option to zfs commands'
|
8
|
-
class_option :debug, :type => :boolean, :default => false,
|
9
|
-
:desc => 'set logging level to debug'
|
10
4
|
class_option :filter, :type => :string, :default => '.+',
|
11
5
|
:desc => 'only act on zfs matching this regexp'
|
12
6
|
desc "destroy", "apply the snapshot destroy policy to zfs"
|
7
|
+
method_option :noop, :type => :boolean, :default => false,
|
8
|
+
:desc => 'pass -n option to zfs commands'
|
9
|
+
method_option :verbose, :type => :boolean, :default => false,
|
10
|
+
:desc => 'pass -v option to zfs commands'
|
13
11
|
def destroy()
|
12
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
14
13
|
ZfsMgmt.global_options = options
|
15
|
-
ZfsMgmt.snapshot_destroy(noop: options[:noop],
|
14
|
+
ZfsMgmt.snapshot_destroy(noop: options[:noop], verbose: options[:verbose], filter: options[:filter])
|
16
15
|
end
|
17
16
|
desc "policy", "print the policy table for zfs"
|
18
17
|
def policy()
|
18
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
19
19
|
ZfsMgmt.global_options = options
|
20
|
-
ZfsMgmt.snapshot_policy(
|
20
|
+
ZfsMgmt.snapshot_policy(filter: options[:filter])
|
21
21
|
end
|
22
22
|
desc "create", "execute zfs snapshot based on zfs properties"
|
23
|
+
method_option :noop, :type => :boolean, :default => false,
|
24
|
+
:desc => 'log snapshot commands without running zfs snapshot'
|
23
25
|
def create()
|
26
|
+
ZfsMgmt.set_log_level(options[:loglevel])
|
24
27
|
ZfsMgmt.global_options = options
|
25
|
-
ZfsMgmt.snapshot_create(
|
28
|
+
ZfsMgmt.snapshot_create(noop: options[:noop], filter: options[:filter])
|
26
29
|
end
|
27
30
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zfs_mgmt
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aran Cox
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-03-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -130,6 +130,7 @@ files:
|
|
130
130
|
- lib/zfs_mgmt/zfs_mgr.rb
|
131
131
|
- lib/zfs_mgmt/zfs_mgr/list.rb
|
132
132
|
- lib/zfs_mgmt/zfs_mgr/restic.rb
|
133
|
+
- lib/zfs_mgmt/zfs_mgr/send.rb
|
133
134
|
- lib/zfs_mgmt/zfs_mgr/snapshot.rb
|
134
135
|
- zfs_mgmt.gemspec
|
135
136
|
homepage: https://github.com/aranc23/zfs_mgmt
|