bitferry 0.0.6 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/bitferry.rb CHANGED
@@ -1,1424 +1,1502 @@
1
- require 'json'
2
- require 'date'
3
- require 'open3'
4
- require 'logger'
5
- require 'pathname'
6
- require 'neatjson'
7
- require 'rbconfig'
8
- require 'fileutils'
9
- require 'shellwords'
10
-
11
-
12
- module Bitferry
13
-
14
-
15
- VERSION = '0.0.6'
16
-
17
-
18
- module Logging
19
- def self.log
20
- unless @log
21
- @log = Logger.new($stderr)
22
- @log.level = Logger::WARN
23
- @log.progname = :bitferry
24
- end
25
- @log
26
- end
27
- def log = Logging.log
28
- end
29
-
30
-
31
- include Logging
32
- extend Logging
33
-
34
-
35
- def self.tag = format('%08x', 2**32*rand)
36
-
37
-
38
- def self.restore
39
- reset
40
- log.info('restoring volumes')
41
- result = true
42
- roots = (environment_mounts + system_mounts + [Dir.home]).uniq
43
- log.info("distilled volume search path: #{roots.join(', ')}")
44
- roots.each do |root|
45
- if File.exist?(File.join(root, Volume::STORAGE))
46
- log.info("trying to restore volume from #{root}")
47
- Volume.restore(root) rescue result = false
48
- end
49
- end
50
- if result
51
- log.info('volumes restored')
52
- else
53
- log.warn('volume restore failure(s) reported')
54
- end
55
- result
56
- end
57
-
58
-
59
- def self.commit
60
- log.info('committing changes')
61
- result = true
62
- modified = false
63
- Volume.registered.each do |volume|
64
- begin
65
- modified = true if volume.modified?
66
- volume.commit
67
- rescue IOError => e
68
- log.error(e.message)
69
- result = false
70
- end
71
- end
72
- if result
73
- log.info(modified ? 'changes committed' : 'commits skipped (no changes)')
74
- else
75
- log.warn('commit failure(s) reported')
76
- end
77
- result
78
- end
79
-
80
-
81
- def self.reset
82
- log.info('resetting state')
83
- Volume.reset
84
- Task.reset
85
- end
86
-
87
-
88
- def self.intact_tasks = Volume.intact.collect { |volume| volume.intact_tasks }.flatten.uniq
89
-
90
- def self.process(*tags, &block)
91
- log.info('processing tasks')
92
- tasks = intact_tasks
93
- if tags.empty?
94
- process = tasks
95
- else
96
- process = []
97
- tags.each do |tag|
98
- case (tasks = Task.match([tag], tasks)).size
99
- when 0 then log.warn("no tasks matching (partial) tag #{tag}")
100
- when 1 then process += tasks
101
- else
102
- tags = tasks.collect { |v| v.tag }.join(', ')
103
- raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
104
- end
105
- end
106
- end
107
- tasks = process.uniq
108
- total = tasks.size
109
- processed = 0
110
- failed = 0
111
- result = tasks.all? do |task|
112
- r = task.process
113
- processed += 1
114
- failed += 1 unless r
115
- yield(total, processed, failed) if block_given?
116
- r
117
- end
118
- result ? log.info('tasks processed') : log.warn('task process failure(s) reported')
119
- result
120
- end
121
-
122
-
123
- def self.endpoint(root)
124
- case root
125
- when /^:(\w+):(.*)/
126
- volumes = Volume.lookup($1)
127
- volume = case volumes.size
128
- when 0 then raise ArgumentError, "no intact volume matching (partial) tag #{$1}"
129
- when 1 then volumes.first
130
- else
131
- tags = volumes.collect { |v| v.tag }.join(', ')
132
- raise ArgumentError, "multiple intact volumes matching (partial) tag #{$1}: #{tags}"
133
- end
134
- Endpoint::Bitferry.new(volume, $2)
135
- when /^(?:local)?:(.*)/ then Endpoint::Local.new($1)
136
- when /^(\w{2,}):(.*)/ then Endpoint::Rclone.new($1, $2)
137
- else Volume.endpoint(root)
138
- end
139
- end
140
-
141
-
142
- @simulate = false
143
- def self.simulate? = @simulate
144
- def self.simulate=(mode) @simulate = mode end
145
-
146
-
147
- @verbosity = :default
148
- def self.verbosity = @verbosity
149
- def self.verbosity=(mode) @verbosity = mode end
150
-
151
-
152
- # Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
153
- def self.windows?
154
- @windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
155
- end
156
-
157
- # Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
158
- # Look for the $BITFERRY_PATH environment variable
159
- def self.environment_mounts
160
- ENV['BITFERRY_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
161
- end
162
-
163
-
164
- # Specify OS-specific path name list separator (such as in the $PATH environment variable)
165
- PATH_LIST_SEPARATOR = windows? ? ';' : ':'
166
-
167
-
168
- # Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Bitferry voulmes
169
- UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc|efi)!
170
-
171
-
172
- # Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
173
- if RUBY_PLATFORM =~ /java/
174
- require 'java'
175
- def self.system_mounts
176
- java.nio.file.FileSystems.getDefault.getFileStores.collect {|x| /^(.*)\s+\(.*\)$/.match(x.to_s)[1]}
177
- end
178
- else
179
- case RbConfig::CONFIG['target_os']
180
- when 'linux'
181
- # Linux OS
182
- def self.system_mounts
183
- # Query /proc for currently mounted file systems
184
- IO.readlines('/proc/mounts').collect do |line|
185
- mount = line.split[1]
186
- UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
187
- end.compact
188
- end
189
- # TODO handle Windows variants
190
- when /^mingw/ # RubyInstaller's MRI
191
- module Kernel32
192
- require 'fiddle'
193
- require 'fiddle/types'
194
- require 'fiddle/import'
195
- extend Fiddle::Importer
196
- dlload('kernel32')
197
- include Fiddle::Win32Types
198
- extern 'DWORD WINAPI GetLogicalDrives()'
199
- end
200
- def self.system_mounts
201
- mounts = []
202
- mask = Kernel32.GetLogicalDrives
203
- ('A'..'Z').each do |x|
204
- mounts << "#{x}:/" if mask & 1 == 1
205
- mask >>= 1
206
- end
207
- mounts
208
- end
209
- else
210
- # Generic *NIX-like OS, including Cygwin & MSYS2
211
- def self.system_mounts
212
- # Use $(mount) system utility to obtain currently mounted file systems
213
- %x(mount).split("\n").collect do |line|
214
- mount = line.split[2]
215
- UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
216
- end.compact
217
- end
218
- end
219
- end
220
-
221
-
222
- class Volume
223
-
224
-
225
- include Logging
226
- extend Logging
227
-
228
-
229
- STORAGE = '.bitferry'
230
- STORAGE_ = '.bitferry~'
231
- STORAGE_MASK = '.bitferry*'
232
-
233
-
234
- attr_reader :tag
235
-
236
-
237
- attr_reader :generation
238
-
239
-
240
- attr_reader :root
241
-
242
-
243
- attr_reader :vault
244
-
245
-
246
- def self.[](tag)
247
- @@registry.each_value { |volume| return volume if volume.tag == tag }
248
- nil
249
- end
250
-
251
-
252
- # Return list of registered volumes whose tags match at least one specified partial
253
- def self.lookup(*tags) = match(tags, registered)
254
-
255
-
256
- def self.match(tags, volumes)
257
- rxs = tags.collect { |x| Regexp.new(x) }
258
- volumes.filter do |volume|
259
- rxs.any? { |rx| !(rx =~ volume.tag).nil? }
260
- end
261
- end
262
-
263
-
264
- def self.new(root, **opts)
265
- volume = allocate
266
- volume.send(:create, root, **opts)
267
- register(volume)
268
- end
269
-
270
-
271
- def self.restore(root)
272
- begin
273
- volume = allocate
274
- volume.send(:restore, root)
275
- volume = register(volume)
276
- log.info("restored volume #{volume.tag} from #{root}")
277
- volume
278
- rescue => e
279
- log.error("failed to restore volume from #{root}")
280
- log.error(e.message) if $DEBUG
281
- raise
282
- end
283
- end
284
-
285
-
286
- def self.delete(*tags, wipe: false)
287
- process = []
288
- tags.each do |tag|
289
- case (volumes = Volume.lookup(tag)).size
290
- when 0 then log.warn("no volumes matching (partial) tag #{tag}")
291
- when 1 then process += volumes
292
- else
293
- tags = volumes.collect { |v| v.tag }.join(', ')
294
- raise ArgumentError, "multiple volumes matching (partial) tag #{tag}: #{tags}"
295
- end
296
- end
297
- process.each { |volume| volume.delete(wipe: wipe) }
298
- end
299
-
300
-
301
- def initialize(root, tag: Bitferry.tag, modified: DateTime.now, overwrite: false)
302
- @tag = tag
303
- @generation = 0
304
- @vault = {}
305
- @modified = modified
306
- @overwrite = overwrite
307
- @root = Pathname.new(root).realdirpath
308
- end
309
-
310
-
311
- def create(*args, **opts)
312
- initialize(*args, **opts)
313
- @state = :pristine
314
- @modified = true
315
- end
316
-
317
-
318
- def restore(root)
319
- hash = JSON.load_file(storage = Pathname(root).join(STORAGE), { symbolize_names: true })
320
- raise IOError, "bad volume storage #{storage}" unless hash.fetch(:bitferry) == "0"
321
- initialize(root, tag: hash.fetch(:volume), modified: DateTime.parse(hash.fetch(:modified)))
322
- hash.fetch(:tasks, []).each { |hash| Task::ROUTE.fetch(hash.fetch(:operation).intern).restore(hash) }
323
- @vault = hash.fetch(:vault, {}).transform_keys { |key| key.to_s }
324
- @state = :intact
325
- @modified = false
326
- end
327
-
328
-
329
- def storage = @storage ||= root.join(STORAGE)
330
- def storage_ = @storage_ ||= root.join(STORAGE_)
331
-
332
-
333
- def commit
334
- if modified?
335
- log.info("commit volume #{tag} (modified)")
336
- case @state
337
- when :pristine
338
- format
339
- store
340
- when :intact
341
- store
342
- when :removing
343
- remove
344
- else
345
- raise
346
- end
347
- committed
348
- else
349
- log.info("skipped committing volume #{tag} (unmodified)")
350
- end
351
- end
352
-
353
-
354
- def self.endpoint(root)
355
- path = Pathname.new(root).realdirpath
356
- intact.sort { |v1, v2| v2.root.to_s.size <=> v1.root.to_s.size }.each do |volume|
357
- begin
358
- stem = path.relative_path_from(volume.root).to_s #.chomp('/')
359
- case stem
360
- when '.' then return volume.endpoint
361
- when /^[^\.].*/ then return volume.endpoint(stem)
362
- end
363
- rescue ArgumentError
364
- # Catch different prefix error on Windows
365
- end
366
- end
367
- raise ArgumentError, "no intact volume encompasses path #{root}"
368
- end
369
-
370
-
371
- def endpoint(path = String.new) = Endpoint::Bitferry.new(self, path)
372
-
373
-
374
- def modified? = @modified || tasks.any? { |t| t.generation > generation }
375
-
376
-
377
- def intact? = @state != :removing
378
-
379
-
380
- def touch
381
- x = tasks.collect { |t| t.generation }.max
382
- @generation = x ? x + 1 : 0
383
- @modified = true
384
- end
385
-
386
-
387
- def delete(wipe: false)
388
- touch
389
- @wipe = wipe
390
- @state = :removing
391
- log.info("marked volume #{tag} for deletion")
392
- end
393
-
394
-
395
- def committed
396
- x = tasks.collect { |t| t.generation }.min
397
- @generation = x ? x : 0
398
- @modified = false
399
- end
400
-
401
-
402
- def store
403
- tasks.each(&:commit)
404
- hash = JSON.neat_generate(externalize, short: false, wrap: 200, afterColon: 1, afterComma: 1)
405
- if Bitferry.simulate?
406
- log.info("skipped volume #{tag} storage modification (simulation)")
407
- else
408
- begin
409
- File.write(storage_, hash)
410
- FileUtils.mv(storage_, storage)
411
- log.info("written volume #{tag} storage #{storage}")
412
- ensure
413
- FileUtils.rm_f(storage_)
414
- end
415
- end
416
- @state = :intact
417
- end
418
-
419
-
420
- def format
421
- raise IOError, "refuse to overwrite existing volume storage #{storage}" if !@overwrite && File.exist?(storage)
422
- if Bitferry.simulate?
423
- log.info("skipped storage formatting (simulation)")
424
- else
425
- FileUtils.mkdir_p(root)
426
- FileUtils.rm_f [storage, storage_]
427
- log.info("formatted volume #{tag} in #{root}")
428
- end
429
- @state = nil
430
- end
431
-
432
-
433
- def remove
434
- unless Bitferry.simulate?
435
- if @wipe
436
- FileUtils.rm_rf(Dir[File.join(root, '*'), File.join(root, '.*')])
437
- log.info("wiped entire volume directory #{root}")
438
- else
439
- FileUtils.rm_f [storage, storage_]
440
- log.info("deleted volume #{tag} storage files #{File.join(root, STORAGE_MASK)}")
441
- end
442
- end
443
- @@registry.delete(root)
444
- @state = nil
445
- end
446
-
447
-
448
- def externalize
449
- tasks = live_tasks
450
- v = vault.filter { |t| !Task[t].nil? && Task[t].live? } # Purge entries from non-existing (deleted) tasks
451
- {
452
- bitferry: "0",
453
- volume: tag,
454
- modified: (@modified = DateTime.now),
455
- tasks: tasks.empty? ? nil : tasks.collect(&:externalize),
456
- vault: v.empty? ? nil : v
457
- }.compact
458
- end
459
-
460
-
461
- def tasks = Task.registered.filter { |task| task.refers?(self) }
462
-
463
-
464
- def live_tasks = Task.live.filter { |task| task.refers?(self) }
465
-
466
-
467
- def intact_tasks = live_tasks.filter { |task| task.intact? }
468
-
469
-
470
- def self.reset = @@registry = {}
471
-
472
-
473
- def self.register(volume) = @@registry[volume.root] = volume
474
-
475
-
476
- def self.registered = @@registry.values
477
-
478
-
479
- def self.intact = registered.filter { |volume| volume.intact? }
480
-
481
-
482
- end
483
-
484
-
485
- def self.optional(option, route)
486
- case option
487
- when Array then option # Array is passed verbatim
488
- when '-' then nil # Disable adding any options with -
489
- when /^-/ then option.split(',') # Split comma-separated string into array --foo,bar --> [--foo, bar]
490
- else route.fetch(option.nil? ? nil : option.to_sym) # Obtain options from the profile database
491
- end
492
- end
493
-
494
-
495
- class Task
496
-
497
-
498
- include Logging
499
- extend Logging
500
-
501
-
502
- attr_reader :tag
503
-
504
-
505
- attr_reader :generation
506
-
507
-
508
- attr_reader :modified
509
-
510
-
511
- attr_reader :include, :exclude
512
-
513
-
514
- def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
515
-
516
-
517
- def self.new(*args, **opts)
518
- task = allocate
519
- task.send(:create, *args, **opts)
520
- register(task)
521
- end
522
-
523
-
524
- def self.restore(hash)
525
- task = allocate
526
- task.send(:restore, hash)
527
- register(task)
528
- end
529
-
530
-
531
- def self.delete(*tags)
532
- process = []
533
- tags.each do |tag|
534
- case (tasks = Task.lookup(tag)).size
535
- when 0 then log.warn("no tasks matching (partial) tag #{tag}")
536
- when 1 then process += tasks
537
- else
538
- tags = tasks.collect { |v| v.tag }.join(', ')
539
- raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
540
- end
541
- end
542
- process.each { |task| task.delete }
543
- end
544
-
545
-
546
- def initialize(tag: Bitferry.tag, modified: DateTime.now, include: [], exclude: [])
547
- @tag = tag
548
- @generation = 0
549
- @include = include
550
- @exclude = exclude
551
- @modified = modified.is_a?(DateTime) ? modified : DateTime.parse(modified)
552
- # FIXME handle process_options at this level
553
- end
554
-
555
-
556
- def create(*args, **opts)
557
- initialize(*args, **opts)
558
- @state = :pristine
559
- touch
560
- end
561
-
562
-
563
- def restore(hash)
564
- @include = hash.fetch(:include, [])
565
- @exclude = hash.fetch(:exclude, [])
566
- @state = :intact
567
- log.info("restored task #{tag}")
568
- end
569
-
570
-
571
- # FIXME move to Endpoint#restore
572
- def restore_endpoint(x) = Endpoint::ROUTE.fetch(x.fetch(:endpoint).intern).restore(x)
573
-
574
-
575
- def externalize
576
- {
577
- task: tag,
578
- modified: modified,
579
- include: include.empty? ? nil : include,
580
- exclude: exclude.empty? ? nil : exclude
581
- }.compact
582
- end
583
-
584
-
585
- def live? = !@state.nil? && @state != :removing
586
-
587
-
588
- def touch = @modified = DateTime.now
589
-
590
-
591
- def delete
592
- touch
593
- @state = :removing
594
- log.info("marked task #{tag} for removal")
595
- end
596
-
597
-
598
- def commit
599
- case @state
600
- when :pristine then format
601
- when :removing then @state = nil
602
- end
603
- end
604
-
605
-
606
- def show_filters
607
- xs = []
608
- xs << 'include: ' + include.join(',') unless include.empty?
609
- xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
610
- xs.join(' ').to_s
611
- end
612
-
613
-
614
- def self.[](tag) = @@registry[tag]
615
-
616
-
617
- # Return list of registered tasks whose tags match at least one of specified partial tags
618
- def self.lookup(*tags) = match(tags, registered)
619
-
620
-
621
- # Return list of specified tasks whose tags match at least one of specified partial tags
622
- def self.match(tags, tasks)
623
- rxs = tags.collect { |x| Regexp.new(x) }
624
- tasks.filter do |task|
625
- rxs.any? { |rx| !(rx =~ task.tag).nil? }
626
- end
627
- end
628
-
629
-
630
- def self.registered = @@registry.values
631
-
632
-
633
- def self.live = registered.filter { |task| task.live? }
634
-
635
-
636
- def self.reset = @@registry = {}
637
-
638
-
639
- def self.register(task)
640
- # Task with newer timestamp replaces already registered task, if any
641
- if (xtag = @@registry[task.tag]).nil?
642
- @@registry[task.tag] = task
643
- elsif xtag.modified < task.modified
644
- @@registry[task.tag] = task
645
- else
646
- xtag
647
- end
648
- end
649
-
650
- def self.intact = live.filter { |task| task.intact? }
651
-
652
-
653
- def self.stale = live.filter { |task| !task.intact? }
654
-
655
-
656
- end
657
-
658
-
659
- module Rclone
660
-
661
-
662
- include Logging
663
- extend Logging
664
-
665
-
666
- def self.executable = @executable ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
667
-
668
-
669
- def self.exec(*args)
670
- cmd = [executable] + args
671
- log.debug(cmd.collect(&:shellescape).join(' '))
672
- stdout, status = Open3.capture2(*cmd)
673
- unless status.success?
674
- msg = "rclone exit code #{status.to_i}"
675
- log.error(msg)
676
- raise RuntimeError, msg
677
- end
678
- stdout.strip
679
- end
680
-
681
-
682
- def self.obscure(plain) = exec('obscure', '--', plain)
683
-
684
-
685
- def self.reveal(token) = exec('reveal', '--', token)
686
-
687
-
688
- class Encryption
689
-
690
-
691
- PROCESS = {
692
- default: ['--crypt-filename-encoding', :base32, '--crypt-filename-encryption', :standard],
693
- extended: ['--crypt-filename-encoding', :base32768, '--crypt-filename-encryption', :standard]
694
- }
695
- PROCESS[nil] = PROCESS[:default]
696
-
697
-
698
- def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
699
-
700
-
701
- def initialize(token, process: nil)
702
- @process_options = Bitferry.optional(process, PROCESS)
703
- @token = token
704
- end
705
-
706
-
707
- def create(password, **opts) = initialize(Rclone.obscure(password), **opts)
708
-
709
-
710
- def restore(hash) = @process_options = hash[:rclone]
711
-
712
-
713
- def externalize = process_options.empty? ? {} : { rclone: process_options }
714
-
715
-
716
- def configure(task) = install_token(task)
717
-
718
-
719
- def process(task) = ENV['RCLONE_CRYPT_PASSWORD'] = obtain_token(task)
720
-
721
-
722
- def arguments(task) = process_options + ['--crypt-remote', encrypted(task).root.to_s]
723
-
724
-
725
- def install_token(task)
726
- x = decrypted(task)
727
- raise TypeError, 'unsupported unencrypted endpoint type' unless x.is_a?(Endpoint::Bitferry)
728
- Volume[x.volume_tag].vault[task.tag] = @token # Token is stored on the decrypted end only
729
- end
730
-
731
-
732
- def obtain_token(task) = Volume[decrypted(task).volume_tag].vault.fetch(task.tag)
733
-
734
-
735
- def self.new(*args, **opts)
736
- obj = allocate
737
- obj.send(:create, *args, **opts)
738
- obj
739
- end
740
-
741
-
742
- def self.restore(hash)
743
- obj = ROUTE.fetch(hash.fetch(:operation).intern).allocate
744
- obj.send(:restore, hash)
745
- obj
746
- end
747
-
748
-
749
- end
750
-
751
-
752
- class Encrypt < Encryption
753
-
754
-
755
- def encrypted(task) = task.destination
756
-
757
-
758
- def decrypted(task) = task.source
759
-
760
-
761
- def externalize = super.merge(operation: :encrypt)
762
-
763
-
764
- def show_operation = 'encrypt+'
765
-
766
-
767
- def arguments(task) = super + [decrypted(task).root.to_s, ':crypt:']
768
-
769
-
770
- end
771
-
772
-
773
- class Decrypt < Encryption
774
-
775
-
776
- def encrypted(task) = task.source
777
-
778
-
779
- def decrypted(task) = task.destination
780
-
781
-
782
- def externalize = super.merge(operation: :decrypt)
783
-
784
-
785
- def show_operation = 'decrypt+'
786
-
787
-
788
- def arguments(task) = super + [':crypt:', decrypted(task).root.to_s]
789
-
790
-
791
- end
792
-
793
-
794
- ROUTE = {
795
- encrypt: Encrypt,
796
- decrypt: Decrypt
797
- }
798
-
799
-
800
- class Task < Bitferry::Task
801
-
802
-
803
- attr_reader :source, :destination
804
-
805
-
806
- attr_reader :encryption
807
-
808
-
809
- attr_reader :token
810
-
811
-
812
- PROCESS = {
813
- default: ['--metadata']
814
- }
815
- PROCESS[nil] = PROCESS[:default]
816
-
817
-
818
- def initialize(source, destination, encryption: nil, process: nil, **opts)
819
- super(**opts)
820
- @process_options = Bitferry.optional(process, PROCESS)
821
- @source = source.is_a?(Endpoint) ? source : Bitferry.endpoint(source)
822
- @destination = destination.is_a?(Endpoint) ? destination : Bitferry.endpoint(destination)
823
- @encryption = encryption
824
- end
825
-
826
-
827
- def create(*args, process: nil, **opts)
828
- super(*args, process: process, **opts)
829
- encryption.configure(self) unless encryption.nil?
830
- end
831
-
832
-
833
- def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
834
-
835
-
836
- def show_operation = encryption.nil? ? '' : encryption.show_operation
837
-
838
-
839
- def show_direction = '-->'
840
-
841
-
842
- def intact? = live? && source.intact? && destination.intact?
843
-
844
-
845
- def refers?(volume) = source.refers?(volume) || destination.refers?(volume)
846
-
847
-
848
- def touch
849
- @generation = [source.generation, destination.generation].max + 1
850
- super
851
- end
852
-
853
-
854
- def format = nil
855
-
856
-
857
- def common_options
858
- [
859
- '--config', Bitferry.windows? ? 'NUL' : '/dev/null',
860
- case Bitferry.verbosity
861
- when :verbose then '--verbose'
862
- when :quiet then '--quiet'
863
- else nil
864
- end,
865
- Bitferry.verbosity == :verbose ? '--progress' : nil,
866
- Bitferry.simulate? ? '--dry-run' : nil,
867
- ].compact
868
- end
869
-
870
-
871
- def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
872
-
873
-
874
- def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
875
-
876
-
877
- def process_arguments
878
- include_filters + exclude_filters + common_options + process_options + (
879
- encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
880
- )
881
- end
882
-
883
-
884
- def execute(*args)
885
- cmd = [Rclone.executable] + args
886
- cms = cmd.collect(&:shellescape).join(' ')
887
- puts cms if Bitferry.verbosity == :verbose
888
- log.info(cms)
889
- status = Open3.pipeline(cmd).first
890
- raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
891
- status.success?
892
- end
893
-
894
-
895
- def process
896
- log.info("processing task #{tag}")
897
- encryption.process(self) unless encryption.nil?
898
- execute(*process_arguments)
899
- end
900
-
901
-
902
- def externalize
903
- super.merge(
904
- source: source.externalize,
905
- destination: destination.externalize,
906
- encryption: encryption.nil? ? nil : encryption.externalize,
907
- rclone: process_options.empty? ? nil : process_options
908
- ).compact
909
- end
910
-
911
-
912
- def restore(hash)
913
- initialize(
914
- restore_endpoint(hash.fetch(:source)),
915
- restore_endpoint(hash.fetch(:destination)),
916
- tag: hash.fetch(:task),
917
- modified: hash.fetch(:modified, DateTime.now),
918
- process: hash[:rclone],
919
- encryption: hash[:encryption].nil? ? nil : Rclone::Encryption.restore(hash[:encryption])
920
- )
921
- super(hash)
922
- end
923
-
924
-
925
- end
926
-
927
-
928
- class Copy < Task
929
-
930
-
931
- def process_arguments = ['copy'] + super
932
-
933
-
934
- def externalize = super.merge(operation: :copy)
935
-
936
-
937
- def show_operation = super + 'copy'
938
-
939
-
940
- end
941
-
942
-
943
- class Update < Task
944
-
945
-
946
- def process_arguments = ['copy', '--update'] + super
947
-
948
-
949
- def externalize = super.merge(operation: :update)
950
-
951
-
952
- def show_operation = super + 'update'
953
-
954
-
955
- end
956
-
957
-
958
- class Synchronize < Task
959
-
960
-
961
- def process_arguments = ['sync'] + super
962
-
963
-
964
- def externalize = super.merge(operation: :synchronize)
965
-
966
-
967
- def show_operation = super + 'synchronize'
968
-
969
-
970
- end
971
-
972
-
973
- class Equalize < Task
974
-
975
-
976
- def process_arguments = ['bisync', '--resync'] + super
977
-
978
-
979
- def externalize = super.merge(operation: :equalize)
980
-
981
-
982
- def show_operation = super + 'equalize'
983
-
984
-
985
- def show_direction = '<->'
986
-
987
-
988
- end
989
-
990
-
991
- end
992
-
993
-
994
- module Restic
995
-
996
-
997
- include Logging
998
- extend Logging
999
-
1000
-
1001
- def self.executable = @executable ||= (restic = ENV['RESTIC']).nil? ? 'restic' : restic
1002
-
1003
-
1004
- def self.exec(*args)
1005
- cmd = [executable] + args
1006
- log.debug(cmd.collect(&:shellescape).join(' '))
1007
- stdout, status = Open3.capture2(*cmd)
1008
- unless status.success?
1009
- msg = "restic exit code #{status.to_i}"
1010
- log.error(msg)
1011
- raise RuntimeError, msg
1012
- end
1013
- stdout.strip
1014
- end
1015
-
1016
-
1017
- class Task < Bitferry::Task
1018
-
1019
-
1020
- attr_reader :directory, :repository
1021
-
1022
-
1023
- def initialize(directory, repository, **opts)
1024
- super(**opts)
1025
- @directory = directory.is_a?(Endpoint) ? directory : Bitferry.endpoint(directory)
1026
- @repository = repository.is_a?(Endpoint) ? repository : Bitferry.endpoint(repository)
1027
- end
1028
-
1029
-
1030
- def create(directory, repository, password, **opts)
1031
- super(directory, repository, **opts)
1032
- raise TypeError, 'unsupported unencrypted endpoint type' unless self.directory.is_a?(Endpoint::Bitferry)
1033
- Volume[self.directory.volume_tag].vault[tag] = Rclone.obscure(@password = password) # Token is stored on the decrypted end only
1034
- end
1035
-
1036
-
1037
- def password = @password ||= Rclone.reveal(Volume[directory.volume_tag].vault.fetch(tag))
1038
-
1039
-
1040
- def intact? = live? && directory.intact? && repository.intact?
1041
-
1042
-
1043
- def refers?(volume) = directory.refers?(volume) || repository.refers?(volume)
1044
-
1045
-
1046
- def touch
1047
- @generation = [directory.generation, repository.generation].max + 1
1048
- super
1049
- end
1050
-
1051
-
1052
- def format = nil
1053
-
1054
-
1055
- def include_filters = include.collect { |x| ['--include', x]}.flatten
1056
-
1057
-
1058
- def common_options
1059
- [
1060
- case Bitferry.verbosity
1061
- when :verbose then '--verbose'
1062
- when :quiet then '--quiet'
1063
- else nil
1064
- end,
1065
- '-r', repository.root.to_s
1066
- ].compact
1067
- end
1068
-
1069
-
1070
- def execute(*args, simulate: false, chdir: nil)
1071
- cmd = [Restic.executable] + args
1072
- ENV['RESTIC_PASSWORD'] = password
1073
- cms = cmd.collect(&:shellescape).join(' ')
1074
- puts cms if Bitferry.verbosity == :verbose
1075
- log.info(cms)
1076
- if simulate
1077
- log.info('(simulated)')
1078
- true
1079
- else
1080
- wd = Dir.getwd unless chdir.nil?
1081
- begin
1082
- Dir.chdir(chdir) unless chdir.nil?
1083
- status = Open3.pipeline(cmd).first
1084
- raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
1085
- status.success?
1086
- ensure
1087
- Dir.chdir(wd) unless chdir.nil?
1088
- end
1089
- end
1090
- end
1091
-
1092
-
1093
- def externalize
1094
- super.merge(
1095
- directory: directory.externalize,
1096
- repository: repository.externalize,
1097
- ).compact
1098
- end
1099
-
1100
-
1101
- def restore(hash)
1102
- initialize(
1103
- restore_endpoint(hash.fetch(:directory)),
1104
- restore_endpoint(hash.fetch(:repository)),
1105
- tag: hash.fetch(:task),
1106
- modified: hash.fetch(:modified, DateTime.now)
1107
- )
1108
- super(hash)
1109
- end
1110
-
1111
-
1112
- end
1113
-
1114
-
1115
- class Backup < Task
1116
-
1117
-
1118
- PROCESS = {
1119
- default: ['--no-cache']
1120
- }
1121
- PROCESS[nil] = PROCESS[:default]
1122
-
1123
-
1124
- FORGET = {
1125
- default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
1126
- }
1127
- FORGET[nil] = nil # Skip processing retention policy by default
1128
-
1129
-
1130
- CHECK = {
1131
- default: ['--no-cache'],
1132
- full: ['--no-cache', '--read-data']
1133
- }
1134
- CHECK[nil] = nil # Skip integrity checking by default
1135
-
1136
-
1137
- attr_reader :forget_options
1138
- attr_reader :check_options
1139
-
1140
-
1141
- def create(*args, format: nil, process: nil, forget: nil, check: nil, **opts)
1142
- super(*args, **opts)
1143
- @format = format
1144
- @process_options = Bitferry.optional(process, PROCESS)
1145
- @forget_options = Bitferry.optional(forget, FORGET)
1146
- @check_options = Bitferry.optional(check, CHECK)
1147
- end
1148
-
1149
-
1150
- def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
1151
-
1152
-
1153
- def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
1154
-
1155
-
1156
- def show_operation = 'encrypt+backup'
1157
-
1158
-
1159
- def show_direction = '-->'
1160
-
1161
-
1162
- def process
1163
- begin
1164
- log.info("processing task #{tag}")
1165
- execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
1166
- unless check_options.nil?
1167
- log.info("checking repository in #{repository.root}")
1168
- execute('check', *check_options, *common_options)
1169
- end
1170
- unless forget_options.nil?
1171
- log.info("performing repository maintenance tasks in #{repository.root}")
1172
- execute('forget', '--tag', "bitferry,#{tag}", *forget_options.collect(&:to_s), *common_options_simulate)
1173
- end
1174
- true
1175
- rescue
1176
- false
1177
- end
1178
- end
1179
-
1180
-
1181
- def common_options_simulate = common_options + [Bitferry.simulate? ? '--dry-run' : nil].compact
1182
-
1183
-
1184
- def externalize
1185
- restic = {
1186
- process: process_options,
1187
- forget: forget_options,
1188
- check: check_options
1189
- }.compact
1190
- super.merge({
1191
- operation: :backup,
1192
- restic: restic.empty? ? nil : restic
1193
- }.compact)
1194
- end
1195
-
1196
-
1197
- def restore(hash)
1198
- super
1199
- opts = hash.fetch(:restic, {})
1200
- @process_options = opts[:process]
1201
- @forget_options = opts[:forget]
1202
- @check_options = opts[:check]
1203
- end
1204
-
1205
-
1206
- def format
1207
- if Bitferry.simulate?
1208
- log.info('skipped repository initialization (simulation)')
1209
- else
1210
- log.info("initializing repository for task #{tag}")
1211
- if @format == true
1212
- log.debug("wiping repository in #{repository.root}")
1213
- ['config', 'data', 'index', 'keys', 'locks', 'snapshots'].each { |x| FileUtils.rm_rf(File.join(repository.root.to_s, x)) }
1214
- end
1215
- if @format == false
1216
- # TODO validate existing repo
1217
- log.info("attached to existing repository for task #{tag} in #{repository.root}")
1218
- else
1219
- begin
1220
- execute(*common_options, 'init')
1221
- log.info("initialized repository for task #{tag} in #{repository.root}")
1222
- rescue
1223
- log.fatal("failed to initialize repository for task #{tag} in #{repository.root}")
1224
- raise
1225
- end
1226
- end
1227
- end
1228
- @state = :intact
1229
- end
1230
-
1231
-
1232
- end
1233
-
1234
-
1235
- class Restore < Task
1236
-
1237
-
1238
- PROCESS = {
1239
- default: ['--no-cache', '--sparse']
1240
- }
1241
- PROCESS[nil] = PROCESS[:default]
1242
-
1243
-
1244
- def create(*args, process: nil, **opts)
1245
- super(*args, **opts)
1246
- @process_options = Bitferry.optional(process, PROCESS)
1247
- end
1248
-
1249
-
1250
- def exclude_filters = exclude.collect { |x| ['--exclude', x]}.flatten
1251
-
1252
-
1253
- def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
1254
-
1255
-
1256
- def show_operation = 'decrypt+restore'
1257
-
1258
-
1259
- def show_direction = '-->'
1260
-
1261
-
1262
- def externalize
1263
- restic = {
1264
- process: process_options
1265
- }.compact
1266
- super.merge({
1267
- operation: :restore,
1268
- restic: restic.empty? ? nil : restic
1269
- }.compact)
1270
- end
1271
-
1272
-
1273
- def restore(hash)
1274
- super
1275
- opts = hash.fetch(:rclone, {})
1276
- @process_options = opts[:process]
1277
- end
1278
-
1279
-
1280
- def process
1281
- log.info("processing task #{tag}")
1282
- begin
1283
- # FIXME restore specifically tagged latest snapshot
1284
- execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
1285
- true
1286
- rescue
1287
- false
1288
- end
1289
- end
1290
-
1291
-
1292
- end
1293
-
1294
-
1295
-
1296
- end
1297
-
1298
-
1299
- Task::ROUTE = {
1300
- copy: Rclone::Copy,
1301
- update: Rclone::Update,
1302
- synchronize: Rclone::Synchronize,
1303
- equalize: Rclone::Equalize,
1304
- backup: Restic::Backup,
1305
- restore: Restic::Restore
1306
- }
1307
-
1308
-
1309
- class Endpoint
1310
-
1311
-
1312
- def self.restore(hash)
1313
- endpoint = allocate
1314
- endpoint.send(:restore, hash)
1315
- endpoint
1316
- end
1317
-
1318
-
1319
- class Local < Endpoint
1320
-
1321
-
1322
- attr_reader :root
1323
-
1324
-
1325
- def initialize(root) = @root = Pathname.new(root).realdirpath
1326
-
1327
-
1328
- def restore(hash) = initialize(hash.fetch(:root))
1329
-
1330
-
1331
- def externalize
1332
- {
1333
- endpoint: :local,
1334
- root: root
1335
- }
1336
- end
1337
-
1338
-
1339
- def show_status = root.to_s
1340
-
1341
-
1342
- def intact? = true
1343
-
1344
-
1345
- def refers?(volume) = false
1346
-
1347
-
1348
- def generation = 0
1349
-
1350
-
1351
- end
1352
-
1353
-
1354
- class Rclone < Endpoint
1355
- # TODO
1356
- end
1357
-
1358
-
1359
- class Bitferry < Endpoint
1360
-
1361
-
1362
- attr_reader :volume_tag
1363
-
1364
-
1365
- attr_reader :path
1366
-
1367
-
1368
- def root = Volume[volume_tag].root.join(path)
1369
-
1370
-
1371
- def initialize(volume, path)
1372
- @volume_tag = volume.tag
1373
- @path = Pathname.new(path)
1374
- raise ArgumentError, "expected relative path but got #{self.path}" unless (/^[\.\/]/ =~ self.path.to_s).nil?
1375
- end
1376
-
1377
-
1378
- def restore(hash)
1379
- @volume_tag = hash.fetch(:volume)
1380
- @path = Pathname.new(hash.fetch(:path, ''))
1381
- end
1382
-
1383
-
1384
- def externalize
1385
- {
1386
- endpoint: :bitferry,
1387
- volume: volume_tag,
1388
- path: path.to_s.empty? ? nil : path
1389
- }.compact
1390
- end
1391
-
1392
-
1393
- def show_status = intact? ? ":#{volume_tag}:#{path}" : ":{#{volume_tag}}:#{path}"
1394
-
1395
-
1396
- def intact? = !Volume[volume_tag].nil?
1397
-
1398
-
1399
- def refers?(volume) = volume.tag == volume_tag
1400
-
1401
-
1402
- def generation
1403
- v = Volume[volume_tag]
1404
- v ? v.generation : 0
1405
- end
1406
-
1407
-
1408
- end
1409
-
1410
-
1411
- ROUTE = {
1412
- local: Local,
1413
- rclone: Rclone,
1414
- bitferry: Bitferry
1415
- }
1416
-
1417
-
1418
- end
1419
-
1420
-
1421
- reset
1422
-
1423
-
1424
- end
1
+ require 'json'
2
+ require 'date'
3
+ require 'open3'
4
+ require 'base64'
5
+ require 'logger'
6
+ require 'openssl'
7
+ require 'pathname'
8
+ require 'rbconfig'
9
+ require 'fileutils'
10
+ require 'shellwords'
11
+
12
+
13
+ module Bitferry
14
+
15
+
16
+ VERSION = '0.0.8'
17
+
18
+
19
+ module Logging
20
+ def self.log
21
+ unless @log
22
+ @log = Logger.new($stderr)
23
+ @log.level = Logger::WARN
24
+ @log.progname = :bitferry
25
+ end
26
+ @log
27
+ end
28
+ def self.log=(log) @log = log end
29
+ def log = Logging.log
30
+ end
31
+
32
+
33
+ include Logging
34
+ extend Logging
35
+
36
+
37
+ def self.tag = format('%08x', 2**32*rand)
38
+
39
+
40
+ def self.restore
41
+ reset
42
+ log.info('restoring volumes')
43
+ result = true
44
+ roots = (environment_mounts + system_mounts + [Dir.home]).uniq
45
+ log.info("distilled volume search path: #{roots.join(', ')}")
46
+ roots.each do |root|
47
+ if File.exist?(File.join(root, Volume::STORAGE))
48
+ log.info("trying to restore volume from #{root}")
49
+ Volume.restore(root) rescue result = false
50
+ end
51
+ end
52
+ if result
53
+ log.info('volumes restored')
54
+ else
55
+ log.warn('volume restore failure(s) reported')
56
+ end
57
+ result
58
+ end
59
+
60
+
61
+ def self.commit
62
+ log.info('committing changes')
63
+ result = true
64
+ modified = false
65
+ Volume.registered.each do |volume|
66
+ begin
67
+ modified = true if volume.modified?
68
+ volume.commit
69
+ rescue IOError => e
70
+ log.error(e.message)
71
+ result = false
72
+ end
73
+ end
74
+ if result
75
+ log.info(modified ? 'changes committed' : 'commits skipped (no changes)')
76
+ else
77
+ log.warn('commit failure(s) reported')
78
+ end
79
+ result
80
+ end
81
+
82
+
83
+ def self.reset
84
+ log.info('resetting state')
85
+ Volume.reset
86
+ Task.reset
87
+ end
88
+
89
+
90
+ def self.intact_tasks = Volume.intact.collect { |volume| volume.intact_tasks }.flatten.uniq
91
+
92
+ def self.process(*tags, &block)
93
+ log.info('processing tasks')
94
+ tasks = intact_tasks
95
+ if tags.empty?
96
+ process = tasks
97
+ else
98
+ process = []
99
+ tags.each do |tag|
100
+ case (tasks = Task.match([tag], tasks)).size
101
+ when 0 then log.warn("no tasks matching (partial) tag #{tag}")
102
+ when 1 then process += tasks
103
+ else
104
+ tags = tasks.collect { |v| v.tag }.join(', ')
105
+ raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
106
+ end
107
+ end
108
+ end
109
+ tasks = process.uniq
110
+ total = tasks.size
111
+ processed = 0
112
+ failed = 0
113
+ result = tasks.all? do |task|
114
+ r = task.process
115
+ processed += 1
116
+ failed += 1 unless r
117
+ yield(total, processed, failed) if block_given?
118
+ r
119
+ end
120
+ result ? log.info('tasks processed') : log.warn('task process failure(s) reported')
121
+ result
122
+ end
123
+
124
+
125
+ def self.endpoint(root)
126
+ case root
127
+ when /^:(\w+):(.*)/
128
+ volumes = Volume.lookup($1)
129
+ volume = case volumes.size
130
+ when 0 then raise ArgumentError, "no intact volume matching (partial) tag #{$1}"
131
+ when 1 then volumes.first
132
+ else
133
+ tags = volumes.collect { |v| v.tag }.join(', ')
134
+ raise ArgumentError, "multiple intact volumes matching (partial) tag #{$1}: #{tags}"
135
+ end
136
+ Endpoint::Bitferry.new(volume, $2)
137
+ when /^(?:local)?:(.*)/ then Endpoint::Local.new($1)
138
+ when /^(\w{2,}):(.*)/ then Endpoint::Rclone.new($1, $2)
139
+ else Volume.endpoint(root)
140
+ end
141
+ end
142
+
143
+
144
+ @simulate = false
145
+ def self.simulate? = @simulate
146
+ def self.simulate=(mode) @simulate = mode end
147
+
148
+
149
+ @verbosity = :default
150
+ def self.verbosity = @verbosity
151
+ def self.verbosity=(mode) @verbosity = mode end
152
+
153
+
154
+ @ui = :cli
155
+ def self.ui = @ui
156
+ def self.ui=(ui) @ui = ui end
157
+
158
+
159
+ # Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
160
+ def self.windows?
161
+ @windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
162
+ end
163
+
164
+ # Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
165
+ # Look for the $BITFERRY_PATH environment variable
166
+ def self.environment_mounts
167
+ ENV['BITFERRY_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
168
+ end
169
+
170
+
171
+ # Specify OS-specific path name list separator (such as in the $PATH environment variable)
172
+ PATH_LIST_SEPARATOR = windows? ? ';' : ':'
173
+
174
+
175
+ # Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Bitferry voulmes
176
+ UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc|efi)!
177
+
178
+
179
+ # Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
180
+ if RUBY_PLATFORM =~ /java/
181
+ require 'java'
182
+ def self.system_mounts
183
+ java.nio.file.FileSystems.getDefault.getFileStores.collect {|x| /^(.*)\s+\(.*\)$/.match(x.to_s)[1]}
184
+ end
185
+ else
186
+ case RbConfig::CONFIG['target_os']
187
+ when 'linux'
188
+ # Linux OS
189
+ def self.system_mounts
190
+ # Query /proc for currently mounted file systems
191
+ IO.readlines('/proc/mounts').collect do |line|
192
+ mount = line.split[1]
193
+ UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
194
+ end.compact
195
+ end
196
+ # TODO handle Windows variants
197
+ when /^mingw/ # RubyInstaller's MRI
198
+ module Kernel32
199
+ require 'fiddle'
200
+ require 'fiddle/types'
201
+ require 'fiddle/import'
202
+ extend Fiddle::Importer
203
+ dlload('kernel32')
204
+ include Fiddle::Win32Types
205
+ extern 'DWORD WINAPI GetLogicalDrives()'
206
+ end
207
+ def self.system_mounts
208
+ mounts = []
209
+ mask = Kernel32.GetLogicalDrives
210
+ ('A'..'Z').each do |x|
211
+ mounts << "#{x}:/" if mask & 1 == 1
212
+ mask >>= 1
213
+ end
214
+ mounts
215
+ end
216
+ else
217
+ # Generic *NIX-like OS, including Cygwin & MSYS2
218
+ def self.system_mounts
219
+ # Use $(mount) system utility to obtain currently mounted file systems
220
+ %x(mount).split("\n").collect do |line|
221
+ mount = line.split[2]
222
+ UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
223
+ end.compact
224
+ end
225
+ end
226
+ end
227
+
228
+
229
+ class Volume
230
+
231
+
232
+ include Logging
233
+ extend Logging
234
+
235
+
236
+ STORAGE = '.bitferry'
237
+ STORAGE_ = '.bitferry~'
238
+ STORAGE_MASK = '.bitferry*'
239
+
240
+
241
+ attr_reader :tag
242
+
243
+
244
+ attr_reader :generation
245
+
246
+
247
+ attr_reader :root
248
+
249
+
250
+ attr_reader :vault
251
+
252
+
253
+ def self.[](tag)
254
+ @@registry.each_value { |volume| return volume if volume.tag == tag }
255
+ nil
256
+ end
257
+
258
+
259
+ # Return list of registered volumes whose tags match at least one specified partial
260
+ def self.lookup(*tags) = match(tags, registered)
261
+
262
+
263
+ def self.match(tags, volumes)
264
+ rxs = tags.collect { |x| Regexp.new(x) }
265
+ volumes.filter do |volume|
266
+ rxs.any? { |rx| !(rx =~ volume.tag).nil? }
267
+ end
268
+ end
269
+
270
+
271
+ def self.new(root, **opts)
272
+ volume = allocate
273
+ volume.send(:create, root, **opts)
274
+ register(volume)
275
+ end
276
+
277
+
278
+ def self.restore(root)
279
+ begin
280
+ volume = allocate
281
+ volume.send(:restore, root)
282
+ volume = register(volume)
283
+ log.info("restored volume #{volume.tag} from #{root}")
284
+ volume
285
+ rescue => e
286
+ log.error("failed to restore volume from #{root}")
287
+ log.error(e.message) if $DEBUG
288
+ raise
289
+ end
290
+ end
291
+
292
+
293
+ def self.delete(*tags, wipe: false)
294
+ process = []
295
+ tags.each do |tag|
296
+ case (volumes = Volume.lookup(tag)).size
297
+ when 0 then log.warn("no volumes matching (partial) tag #{tag}")
298
+ when 1 then process += volumes
299
+ else
300
+ tags = volumes.collect { |v| v.tag }.join(', ')
301
+ raise ArgumentError, "multiple volumes matching (partial) tag #{tag}: #{tags}"
302
+ end
303
+ end
304
+ process.each { |volume| volume.delete(wipe: wipe) }
305
+ end
306
+
307
+
308
+ def initialize(root, tag: Bitferry.tag, modified: nil, overwrite: false)
309
+ @tag = tag
310
+ @generation = 0
311
+ @vault = {}
312
+ @modified = case modified
313
+ when nil then DateTime.now
314
+ when DateTime then modified
315
+ else DateTime.parse(modified)
316
+ end
317
+ @overwrite = overwrite
318
+ @root = Pathname.new(root).realdirpath
319
+ end
320
+
321
+
322
+ def create(*args, **opts)
323
+ initialize(*args, **opts)
324
+ @state = :pristine
325
+ @modified = true
326
+ end
327
+
328
+
329
+ def restore(root)
330
+ hash = JSON.load_file(storage = Pathname(root).join(STORAGE), { symbolize_names: true })
331
+ raise IOError, "bad volume storage #{storage}" unless hash.fetch(:bitferry) == "0"
332
+ volume = hash.fetch(:volume)
333
+ begin
334
+ modified = DateTime.parse(hash.fetch(:modified))
335
+ @modified = false
336
+ rescue
337
+ modified = nil
338
+ @modified = true
339
+ log.warn("modified key missing - flagging volume #{volume} out of date")
340
+ end
341
+ initialize(root, tag: volume, modified: modified)
342
+ hash.fetch(:tasks, []).each { |hash| Task::ROUTE.fetch(hash.fetch(:operation).intern).restore(hash) }
343
+ @vault = hash.fetch(:vault, {}).transform_keys { |key| key.to_s }
344
+ @state = :intact
345
+ end
346
+
347
+
348
+ def storage = @storage ||= root.join(STORAGE)
349
+ def storage_ = @storage_ ||= root.join(STORAGE_)
350
+
351
+
352
+ def commit
353
+ if modified?
354
+ log.info("commit volume #{tag} (modified)")
355
+ case @state
356
+ when :pristine
357
+ format
358
+ store
359
+ when :intact
360
+ store
361
+ when :removing
362
+ remove
363
+ else
364
+ raise
365
+ end
366
+ committed
367
+ else
368
+ log.info("skipped committing volume #{tag} (unmodified)")
369
+ end
370
+ end
371
+
372
+
373
+ ENCOMPASSING_PATH_PREFIX = /^(?!(\.\.$|\.\.\/))/ # not .. by itself and not starting with ../
374
+
375
+
376
+ def self.endpoint(root)
377
+ path = Pathname.new(root).realdirpath
378
+ intact.sort { |v1, v2| v2.root.to_s.size <=> v1.root.to_s.size }.each do |volume|
379
+ begin
380
+ stem = path.relative_path_from(volume.root).to_s #.chomp('/')
381
+ case stem
382
+ when '.' then return volume.endpoint
383
+ when ENCOMPASSING_PATH_PREFIX then return volume.endpoint(stem)
384
+ end
385
+ rescue ArgumentError
386
+ # Catch different prefix error on Windows
387
+ end
388
+ end
389
+ raise ArgumentError, "no intact volume encompasses path #{root}"
390
+ end
391
+
392
+
393
+ def endpoint(path = String.new) = Endpoint::Bitferry.new(self, path)
394
+
395
+
396
+ def modified? = @modified || tasks.any? { |t| t.generation > generation }
397
+
398
+
399
+ def intact? = @state != :removing
400
+
401
+
402
+ def touch
403
+ x = tasks.collect { |t| t.generation }.max
404
+ @generation = x ? x + 1 : 0
405
+ @modified = true
406
+ end
407
+
408
+
409
+ def delete(wipe: false)
410
+ touch
411
+ @wipe = wipe
412
+ @state = :removing
413
+ log.info("marked volume #{tag} for deletion")
414
+ end
415
+
416
+
417
+ def committed
418
+ x = tasks.collect { |t| t.generation }.min
419
+ @generation = x ? x : 0
420
+ @modified = false
421
+ end
422
+
423
+
424
+ def store
425
+ require 'neatjson'
426
+ tasks.each(&:commit)
427
+ hash = JSON.neat_generate(externalize, short: false, wrap: 200, afterColon: 1, afterComma: 1)
428
+ if Bitferry.simulate?
429
+ log.info("skipped volume #{tag} storage modification (simulation)")
430
+ else
431
+ begin
432
+ File.write(storage_, hash)
433
+ FileUtils.mv(storage_, storage)
434
+ log.info("written volume #{tag} storage #{storage}")
435
+ ensure
436
+ FileUtils.rm_f(storage_)
437
+ end
438
+ end
439
+ @state = :intact
440
+ end
441
+
442
+
443
+ def format
444
+ raise IOError, "refuse to overwrite existing volume storage #{storage}" if !@overwrite && File.exist?(storage)
445
+ if Bitferry.simulate?
446
+ log.info("skipped storage formatting (simulation)")
447
+ else
448
+ FileUtils.mkdir_p(root)
449
+ FileUtils.rm_f [storage, storage_]
450
+ log.info("formatted volume #{tag} in #{root}")
451
+ end
452
+ @state = nil
453
+ end
454
+
455
+
456
+ def remove
457
+ unless Bitferry.simulate?
458
+ if @wipe
459
+ FileUtils.rm_rf(Dir[File.join(root, '*'), File.join(root, '.*')])
460
+ log.info("wiped entire volume directory #{root}")
461
+ else
462
+ FileUtils.rm_f [storage, storage_]
463
+ log.info("deleted volume #{tag} storage files #{File.join(root, STORAGE_MASK)}")
464
+ end
465
+ end
466
+ @@registry.delete(root)
467
+ @state = nil
468
+ end
469
+
470
+
471
+ def externalize
472
+ tasks = live_tasks
473
+ v = vault.filter { |t| !Task[t].nil? && Task[t].live? } # Purge entries from non-existing (deleted) tasks
474
+ {
475
+ bitferry: "0",
476
+ volume: tag,
477
+ modified: (@modified = DateTime.now),
478
+ tasks: tasks.empty? ? nil : tasks.collect(&:externalize),
479
+ vault: v.empty? ? nil : v
480
+ }.compact
481
+ end
482
+
483
+
484
+ def tasks = Task.registered.filter { |task| task.refers?(self) }
485
+
486
+
487
+ def live_tasks = Task.live.filter { |task| task.refers?(self) }
488
+
489
+
490
+ def intact_tasks = live_tasks.filter { |task| task.intact? }
491
+
492
+
493
+ def self.reset = @@registry = {}
494
+
495
+
496
+ def self.register(volume) = @@registry[volume.root] = volume
497
+
498
+
499
+ def self.registered = @@registry.values
500
+
501
+
502
+ def self.intact = registered.filter { |volume| volume.intact? }
503
+
504
+
505
+ end
506
+
507
+
508
+ def self.optional(option, route)
509
+ case option
510
+ when Array then option # Array is passed verbatim
511
+ when '-' then nil # Disable adding any options with -
512
+ when /^-/ then option.split(',') # Split comma-separated string into array --foo,bar --> [--foo, bar]
513
+ else route.fetch(option.nil? ? nil : option.to_sym) # Obtain options from the profile database
514
+ end
515
+ end
516
+
517
+
518
+ class Task
519
+
520
+
521
+ include Logging
522
+ extend Logging
523
+
524
+
525
+ attr_reader :tag
526
+
527
+
528
+ attr_reader :generation
529
+
530
+
531
+ attr_reader :modified
532
+
533
+
534
+ attr_reader :include, :exclude
535
+
536
+
537
+ def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
538
+
539
+
540
+ def self.new(*args, **opts)
541
+ task = allocate
542
+ task.send(:create, *args, **opts)
543
+ register(task)
544
+ end
545
+
546
+
547
+ def self.restore(hash)
548
+ task = allocate
549
+ task.send(:restore, hash)
550
+ register(task)
551
+ end
552
+
553
+
554
+ def self.delete(*tags)
555
+ process = []
556
+ tags.each do |tag|
557
+ case (tasks = Task.lookup(tag)).size
558
+ when 0 then log.warn("no tasks matching (partial) tag #{tag}")
559
+ when 1 then process += tasks
560
+ else
561
+ tags = tasks.collect { |v| v.tag }.join(', ')
562
+ raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
563
+ end
564
+ end
565
+ process.each { |task| task.delete }
566
+ end
567
+
568
+
569
+ def initialize(tag: Bitferry.tag, modified: nil, include: [], exclude: [])
570
+ @tag = tag
571
+ @generation = 0
572
+ @include = include
573
+ @exclude = exclude
574
+ @modified = case modified
575
+ when nil then DateTime.now
576
+ when DateTime then modified
577
+ else DateTime.parse(modified)
578
+ end
579
+ # FIXME handle process_options at this level
580
+ end
581
+
582
+
583
+ def create(*args, **opts)
584
+ initialize(*args, **opts)
585
+ @state = :pristine
586
+ touch
587
+ end
588
+
589
+
590
+ def restore(hash)
591
+ @include = hash.fetch(:include, [])
592
+ @exclude = hash.fetch(:exclude, [])
593
+ @state = :intact
594
+ log.info("restored task #{tag}")
595
+ end
596
+
597
+
598
+ # FIXME move to Endpoint#restore
599
+ def restore_endpoint(x) = Endpoint::ROUTE.fetch(x.fetch(:endpoint).intern).restore(x)
600
+
601
+
602
+ def externalize
603
+ {
604
+ task: tag,
605
+ modified: modified,
606
+ include: include.empty? ? nil : include,
607
+ exclude: exclude.empty? ? nil : exclude
608
+ }.compact
609
+ end
610
+
611
+
612
+ def live? = !@state.nil? && @state != :removing
613
+
614
+
615
+ def touch = @modified = DateTime.now
616
+
617
+
618
+ def delete
619
+ touch
620
+ @state = :removing
621
+ log.info("marked task #{tag} for removal")
622
+ end
623
+
624
+
625
+ def commit
626
+ case @state
627
+ when :pristine then format
628
+ when :removing then @state = nil
629
+ end
630
+ end
631
+
632
+
633
+ def show_filters
634
+ xs = []
635
+ xs << 'include: ' + include.join(',') unless include.empty?
636
+ xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
637
+ xs.join(' ').to_s
638
+ end
639
+
640
+
641
+ def self.[](tag) = @@registry[tag]
642
+
643
+
644
+ # Return list of registered tasks whose tags match at least one of specified partial tags
645
+ def self.lookup(*tags) = match(tags, registered)
646
+
647
+
648
+ # Return list of specified tasks whose tags match at least one of specified partial tags
649
+ def self.match(tags, tasks)
650
+ rxs = tags.collect { |x| Regexp.new(x) }
651
+ tasks.filter do |task|
652
+ rxs.any? { |rx| !(rx =~ task.tag).nil? }
653
+ end
654
+ end
655
+
656
+
657
+ def self.registered = @@registry.values
658
+
659
+
660
+ def self.live = registered.filter { |task| task.live? }
661
+
662
+
663
+ def self.reset = @@registry = {}
664
+
665
+
666
+ def self.register(task)
667
+ # Task with newer timestamp replaces already registered task, if any
668
+ if (xtag = @@registry[task.tag]).nil?
669
+ @@registry[task.tag] = task
670
+ elsif xtag.modified < task.modified
671
+ @@registry[task.tag] = task
672
+ else
673
+ xtag
674
+ end
675
+ end
676
+
677
+ def self.intact = live.filter { |task| task.intact? }
678
+
679
+
680
+ def self.stale = live.filter { |task| !task.intact? }
681
+
682
+
683
+ end
684
+
685
+
686
+ module Rclone
687
+
688
+
689
+ include Logging
690
+ extend Logging
691
+
692
+
693
+ def self.executable = @executable ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
694
+
695
+
696
+ def self.exec(*args)
697
+ cmd = [executable] + args
698
+ log.debug(cmd.collect(&:shellescape).join(' '))
699
+ stdout, status = Open3.capture2e(*cmd)
700
+ unless status.success?
701
+ msg = "rclone exit code #{status.exitstatus}"
702
+ log.error(msg)
703
+ raise RuntimeError, msg
704
+ end
705
+ stdout.strip
706
+ end
707
+
708
+
709
+ # https://github.com/rclone/rclone/blob/master/fs/config/obscure/obscure.go
710
+ SECRET = "\x9c\x93\x5b\x48\x73\x0a\x55\x4d\x6b\xfd\x7c\x63\xc8\x86\xa9\x2b\xd3\x90\x19\x8e\xb8\x12\x8a\xfb\xf4\xde\x16\x2b\x8b\x95\xf6\x38"
711
+
712
+
713
+ def self.obscure(plain)
714
+ cipher = OpenSSL::Cipher.new('AES-256-CTR')
715
+ cipher.encrypt
716
+ cipher.key = SECRET
717
+ Base64.urlsafe_encode64(cipher.random_iv + cipher.update(plain) + cipher.final, padding: false)
718
+ end
719
+
720
+
721
+ def self.reveal(token)
722
+ data = Base64.urlsafe_decode64(token)
723
+ cipher = OpenSSL::Cipher.new('AES-256-CTR')
724
+ cipher.decrypt
725
+ cipher.key = SECRET
726
+ cipher.iv = data[0...cipher.iv_len]
727
+ cipher.update(data[cipher.iv_len..-1]) + cipher.final
728
+ end
729
+
730
+
731
+ class Encryption
732
+
733
+
734
+ PROCESS = {
735
+ default: ['--crypt-filename-encoding', :base32, '--crypt-filename-encryption', :standard],
736
+ extended: ['--crypt-filename-encoding', :base32768, '--crypt-filename-encryption', :standard]
737
+ }
738
+ PROCESS[nil] = PROCESS[:default]
739
+
740
+
741
+ def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
742
+
743
+
744
+ def initialize(token, process: nil)
745
+ @process_options = Bitferry.optional(process, PROCESS)
746
+ @token = token
747
+ end
748
+
749
+
750
+ def create(password, **opts) = initialize(Rclone.obscure(password), **opts)
751
+
752
+
753
+ def restore(hash) = @process_options = hash[:rclone]
754
+
755
+
756
+ def externalize = process_options.empty? ? {} : { rclone: process_options }
757
+
758
+
759
+ def configure(task) = install_token(task)
760
+
761
+
762
+ def process(task) = ENV['RCLONE_CRYPT_PASSWORD'] = obtain_token(task)
763
+
764
+
765
+ def arguments(task) = process_options + ['--crypt-remote', encrypted(task).root.to_s]
766
+
767
+
768
+ def install_token(task)
769
+ x = decrypted(task)
770
+ raise TypeError, 'unsupported unencrypted endpoint type' unless x.is_a?(Endpoint::Bitferry)
771
+ Volume[x.volume_tag].vault[task.tag] = @token # Token is stored on the decrypted end only
772
+ end
773
+
774
+
775
+ def obtain_token(task) = Volume[decrypted(task).volume_tag].vault.fetch(task.tag)
776
+
777
+
778
+ def self.new(*args, **opts)
779
+ obj = allocate
780
+ obj.send(:create, *args, **opts)
781
+ obj
782
+ end
783
+
784
+
785
+ def self.restore(hash)
786
+ obj = ROUTE.fetch(hash.fetch(:operation).intern).allocate
787
+ obj.send(:restore, hash)
788
+ obj
789
+ end
790
+
791
+
792
+ end
793
+
794
+
795
+ class Encrypt < Encryption
796
+
797
+
798
+ def encrypted(task) = task.destination
799
+
800
+
801
+ def decrypted(task) = task.source
802
+
803
+
804
+ def externalize = super.merge(operation: :encrypt)
805
+
806
+
807
+ def show_operation = 'encrypt+'
808
+
809
+
810
+ def arguments(task) = super + [decrypted(task).root.to_s, ':crypt:']
811
+
812
+
813
+ end
814
+
815
+
816
+ class Decrypt < Encryption
817
+
818
+
819
+ def encrypted(task) = task.source
820
+
821
+
822
+ def decrypted(task) = task.destination
823
+
824
+
825
+ def externalize = super.merge(operation: :decrypt)
826
+
827
+
828
+ def show_operation = 'decrypt+'
829
+
830
+
831
+ def arguments(task) = super + [':crypt:', decrypted(task).root.to_s]
832
+
833
+
834
+ end
835
+
836
+
837
+ ROUTE = {
838
+ encrypt: Encrypt,
839
+ decrypt: Decrypt
840
+ }
841
+
842
+
843
+ class Task < Bitferry::Task
844
+
845
+
846
+ attr_reader :source, :destination
847
+
848
+
849
+ attr_reader :encryption
850
+
851
+
852
+ attr_reader :token
853
+
854
+
855
+ PROCESS = {
856
+ default: ['--metadata']
857
+ }
858
+ PROCESS[nil] = PROCESS[:default]
859
+
860
+
861
+ def initialize(source, destination, encryption: nil, process: nil, **opts)
862
+ super(**opts)
863
+ @process_options = Bitferry.optional(process, PROCESS)
864
+ @source = source.is_a?(Endpoint) ? source : Bitferry.endpoint(source)
865
+ @destination = destination.is_a?(Endpoint) ? destination : Bitferry.endpoint(destination)
866
+ @encryption = encryption
867
+ end
868
+
869
+
870
+ def create(*args, process: nil, **opts)
871
+ super(*args, process: process, **opts)
872
+ encryption.configure(self) unless encryption.nil?
873
+ end
874
+
875
+
876
+ def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
877
+
878
+
879
+ def show_operation = encryption.nil? ? '' : encryption.show_operation
880
+
881
+
882
+ def show_direction = '-->'
883
+
884
+
885
+ def intact? = live? && source.intact? && destination.intact?
886
+
887
+
888
+ def refers?(volume) = source.refers?(volume) || destination.refers?(volume)
889
+
890
+
891
+ def touch
892
+ @generation = [source.generation, destination.generation].max + 1
893
+ super
894
+ end
895
+
896
+
897
+ def format = nil
898
+
899
+
900
+ def common_options
901
+ [
902
+ '--config', Bitferry.windows? ? 'NUL' : '/dev/null',
903
+ case Bitferry.verbosity
904
+ when :verbose then '--verbose'
905
+ when :quiet then '--quiet'
906
+ else nil
907
+ end,
908
+ Bitferry.verbosity == :verbose ? '--progress' : nil,
909
+ Bitferry.simulate? ? '--dry-run' : nil,
910
+ ].compact
911
+ end
912
+
913
+
914
+ def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
915
+
916
+
917
+ def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
918
+
919
+
920
+ def process_arguments
921
+ include_filters + exclude_filters + common_options + process_options + (
922
+ encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
923
+ )
924
+ end
925
+
926
+
927
+ def execute(*args)
928
+ cmd = [Rclone.executable] + args
929
+ cms = cmd.collect(&:shellescape).join(' ')
930
+ $stdout.puts cms if Bitferry.verbosity == :verbose
931
+ log.info(cms)
932
+ if Bitferry.ui == :gui
933
+ t = nil
934
+ Open3.popen2e(*cmd) do |i, oe, thr|
935
+ while x = oe.gets; $stdout.puts(x) end
936
+ t = thr
937
+ end
938
+ status = t.value
939
+ else
940
+ status = Open3.pipeline(cmd).first
941
+ end
942
+ raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
943
+ status.success?
944
+ end
945
+
946
+
947
+ def process
948
+ log.info("processing task #{tag}")
949
+ encryption.process(self) unless encryption.nil?
950
+ execute(*process_arguments)
951
+ end
952
+
953
+
954
+ def externalize
955
+ super.merge(
956
+ source: source.externalize,
957
+ destination: destination.externalize,
958
+ encryption: encryption.nil? ? nil : encryption.externalize,
959
+ rclone: process_options.empty? ? nil : process_options
960
+ ).compact
961
+ end
962
+
963
+
964
+ def restore(hash)
965
+ task = hash.fetch(:task)
966
+ begin
967
+ modified = hash.fetch(:modified)
968
+ rescue
969
+ modified = nil
970
+ log.warn("modified key missing - flagging task #{task} out of date")
971
+ end
972
+ initialize(
973
+ restore_endpoint(hash.fetch(:source)),
974
+ restore_endpoint(hash.fetch(:destination)),
975
+ tag: task,
976
+ modified: modified,
977
+ process: hash[:rclone],
978
+ encryption: hash[:encryption].nil? ? nil : Rclone::Encryption.restore(hash[:encryption])
979
+ )
980
+ super(hash)
981
+ touch if modified.nil?
982
+ end
983
+
984
+
985
+ end
986
+
987
+
988
+ class Copy < Task
989
+
990
+
991
+ def process_arguments = ['copy'] + super
992
+
993
+
994
+ def externalize = super.merge(operation: :copy)
995
+
996
+
997
+ def show_operation = super + 'copy'
998
+
999
+
1000
+ end
1001
+
1002
+
1003
+ class Update < Task
1004
+
1005
+
1006
+ def process_arguments = ['copy', '--update'] + super
1007
+
1008
+
1009
+ def externalize = super.merge(operation: :update)
1010
+
1011
+
1012
+ def show_operation = super + 'update'
1013
+
1014
+
1015
+ end
1016
+
1017
+
1018
+ class Synchronize < Task
1019
+
1020
+
1021
+ def process_arguments = ['sync'] + super
1022
+
1023
+
1024
+ def externalize = super.merge(operation: :synchronize)
1025
+
1026
+
1027
+ def show_operation = super + 'synchronize'
1028
+
1029
+
1030
+ end
1031
+
1032
+
1033
+ class Equalize < Task
1034
+
1035
+
1036
+ def process_arguments = ['bisync', '--resync'] + super
1037
+
1038
+
1039
+ def externalize = super.merge(operation: :equalize)
1040
+
1041
+
1042
+ def show_operation = super + 'equalize'
1043
+
1044
+
1045
+ def show_direction = '<->'
1046
+
1047
+
1048
+ end
1049
+
1050
+
1051
+ end
1052
+
1053
+
1054
+ module Restic
1055
+
1056
+
1057
+ include Logging
1058
+ extend Logging
1059
+
1060
+
1061
+ def self.executable = @executable ||= (restic = ENV['RESTIC']).nil? ? 'restic' : restic
1062
+
1063
+
1064
+ def self.exec(*args)
1065
+ cmd = [executable] + args
1066
+ log.debug(cmd.collect(&:shellescape).join(' '))
1067
+ stdout, status = Open3.capture2(*cmd)
1068
+ unless status.success?
1069
+ msg = "restic exit code #{status.to_i}"
1070
+ log.error(msg)
1071
+ raise RuntimeError, msg
1072
+ end
1073
+ stdout.strip
1074
+ end
1075
+
1076
+
1077
+ class Task < Bitferry::Task
1078
+
1079
+
1080
+ attr_reader :directory, :repository
1081
+
1082
+
1083
+ def initialize(directory, repository, **opts)
1084
+ super(**opts)
1085
+ @directory = directory.is_a?(Endpoint) ? directory : Bitferry.endpoint(directory)
1086
+ @repository = repository.is_a?(Endpoint) ? repository : Bitferry.endpoint(repository)
1087
+ end
1088
+
1089
+
1090
+ def create(directory, repository, password, **opts)
1091
+ super(directory, repository, **opts)
1092
+ raise TypeError, 'unsupported unencrypted endpoint type' unless self.directory.is_a?(Endpoint::Bitferry)
1093
+ Volume[self.directory.volume_tag].vault[tag] = Rclone.obscure(@password = password) # Token is stored on the decrypted end only
1094
+ end
1095
+
1096
+
1097
+ def password = @password ||= Rclone.reveal(Volume[directory.volume_tag].vault.fetch(tag))
1098
+
1099
+
1100
+ def intact? = live? && directory.intact? && repository.intact?
1101
+
1102
+
1103
+ def refers?(volume) = directory.refers?(volume) || repository.refers?(volume)
1104
+
1105
+
1106
+ def touch
1107
+ @generation = [directory.generation, repository.generation].max + 1
1108
+ super
1109
+ end
1110
+
1111
+
1112
+ def format = nil
1113
+
1114
+
1115
+ def include_filters = include.collect { |x| ['--include', x]}.flatten
1116
+
1117
+
1118
+ def common_options
1119
+ [
1120
+ case Bitferry.verbosity
1121
+ when :verbose then '--verbose'
1122
+ when :quiet then '--quiet'
1123
+ else nil
1124
+ end,
1125
+ '-r', repository.root.to_s
1126
+ ].compact
1127
+ end
1128
+
1129
+
1130
+ def execute(*args, simulate: false, chdir: nil)
1131
+ cmd = [Restic.executable] + args
1132
+ ENV['RESTIC_PASSWORD'] = password
1133
+ ENV['RESTIC_PROGRESS_FPS'] = 1.to_s if Bitferry.verbosity == :verbose && Bitferry.ui == :gui
1134
+ cms = cmd.collect(&:shellescape).join(' ')
1135
+ $stdout.puts cms if Bitferry.verbosity == :verbose
1136
+ log.info(cms)
1137
+ if simulate
1138
+ log.info('(simulated)')
1139
+ true
1140
+ else
1141
+ wd = Dir.getwd unless chdir.nil?
1142
+ begin
1143
+ Dir.chdir(chdir) unless chdir.nil?
1144
+ if Bitferry.ui == :gui
1145
+ t = nil
1146
+ Open3.popen2e(*cmd) do |i, oe, thr|
1147
+ while x = oe.gets; $stdout.puts(x) end
1148
+ t = thr
1149
+ end
1150
+ status = t.value
1151
+ else
1152
+ status = Open3.pipeline(cmd).first
1153
+ end
1154
+ raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
1155
+ status.success?
1156
+ ensure
1157
+ Dir.chdir(wd) unless chdir.nil?
1158
+ end
1159
+ end
1160
+ end
1161
+
1162
+
1163
+ def externalize
1164
+ super.merge(
1165
+ directory: directory.externalize,
1166
+ repository: repository.externalize,
1167
+ ).compact
1168
+ end
1169
+
1170
+
1171
+ def restore(hash)
1172
+ initialize(
1173
+ restore_endpoint(hash.fetch(:directory)),
1174
+ restore_endpoint(hash.fetch(:repository)),
1175
+ tag: hash.fetch(:task),
1176
+ modified: hash.fetch(:modified, DateTime.now)
1177
+ )
1178
+ super(hash)
1179
+ end
1180
+
1181
+
1182
+ end
1183
+
1184
+
1185
+ class Backup < Task
1186
+
1187
+
1188
+ PROCESS = {
1189
+ default: ['--no-cache']
1190
+ }
1191
+ PROCESS[nil] = PROCESS[:default]
1192
+
1193
+
1194
+ FORGET = {
1195
+ default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
1196
+ }
1197
+ FORGET[nil] = nil # Skip processing retention policy by default
1198
+
1199
+
1200
+ CHECK = {
1201
+ default: ['--no-cache'],
1202
+ full: ['--no-cache', '--read-data']
1203
+ }
1204
+ CHECK[nil] = nil # Skip integrity checking by default
1205
+
1206
+
1207
+ attr_reader :forget_options
1208
+ attr_reader :check_options
1209
+
1210
+
1211
+ def create(*args, format: nil, process: nil, forget: nil, check: nil, **opts)
1212
+ super(*args, **opts)
1213
+ @format = format
1214
+ @process_options = Bitferry.optional(process, PROCESS)
1215
+ @forget_options = Bitferry.optional(forget, FORGET)
1216
+ @check_options = Bitferry.optional(check, CHECK)
1217
+ end
1218
+
1219
+
1220
+ def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
1221
+
1222
+
1223
+ def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
1224
+
1225
+
1226
+ def show_operation = 'encrypt+backup'
1227
+
1228
+
1229
+ def show_direction = '-->'
1230
+
1231
+
1232
+ alias :source :directory
1233
+ alias :destination :repository
1234
+
1235
+
1236
+ def process
1237
+ begin
1238
+ log.info("processing task #{tag}")
1239
+ execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
1240
+ unless check_options.nil?
1241
+ log.info("checking repository in #{repository.root}")
1242
+ execute('check', *check_options, *common_options)
1243
+ end
1244
+ unless forget_options.nil?
1245
+ log.info("performing repository maintenance tasks in #{repository.root}")
1246
+ execute('forget', '--tag', "bitferry,#{tag}", *forget_options.collect(&:to_s), *common_options_simulate)
1247
+ end
1248
+ true
1249
+ rescue
1250
+ false
1251
+ end
1252
+ end
1253
+
1254
+
1255
+ def common_options_simulate = common_options + [Bitferry.simulate? ? '--dry-run' : nil].compact
1256
+
1257
+
1258
+ def externalize
1259
+ restic = {
1260
+ process: process_options,
1261
+ forget: forget_options,
1262
+ check: check_options
1263
+ }.compact
1264
+ super.merge({
1265
+ operation: :backup,
1266
+ restic: restic.empty? ? nil : restic
1267
+ }.compact)
1268
+ end
1269
+
1270
+
1271
+ def restore(hash)
1272
+ super
1273
+ opts = hash.fetch(:restic, {})
1274
+ @process_options = opts[:process]
1275
+ @forget_options = opts[:forget]
1276
+ @check_options = opts[:check]
1277
+ end
1278
+
1279
+
1280
+ def format
1281
+ if Bitferry.simulate?
1282
+ log.info('skipped repository initialization (simulation)')
1283
+ else
1284
+ log.info("initializing repository for task #{tag}")
1285
+ if @format == true
1286
+ log.debug("wiping repository in #{repository.root}")
1287
+ ['config', 'data', 'index', 'keys', 'locks', 'snapshots'].each { |x| FileUtils.rm_rf(File.join(repository.root.to_s, x)) }
1288
+ end
1289
+ if @format == false
1290
+ # TODO validate existing repo
1291
+ log.info("attached to existing repository for task #{tag} in #{repository.root}")
1292
+ else
1293
+ begin
1294
+ execute(*common_options, 'init')
1295
+ log.info("initialized repository for task #{tag} in #{repository.root}")
1296
+ rescue
1297
+ log.fatal("failed to initialize repository for task #{tag} in #{repository.root}")
1298
+ raise
1299
+ end
1300
+ end
1301
+ end
1302
+ @state = :intact
1303
+ end
1304
+
1305
+
1306
+ end
1307
+
1308
+
1309
+ class Restore < Task
1310
+
1311
+
1312
+ PROCESS = {
1313
+ default: ['--no-cache', '--sparse']
1314
+ }
1315
+ PROCESS[nil] = PROCESS[:default]
1316
+
1317
+
1318
+ def create(*args, process: nil, **opts)
1319
+ super(*args, **opts)
1320
+ @process_options = Bitferry.optional(process, PROCESS)
1321
+ end
1322
+
1323
+
1324
+ def exclude_filters = exclude.collect { |x| ['--exclude', x] }.flatten
1325
+
1326
+
1327
+ def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
1328
+
1329
+
1330
+ def show_operation = 'decrypt+restore'
1331
+
1332
+
1333
+ def show_direction = '-->'
1334
+
1335
+
1336
+ alias :destination :directory
1337
+ alias :source :repository
1338
+
1339
+
1340
+ def externalize
1341
+ restic = {
1342
+ process: process_options
1343
+ }.compact
1344
+ super.merge({
1345
+ operation: :restore,
1346
+ restic: restic.empty? ? nil : restic
1347
+ }.compact)
1348
+ end
1349
+
1350
+
1351
+ def restore(hash)
1352
+ super
1353
+ opts = hash.fetch(:rclone, {})
1354
+ @process_options = opts[:process]
1355
+ end
1356
+
1357
+
1358
+ def process
1359
+ log.info("processing task #{tag}")
1360
+ begin
1361
+ # FIXME restore specifically tagged latest snapshot
1362
+ execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
1363
+ true
1364
+ rescue
1365
+ false
1366
+ end
1367
+ end
1368
+
1369
+
1370
+ end
1371
+
1372
+
1373
+
1374
+ end
1375
+
1376
+
1377
+ Task::ROUTE = {
1378
+ copy: Rclone::Copy,
1379
+ update: Rclone::Update,
1380
+ synchronize: Rclone::Synchronize,
1381
+ equalize: Rclone::Equalize,
1382
+ backup: Restic::Backup,
1383
+ restore: Restic::Restore
1384
+ }
1385
+
1386
+
1387
+ class Endpoint
1388
+
1389
+
1390
+ def self.restore(hash)
1391
+ endpoint = allocate
1392
+ endpoint.send(:restore, hash)
1393
+ endpoint
1394
+ end
1395
+
1396
+
1397
+ class Local < Endpoint
1398
+
1399
+
1400
+ attr_reader :root
1401
+
1402
+
1403
+ def initialize(root) = @root = Pathname.new(root).realdirpath
1404
+
1405
+
1406
+ def restore(hash) = initialize(hash.fetch(:root))
1407
+
1408
+
1409
+ def externalize
1410
+ {
1411
+ endpoint: :local,
1412
+ root: root
1413
+ }
1414
+ end
1415
+
1416
+
1417
+ def show_status = root.to_s
1418
+
1419
+
1420
+ def intact? = true
1421
+
1422
+
1423
+ def refers?(volume) = false
1424
+
1425
+
1426
+ def generation = 0
1427
+
1428
+
1429
+ end
1430
+
1431
+
1432
+ class Rclone < Endpoint
1433
+ # TODO
1434
+ end
1435
+
1436
+
1437
+ class Bitferry < Endpoint
1438
+
1439
+
1440
+ attr_reader :volume_tag
1441
+
1442
+
1443
+ attr_reader :path
1444
+
1445
+
1446
+ def root = Volume[volume_tag].root.join(path)
1447
+
1448
+
1449
+ def initialize(volume, path)
1450
+ @volume_tag = volume.tag
1451
+ @path = Pathname.new(path)
1452
+ raise ArgumentError, "expected relative path but got #{self.path}" if (Volume::ENCOMPASSING_PATH_PREFIX =~ self.path.to_s).nil?
1453
+ end
1454
+
1455
+
1456
+ def restore(hash)
1457
+ @volume_tag = hash.fetch(:volume)
1458
+ @path = Pathname.new(hash.fetch(:path, ''))
1459
+ end
1460
+
1461
+
1462
+ def externalize
1463
+ {
1464
+ endpoint: :bitferry,
1465
+ volume: volume_tag,
1466
+ path: path.to_s.empty? ? nil : path
1467
+ }.compact
1468
+ end
1469
+
1470
+
1471
+ def show_status = intact? ? ":#{volume_tag}:#{path}" : ":{#{volume_tag}}:#{path}"
1472
+
1473
+
1474
+ def intact? = !Volume[volume_tag].nil?
1475
+
1476
+
1477
+ def refers?(volume) = volume.tag == volume_tag
1478
+
1479
+
1480
+ def generation
1481
+ v = Volume[volume_tag]
1482
+ v ? v.generation : 0
1483
+ end
1484
+
1485
+
1486
+ end
1487
+
1488
+
1489
+ ROUTE = {
1490
+ local: Local,
1491
+ rclone: Rclone,
1492
+ bitferry: Bitferry
1493
+ }
1494
+
1495
+
1496
+ end
1497
+
1498
+
1499
+ reset
1500
+
1501
+
1502
+ end