bitferry 0.0.7 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/bitferry.rb CHANGED
@@ -1,1475 +1,1502 @@
1
- require 'json'
2
- require 'date'
3
- require 'open3'
4
- require 'base64'
5
- require 'logger'
6
- require 'openssl'
7
- require 'pathname'
8
- require 'rbconfig'
9
- require 'fileutils'
10
- require 'shellwords'
11
-
12
-
13
- module Bitferry
14
-
15
-
16
- VERSION = '0.0.7'
17
-
18
-
19
- module Logging
20
- def self.log
21
- unless @log
22
- @log = Logger.new($stderr)
23
- @log.level = Logger::WARN
24
- @log.progname = :bitferry
25
- end
26
- @log
27
- end
28
- def self.log=(log) @log = log end
29
- def log = Logging.log
30
- end
31
-
32
-
33
- include Logging
34
- extend Logging
35
-
36
-
37
- def self.tag = format('%08x', 2**32*rand)
38
-
39
-
40
- def self.restore
41
- reset
42
- log.info('restoring volumes')
43
- result = true
44
- roots = (environment_mounts + system_mounts + [Dir.home]).uniq
45
- log.info("distilled volume search path: #{roots.join(', ')}")
46
- roots.each do |root|
47
- if File.exist?(File.join(root, Volume::STORAGE))
48
- log.info("trying to restore volume from #{root}")
49
- Volume.restore(root) rescue result = false
50
- end
51
- end
52
- if result
53
- log.info('volumes restored')
54
- else
55
- log.warn('volume restore failure(s) reported')
56
- end
57
- result
58
- end
59
-
60
-
61
- def self.commit
62
- log.info('committing changes')
63
- result = true
64
- modified = false
65
- Volume.registered.each do |volume|
66
- begin
67
- modified = true if volume.modified?
68
- volume.commit
69
- rescue IOError => e
70
- log.error(e.message)
71
- result = false
72
- end
73
- end
74
- if result
75
- log.info(modified ? 'changes committed' : 'commits skipped (no changes)')
76
- else
77
- log.warn('commit failure(s) reported')
78
- end
79
- result
80
- end
81
-
82
-
83
- def self.reset
84
- log.info('resetting state')
85
- Volume.reset
86
- Task.reset
87
- end
88
-
89
-
90
- def self.intact_tasks = Volume.intact.collect { |volume| volume.intact_tasks }.flatten.uniq
91
-
92
- def self.process(*tags, &block)
93
- log.info('processing tasks')
94
- tasks = intact_tasks
95
- if tags.empty?
96
- process = tasks
97
- else
98
- process = []
99
- tags.each do |tag|
100
- case (tasks = Task.match([tag], tasks)).size
101
- when 0 then log.warn("no tasks matching (partial) tag #{tag}")
102
- when 1 then process += tasks
103
- else
104
- tags = tasks.collect { |v| v.tag }.join(', ')
105
- raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
106
- end
107
- end
108
- end
109
- tasks = process.uniq
110
- total = tasks.size
111
- processed = 0
112
- failed = 0
113
- result = tasks.all? do |task|
114
- r = task.process
115
- processed += 1
116
- failed += 1 unless r
117
- yield(total, processed, failed) if block_given?
118
- r
119
- end
120
- result ? log.info('tasks processed') : log.warn('task process failure(s) reported')
121
- result
122
- end
123
-
124
-
125
- def self.endpoint(root)
126
- case root
127
- when /^:(\w+):(.*)/
128
- volumes = Volume.lookup($1)
129
- volume = case volumes.size
130
- when 0 then raise ArgumentError, "no intact volume matching (partial) tag #{$1}"
131
- when 1 then volumes.first
132
- else
133
- tags = volumes.collect { |v| v.tag }.join(', ')
134
- raise ArgumentError, "multiple intact volumes matching (partial) tag #{$1}: #{tags}"
135
- end
136
- Endpoint::Bitferry.new(volume, $2)
137
- when /^(?:local)?:(.*)/ then Endpoint::Local.new($1)
138
- when /^(\w{2,}):(.*)/ then Endpoint::Rclone.new($1, $2)
139
- else Volume.endpoint(root)
140
- end
141
- end
142
-
143
-
144
- @simulate = false
145
- def self.simulate? = @simulate
146
- def self.simulate=(mode) @simulate = mode end
147
-
148
-
149
- @verbosity = :default
150
- def self.verbosity = @verbosity
151
- def self.verbosity=(mode) @verbosity = mode end
152
-
153
-
154
- @ui = :cli
155
- def self.ui = @ui
156
- def self.ui=(ui) @ui = ui end
157
-
158
-
159
- # Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
160
- def self.windows?
161
- @windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
162
- end
163
-
164
- # Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
165
- # Look for the $BITFERRY_PATH environment variable
166
- def self.environment_mounts
167
- ENV['BITFERRY_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
168
- end
169
-
170
-
171
- # Specify OS-specific path name list separator (such as in the $PATH environment variable)
172
- PATH_LIST_SEPARATOR = windows? ? ';' : ':'
173
-
174
-
175
- # Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Bitferry voulmes
176
- UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc|efi)!
177
-
178
-
179
- # Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
180
- if RUBY_PLATFORM =~ /java/
181
- require 'java'
182
- def self.system_mounts
183
- java.nio.file.FileSystems.getDefault.getFileStores.collect {|x| /^(.*)\s+\(.*\)$/.match(x.to_s)[1]}
184
- end
185
- else
186
- case RbConfig::CONFIG['target_os']
187
- when 'linux'
188
- # Linux OS
189
- def self.system_mounts
190
- # Query /proc for currently mounted file systems
191
- IO.readlines('/proc/mounts').collect do |line|
192
- mount = line.split[1]
193
- UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
194
- end.compact
195
- end
196
- # TODO handle Windows variants
197
- when /^mingw/ # RubyInstaller's MRI
198
- module Kernel32
199
- require 'fiddle'
200
- require 'fiddle/types'
201
- require 'fiddle/import'
202
- extend Fiddle::Importer
203
- dlload('kernel32')
204
- include Fiddle::Win32Types
205
- extern 'DWORD WINAPI GetLogicalDrives()'
206
- end
207
- def self.system_mounts
208
- mounts = []
209
- mask = Kernel32.GetLogicalDrives
210
- ('A'..'Z').each do |x|
211
- mounts << "#{x}:/" if mask & 1 == 1
212
- mask >>= 1
213
- end
214
- mounts
215
- end
216
- else
217
- # Generic *NIX-like OS, including Cygwin & MSYS2
218
- def self.system_mounts
219
- # Use $(mount) system utility to obtain currently mounted file systems
220
- %x(mount).split("\n").collect do |line|
221
- mount = line.split[2]
222
- UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
223
- end.compact
224
- end
225
- end
226
- end
227
-
228
-
229
- class Volume
230
-
231
-
232
- include Logging
233
- extend Logging
234
-
235
-
236
- STORAGE = '.bitferry'
237
- STORAGE_ = '.bitferry~'
238
- STORAGE_MASK = '.bitferry*'
239
-
240
-
241
- attr_reader :tag
242
-
243
-
244
- attr_reader :generation
245
-
246
-
247
- attr_reader :root
248
-
249
-
250
- attr_reader :vault
251
-
252
-
253
- def self.[](tag)
254
- @@registry.each_value { |volume| return volume if volume.tag == tag }
255
- nil
256
- end
257
-
258
-
259
- # Return list of registered volumes whose tags match at least one specified partial
260
- def self.lookup(*tags) = match(tags, registered)
261
-
262
-
263
- def self.match(tags, volumes)
264
- rxs = tags.collect { |x| Regexp.new(x) }
265
- volumes.filter do |volume|
266
- rxs.any? { |rx| !(rx =~ volume.tag).nil? }
267
- end
268
- end
269
-
270
-
271
- def self.new(root, **opts)
272
- volume = allocate
273
- volume.send(:create, root, **opts)
274
- register(volume)
275
- end
276
-
277
-
278
- def self.restore(root)
279
- begin
280
- volume = allocate
281
- volume.send(:restore, root)
282
- volume = register(volume)
283
- log.info("restored volume #{volume.tag} from #{root}")
284
- volume
285
- rescue => e
286
- log.error("failed to restore volume from #{root}")
287
- log.error(e.message) if $DEBUG
288
- raise
289
- end
290
- end
291
-
292
-
293
- def self.delete(*tags, wipe: false)
294
- process = []
295
- tags.each do |tag|
296
- case (volumes = Volume.lookup(tag)).size
297
- when 0 then log.warn("no volumes matching (partial) tag #{tag}")
298
- when 1 then process += volumes
299
- else
300
- tags = volumes.collect { |v| v.tag }.join(', ')
301
- raise ArgumentError, "multiple volumes matching (partial) tag #{tag}: #{tags}"
302
- end
303
- end
304
- process.each { |volume| volume.delete(wipe: wipe) }
305
- end
306
-
307
-
308
- def initialize(root, tag: Bitferry.tag, modified: DateTime.now, overwrite: false)
309
- @tag = tag
310
- @generation = 0
311
- @vault = {}
312
- @modified = modified
313
- @overwrite = overwrite
314
- @root = Pathname.new(root).realdirpath
315
- end
316
-
317
-
318
- def create(*args, **opts)
319
- initialize(*args, **opts)
320
- @state = :pristine
321
- @modified = true
322
- end
323
-
324
-
325
- def restore(root)
326
- hash = JSON.load_file(storage = Pathname(root).join(STORAGE), { symbolize_names: true })
327
- raise IOError, "bad volume storage #{storage}" unless hash.fetch(:bitferry) == "0"
328
- initialize(root, tag: hash.fetch(:volume), modified: DateTime.parse(hash.fetch(:modified)))
329
- hash.fetch(:tasks, []).each { |hash| Task::ROUTE.fetch(hash.fetch(:operation).intern).restore(hash) }
330
- @vault = hash.fetch(:vault, {}).transform_keys { |key| key.to_s }
331
- @state = :intact
332
- @modified = false
333
- end
334
-
335
-
336
- def storage = @storage ||= root.join(STORAGE)
337
- def storage_ = @storage_ ||= root.join(STORAGE_)
338
-
339
-
340
- def commit
341
- if modified?
342
- log.info("commit volume #{tag} (modified)")
343
- case @state
344
- when :pristine
345
- format
346
- store
347
- when :intact
348
- store
349
- when :removing
350
- remove
351
- else
352
- raise
353
- end
354
- committed
355
- else
356
- log.info("skipped committing volume #{tag} (unmodified)")
357
- end
358
- end
359
-
360
-
361
- def self.endpoint(root)
362
- path = Pathname.new(root).realdirpath
363
- intact.sort { |v1, v2| v2.root.to_s.size <=> v1.root.to_s.size }.each do |volume|
364
- begin
365
- stem = path.relative_path_from(volume.root).to_s #.chomp('/')
366
- case stem
367
- when '.' then return volume.endpoint
368
- when /^[^\.].*/ then return volume.endpoint(stem)
369
- end
370
- rescue ArgumentError
371
- # Catch different prefix error on Windows
372
- end
373
- end
374
- raise ArgumentError, "no intact volume encompasses path #{root}"
375
- end
376
-
377
-
378
- def endpoint(path = String.new) = Endpoint::Bitferry.new(self, path)
379
-
380
-
381
- def modified? = @modified || tasks.any? { |t| t.generation > generation }
382
-
383
-
384
- def intact? = @state != :removing
385
-
386
-
387
- def touch
388
- x = tasks.collect { |t| t.generation }.max
389
- @generation = x ? x + 1 : 0
390
- @modified = true
391
- end
392
-
393
-
394
- def delete(wipe: false)
395
- touch
396
- @wipe = wipe
397
- @state = :removing
398
- log.info("marked volume #{tag} for deletion")
399
- end
400
-
401
-
402
- def committed
403
- x = tasks.collect { |t| t.generation }.min
404
- @generation = x ? x : 0
405
- @modified = false
406
- end
407
-
408
-
409
- def store
410
- require 'neatjson'
411
- tasks.each(&:commit)
412
- hash = JSON.neat_generate(externalize, short: false, wrap: 200, afterColon: 1, afterComma: 1)
413
- if Bitferry.simulate?
414
- log.info("skipped volume #{tag} storage modification (simulation)")
415
- else
416
- begin
417
- File.write(storage_, hash)
418
- FileUtils.mv(storage_, storage)
419
- log.info("written volume #{tag} storage #{storage}")
420
- ensure
421
- FileUtils.rm_f(storage_)
422
- end
423
- end
424
- @state = :intact
425
- end
426
-
427
-
428
- def format
429
- raise IOError, "refuse to overwrite existing volume storage #{storage}" if !@overwrite && File.exist?(storage)
430
- if Bitferry.simulate?
431
- log.info("skipped storage formatting (simulation)")
432
- else
433
- FileUtils.mkdir_p(root)
434
- FileUtils.rm_f [storage, storage_]
435
- log.info("formatted volume #{tag} in #{root}")
436
- end
437
- @state = nil
438
- end
439
-
440
-
441
- def remove
442
- unless Bitferry.simulate?
443
- if @wipe
444
- FileUtils.rm_rf(Dir[File.join(root, '*'), File.join(root, '.*')])
445
- log.info("wiped entire volume directory #{root}")
446
- else
447
- FileUtils.rm_f [storage, storage_]
448
- log.info("deleted volume #{tag} storage files #{File.join(root, STORAGE_MASK)}")
449
- end
450
- end
451
- @@registry.delete(root)
452
- @state = nil
453
- end
454
-
455
-
456
- def externalize
457
- tasks = live_tasks
458
- v = vault.filter { |t| !Task[t].nil? && Task[t].live? } # Purge entries from non-existing (deleted) tasks
459
- {
460
- bitferry: "0",
461
- volume: tag,
462
- modified: (@modified = DateTime.now),
463
- tasks: tasks.empty? ? nil : tasks.collect(&:externalize),
464
- vault: v.empty? ? nil : v
465
- }.compact
466
- end
467
-
468
-
469
- def tasks = Task.registered.filter { |task| task.refers?(self) }
470
-
471
-
472
- def live_tasks = Task.live.filter { |task| task.refers?(self) }
473
-
474
-
475
- def intact_tasks = live_tasks.filter { |task| task.intact? }
476
-
477
-
478
- def self.reset = @@registry = {}
479
-
480
-
481
- def self.register(volume) = @@registry[volume.root] = volume
482
-
483
-
484
- def self.registered = @@registry.values
485
-
486
-
487
- def self.intact = registered.filter { |volume| volume.intact? }
488
-
489
-
490
- end
491
-
492
-
493
- def self.optional(option, route)
494
- case option
495
- when Array then option # Array is passed verbatim
496
- when '-' then nil # Disable adding any options with -
497
- when /^-/ then option.split(',') # Split comma-separated string into array --foo,bar --> [--foo, bar]
498
- else route.fetch(option.nil? ? nil : option.to_sym) # Obtain options from the profile database
499
- end
500
- end
501
-
502
-
503
- class Task
504
-
505
-
506
- include Logging
507
- extend Logging
508
-
509
-
510
- attr_reader :tag
511
-
512
-
513
- attr_reader :generation
514
-
515
-
516
- attr_reader :modified
517
-
518
-
519
- attr_reader :include, :exclude
520
-
521
-
522
- def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
523
-
524
-
525
- def self.new(*args, **opts)
526
- task = allocate
527
- task.send(:create, *args, **opts)
528
- register(task)
529
- end
530
-
531
-
532
- def self.restore(hash)
533
- task = allocate
534
- task.send(:restore, hash)
535
- register(task)
536
- end
537
-
538
-
539
- def self.delete(*tags)
540
- process = []
541
- tags.each do |tag|
542
- case (tasks = Task.lookup(tag)).size
543
- when 0 then log.warn("no tasks matching (partial) tag #{tag}")
544
- when 1 then process += tasks
545
- else
546
- tags = tasks.collect { |v| v.tag }.join(', ')
547
- raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
548
- end
549
- end
550
- process.each { |task| task.delete }
551
- end
552
-
553
-
554
- def initialize(tag: Bitferry.tag, modified: DateTime.now, include: [], exclude: [])
555
- @tag = tag
556
- @generation = 0
557
- @include = include
558
- @exclude = exclude
559
- @modified = modified.is_a?(DateTime) ? modified : DateTime.parse(modified)
560
- # FIXME handle process_options at this level
561
- end
562
-
563
-
564
- def create(*args, **opts)
565
- initialize(*args, **opts)
566
- @state = :pristine
567
- touch
568
- end
569
-
570
-
571
- def restore(hash)
572
- @include = hash.fetch(:include, [])
573
- @exclude = hash.fetch(:exclude, [])
574
- @state = :intact
575
- log.info("restored task #{tag}")
576
- end
577
-
578
-
579
- # FIXME move to Endpoint#restore
580
- def restore_endpoint(x) = Endpoint::ROUTE.fetch(x.fetch(:endpoint).intern).restore(x)
581
-
582
-
583
- def externalize
584
- {
585
- task: tag,
586
- modified: modified,
587
- include: include.empty? ? nil : include,
588
- exclude: exclude.empty? ? nil : exclude
589
- }.compact
590
- end
591
-
592
-
593
- def live? = !@state.nil? && @state != :removing
594
-
595
-
596
- def touch = @modified = DateTime.now
597
-
598
-
599
- def delete
600
- touch
601
- @state = :removing
602
- log.info("marked task #{tag} for removal")
603
- end
604
-
605
-
606
- def commit
607
- case @state
608
- when :pristine then format
609
- when :removing then @state = nil
610
- end
611
- end
612
-
613
-
614
- def show_filters
615
- xs = []
616
- xs << 'include: ' + include.join(',') unless include.empty?
617
- xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
618
- xs.join(' ').to_s
619
- end
620
-
621
-
622
- def self.[](tag) = @@registry[tag]
623
-
624
-
625
- # Return list of registered tasks whose tags match at least one of specified partial tags
626
- def self.lookup(*tags) = match(tags, registered)
627
-
628
-
629
- # Return list of specified tasks whose tags match at least one of specified partial tags
630
- def self.match(tags, tasks)
631
- rxs = tags.collect { |x| Regexp.new(x) }
632
- tasks.filter do |task|
633
- rxs.any? { |rx| !(rx =~ task.tag).nil? }
634
- end
635
- end
636
-
637
-
638
- def self.registered = @@registry.values
639
-
640
-
641
- def self.live = registered.filter { |task| task.live? }
642
-
643
-
644
- def self.reset = @@registry = {}
645
-
646
-
647
- def self.register(task)
648
- # Task with newer timestamp replaces already registered task, if any
649
- if (xtag = @@registry[task.tag]).nil?
650
- @@registry[task.tag] = task
651
- elsif xtag.modified < task.modified
652
- @@registry[task.tag] = task
653
- else
654
- xtag
655
- end
656
- end
657
-
658
- def self.intact = live.filter { |task| task.intact? }
659
-
660
-
661
- def self.stale = live.filter { |task| !task.intact? }
662
-
663
-
664
- end
665
-
666
-
667
- module Rclone
668
-
669
-
670
- include Logging
671
- extend Logging
672
-
673
-
674
- def self.executable = @executable ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
675
-
676
-
677
- def self.exec(*args)
678
- cmd = [executable] + args
679
- log.debug(cmd.collect(&:shellescape).join(' '))
680
- stdout, status = Open3.capture2e(*cmd)
681
- unless status.success?
682
- msg = "rclone exit code #{status.exitstatus}"
683
- log.error(msg)
684
- raise RuntimeError, msg
685
- end
686
- stdout.strip
687
- end
688
-
689
-
690
- # https://github.com/rclone/rclone/blob/master/fs/config/obscure/obscure.go
691
- SECRET = "\x9c\x93\x5b\x48\x73\x0a\x55\x4d\x6b\xfd\x7c\x63\xc8\x86\xa9\x2b\xd3\x90\x19\x8e\xb8\x12\x8a\xfb\xf4\xde\x16\x2b\x8b\x95\xf6\x38"
692
-
693
-
694
- def self.obscure(plain)
695
- cipher = OpenSSL::Cipher.new('AES-256-CTR')
696
- cipher.encrypt
697
- cipher.key = SECRET
698
- Base64.urlsafe_encode64(cipher.random_iv + cipher.update(plain) + cipher.final, padding: false)
699
- end
700
-
701
-
702
- def self.reveal(token)
703
- data = Base64.urlsafe_decode64(token)
704
- cipher = OpenSSL::Cipher.new('AES-256-CTR')
705
- cipher.decrypt
706
- cipher.key = SECRET
707
- cipher.iv = data[0...cipher.iv_len]
708
- cipher.update(data[cipher.iv_len..-1]) + cipher.final
709
- end
710
-
711
-
712
- class Encryption
713
-
714
-
715
- PROCESS = {
716
- default: ['--crypt-filename-encoding', :base32, '--crypt-filename-encryption', :standard],
717
- extended: ['--crypt-filename-encoding', :base32768, '--crypt-filename-encryption', :standard]
718
- }
719
- PROCESS[nil] = PROCESS[:default]
720
-
721
-
722
- def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
723
-
724
-
725
- def initialize(token, process: nil)
726
- @process_options = Bitferry.optional(process, PROCESS)
727
- @token = token
728
- end
729
-
730
-
731
- def create(password, **opts) = initialize(Rclone.obscure(password), **opts)
732
-
733
-
734
- def restore(hash) = @process_options = hash[:rclone]
735
-
736
-
737
- def externalize = process_options.empty? ? {} : { rclone: process_options }
738
-
739
-
740
- def configure(task) = install_token(task)
741
-
742
-
743
- def process(task) = ENV['RCLONE_CRYPT_PASSWORD'] = obtain_token(task)
744
-
745
-
746
- def arguments(task) = process_options + ['--crypt-remote', encrypted(task).root.to_s]
747
-
748
-
749
- def install_token(task)
750
- x = decrypted(task)
751
- raise TypeError, 'unsupported unencrypted endpoint type' unless x.is_a?(Endpoint::Bitferry)
752
- Volume[x.volume_tag].vault[task.tag] = @token # Token is stored on the decrypted end only
753
- end
754
-
755
-
756
- def obtain_token(task) = Volume[decrypted(task).volume_tag].vault.fetch(task.tag)
757
-
758
-
759
- def self.new(*args, **opts)
760
- obj = allocate
761
- obj.send(:create, *args, **opts)
762
- obj
763
- end
764
-
765
-
766
- def self.restore(hash)
767
- obj = ROUTE.fetch(hash.fetch(:operation).intern).allocate
768
- obj.send(:restore, hash)
769
- obj
770
- end
771
-
772
-
773
- end
774
-
775
-
776
- class Encrypt < Encryption
777
-
778
-
779
- def encrypted(task) = task.destination
780
-
781
-
782
- def decrypted(task) = task.source
783
-
784
-
785
- def externalize = super.merge(operation: :encrypt)
786
-
787
-
788
- def show_operation = 'encrypt+'
789
-
790
-
791
- def arguments(task) = super + [decrypted(task).root.to_s, ':crypt:']
792
-
793
-
794
- end
795
-
796
-
797
- class Decrypt < Encryption
798
-
799
-
800
- def encrypted(task) = task.source
801
-
802
-
803
- def decrypted(task) = task.destination
804
-
805
-
806
- def externalize = super.merge(operation: :decrypt)
807
-
808
-
809
- def show_operation = 'decrypt+'
810
-
811
-
812
- def arguments(task) = super + [':crypt:', decrypted(task).root.to_s]
813
-
814
-
815
- end
816
-
817
-
818
- ROUTE = {
819
- encrypt: Encrypt,
820
- decrypt: Decrypt
821
- }
822
-
823
-
824
- class Task < Bitferry::Task
825
-
826
-
827
- attr_reader :source, :destination
828
-
829
-
830
- attr_reader :encryption
831
-
832
-
833
- attr_reader :token
834
-
835
-
836
- PROCESS = {
837
- default: ['--metadata']
838
- }
839
- PROCESS[nil] = PROCESS[:default]
840
-
841
-
842
- def initialize(source, destination, encryption: nil, process: nil, **opts)
843
- super(**opts)
844
- @process_options = Bitferry.optional(process, PROCESS)
845
- @source = source.is_a?(Endpoint) ? source : Bitferry.endpoint(source)
846
- @destination = destination.is_a?(Endpoint) ? destination : Bitferry.endpoint(destination)
847
- @encryption = encryption
848
- end
849
-
850
-
851
- def create(*args, process: nil, **opts)
852
- super(*args, process: process, **opts)
853
- encryption.configure(self) unless encryption.nil?
854
- end
855
-
856
-
857
- def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
858
-
859
-
860
- def show_operation = encryption.nil? ? '' : encryption.show_operation
861
-
862
-
863
- def show_direction = '-->'
864
-
865
-
866
- def intact? = live? && source.intact? && destination.intact?
867
-
868
-
869
- def refers?(volume) = source.refers?(volume) || destination.refers?(volume)
870
-
871
-
872
- def touch
873
- @generation = [source.generation, destination.generation].max + 1
874
- super
875
- end
876
-
877
-
878
- def format = nil
879
-
880
-
881
- def common_options
882
- [
883
- '--config', Bitferry.windows? ? 'NUL' : '/dev/null',
884
- case Bitferry.verbosity
885
- when :verbose then '--verbose'
886
- when :quiet then '--quiet'
887
- else nil
888
- end,
889
- Bitferry.verbosity == :verbose ? '--progress' : nil,
890
- Bitferry.simulate? ? '--dry-run' : nil,
891
- ].compact
892
- end
893
-
894
-
895
- def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
896
-
897
-
898
- def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
899
-
900
-
901
- def process_arguments
902
- include_filters + exclude_filters + common_options + process_options + (
903
- encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
904
- )
905
- end
906
-
907
-
908
- def execute(*args)
909
- cmd = [Rclone.executable] + args
910
- cms = cmd.collect(&:shellescape).join(' ')
911
- $stdout.puts cms if Bitferry.verbosity == :verbose
912
- log.info(cms)
913
- if Bitferry.ui == :gui
914
- t = nil
915
- Open3.popen2e(*cmd) do |i, oe, thr|
916
- while x = oe.gets; $stdout.puts(x) end
917
- t = thr
918
- end
919
- status = t.value
920
- else
921
- status = Open3.pipeline(cmd).first
922
- end
923
- raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
924
- status.success?
925
- end
926
-
927
-
928
- def process
929
- log.info("processing task #{tag}")
930
- encryption.process(self) unless encryption.nil?
931
- execute(*process_arguments)
932
- end
933
-
934
-
935
- def externalize
936
- super.merge(
937
- source: source.externalize,
938
- destination: destination.externalize,
939
- encryption: encryption.nil? ? nil : encryption.externalize,
940
- rclone: process_options.empty? ? nil : process_options
941
- ).compact
942
- end
943
-
944
-
945
- def restore(hash)
946
- initialize(
947
- restore_endpoint(hash.fetch(:source)),
948
- restore_endpoint(hash.fetch(:destination)),
949
- tag: hash.fetch(:task),
950
- modified: hash.fetch(:modified, DateTime.now),
951
- process: hash[:rclone],
952
- encryption: hash[:encryption].nil? ? nil : Rclone::Encryption.restore(hash[:encryption])
953
- )
954
- super(hash)
955
- end
956
-
957
-
958
- end
959
-
960
-
961
- class Copy < Task
962
-
963
-
964
- def process_arguments = ['copy'] + super
965
-
966
-
967
- def externalize = super.merge(operation: :copy)
968
-
969
-
970
- def show_operation = super + 'copy'
971
-
972
-
973
- end
974
-
975
-
976
- class Update < Task
977
-
978
-
979
- def process_arguments = ['copy', '--update'] + super
980
-
981
-
982
- def externalize = super.merge(operation: :update)
983
-
984
-
985
- def show_operation = super + 'update'
986
-
987
-
988
- end
989
-
990
-
991
- class Synchronize < Task
992
-
993
-
994
- def process_arguments = ['sync'] + super
995
-
996
-
997
- def externalize = super.merge(operation: :synchronize)
998
-
999
-
1000
- def show_operation = super + 'synchronize'
1001
-
1002
-
1003
- end
1004
-
1005
-
1006
- class Equalize < Task
1007
-
1008
-
1009
- def process_arguments = ['bisync', '--resync'] + super
1010
-
1011
-
1012
- def externalize = super.merge(operation: :equalize)
1013
-
1014
-
1015
- def show_operation = super + 'equalize'
1016
-
1017
-
1018
- def show_direction = '<->'
1019
-
1020
-
1021
- end
1022
-
1023
-
1024
- end
1025
-
1026
-
1027
- module Restic
1028
-
1029
-
1030
- include Logging
1031
- extend Logging
1032
-
1033
-
1034
- def self.executable = @executable ||= (restic = ENV['RESTIC']).nil? ? 'restic' : restic
1035
-
1036
-
1037
- def self.exec(*args)
1038
- cmd = [executable] + args
1039
- log.debug(cmd.collect(&:shellescape).join(' '))
1040
- stdout, status = Open3.capture2(*cmd)
1041
- unless status.success?
1042
- msg = "restic exit code #{status.to_i}"
1043
- log.error(msg)
1044
- raise RuntimeError, msg
1045
- end
1046
- stdout.strip
1047
- end
1048
-
1049
-
1050
- class Task < Bitferry::Task
1051
-
1052
-
1053
- attr_reader :directory, :repository
1054
-
1055
-
1056
- def initialize(directory, repository, **opts)
1057
- super(**opts)
1058
- @directory = directory.is_a?(Endpoint) ? directory : Bitferry.endpoint(directory)
1059
- @repository = repository.is_a?(Endpoint) ? repository : Bitferry.endpoint(repository)
1060
- end
1061
-
1062
-
1063
- def create(directory, repository, password, **opts)
1064
- super(directory, repository, **opts)
1065
- raise TypeError, 'unsupported unencrypted endpoint type' unless self.directory.is_a?(Endpoint::Bitferry)
1066
- Volume[self.directory.volume_tag].vault[tag] = Rclone.obscure(@password = password) # Token is stored on the decrypted end only
1067
- end
1068
-
1069
-
1070
- def password = @password ||= Rclone.reveal(Volume[directory.volume_tag].vault.fetch(tag))
1071
-
1072
-
1073
- def intact? = live? && directory.intact? && repository.intact?
1074
-
1075
-
1076
- def refers?(volume) = directory.refers?(volume) || repository.refers?(volume)
1077
-
1078
-
1079
- def touch
1080
- @generation = [directory.generation, repository.generation].max + 1
1081
- super
1082
- end
1083
-
1084
-
1085
- def format = nil
1086
-
1087
-
1088
- def include_filters = include.collect { |x| ['--include', x]}.flatten
1089
-
1090
-
1091
- def common_options
1092
- [
1093
- case Bitferry.verbosity
1094
- when :verbose then '--verbose'
1095
- when :quiet then '--quiet'
1096
- else nil
1097
- end,
1098
- '-r', repository.root.to_s
1099
- ].compact
1100
- end
1101
-
1102
-
1103
- def execute(*args, simulate: false, chdir: nil)
1104
- cmd = [Restic.executable] + args
1105
- ENV['RESTIC_PASSWORD'] = password
1106
- ENV['RESTIC_PROGRESS_FPS'] = 1.to_s if Bitferry.verbosity == :verbose && Bitferry.ui == :gui
1107
- cms = cmd.collect(&:shellescape).join(' ')
1108
- $stdout.puts cms if Bitferry.verbosity == :verbose
1109
- log.info(cms)
1110
- if simulate
1111
- log.info('(simulated)')
1112
- true
1113
- else
1114
- wd = Dir.getwd unless chdir.nil?
1115
- begin
1116
- Dir.chdir(chdir) unless chdir.nil?
1117
- if Bitferry.ui == :gui
1118
- t = nil
1119
- Open3.popen2e(*cmd) do |i, oe, thr|
1120
- while x = oe.gets; $stdout.puts(x) end
1121
- t = thr
1122
- end
1123
- status = t.value
1124
- else
1125
- status = Open3.pipeline(cmd).first
1126
- end
1127
- raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
1128
- status.success?
1129
- ensure
1130
- Dir.chdir(wd) unless chdir.nil?
1131
- end
1132
- end
1133
- end
1134
-
1135
-
1136
- def externalize
1137
- super.merge(
1138
- directory: directory.externalize,
1139
- repository: repository.externalize,
1140
- ).compact
1141
- end
1142
-
1143
-
1144
- def restore(hash)
1145
- initialize(
1146
- restore_endpoint(hash.fetch(:directory)),
1147
- restore_endpoint(hash.fetch(:repository)),
1148
- tag: hash.fetch(:task),
1149
- modified: hash.fetch(:modified, DateTime.now)
1150
- )
1151
- super(hash)
1152
- end
1153
-
1154
-
1155
- end
1156
-
1157
-
1158
- class Backup < Task
1159
-
1160
-
1161
- PROCESS = {
1162
- default: ['--no-cache']
1163
- }
1164
- PROCESS[nil] = PROCESS[:default]
1165
-
1166
-
1167
- FORGET = {
1168
- default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
1169
- }
1170
- FORGET[nil] = nil # Skip processing retention policy by default
1171
-
1172
-
1173
- CHECK = {
1174
- default: ['--no-cache'],
1175
- full: ['--no-cache', '--read-data']
1176
- }
1177
- CHECK[nil] = nil # Skip integrity checking by default
1178
-
1179
-
1180
- attr_reader :forget_options
1181
- attr_reader :check_options
1182
-
1183
-
1184
- def create(*args, format: nil, process: nil, forget: nil, check: nil, **opts)
1185
- super(*args, **opts)
1186
- @format = format
1187
- @process_options = Bitferry.optional(process, PROCESS)
1188
- @forget_options = Bitferry.optional(forget, FORGET)
1189
- @check_options = Bitferry.optional(check, CHECK)
1190
- end
1191
-
1192
-
1193
- def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
1194
-
1195
-
1196
- def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
1197
-
1198
-
1199
- def show_operation = 'encrypt+backup'
1200
-
1201
-
1202
- def show_direction = '-->'
1203
-
1204
-
1205
- alias :source :directory
1206
- alias :destination :repository
1207
-
1208
-
1209
- def process
1210
- begin
1211
- log.info("processing task #{tag}")
1212
- execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
1213
- unless check_options.nil?
1214
- log.info("checking repository in #{repository.root}")
1215
- execute('check', *check_options, *common_options)
1216
- end
1217
- unless forget_options.nil?
1218
- log.info("performing repository maintenance tasks in #{repository.root}")
1219
- execute('forget', '--tag', "bitferry,#{tag}", *forget_options.collect(&:to_s), *common_options_simulate)
1220
- end
1221
- true
1222
- rescue
1223
- false
1224
- end
1225
- end
1226
-
1227
-
1228
- def common_options_simulate = common_options + [Bitferry.simulate? ? '--dry-run' : nil].compact
1229
-
1230
-
1231
- def externalize
1232
- restic = {
1233
- process: process_options,
1234
- forget: forget_options,
1235
- check: check_options
1236
- }.compact
1237
- super.merge({
1238
- operation: :backup,
1239
- restic: restic.empty? ? nil : restic
1240
- }.compact)
1241
- end
1242
-
1243
-
1244
- def restore(hash)
1245
- super
1246
- opts = hash.fetch(:restic, {})
1247
- @process_options = opts[:process]
1248
- @forget_options = opts[:forget]
1249
- @check_options = opts[:check]
1250
- end
1251
-
1252
-
1253
- def format
1254
- if Bitferry.simulate?
1255
- log.info('skipped repository initialization (simulation)')
1256
- else
1257
- log.info("initializing repository for task #{tag}")
1258
- if @format == true
1259
- log.debug("wiping repository in #{repository.root}")
1260
- ['config', 'data', 'index', 'keys', 'locks', 'snapshots'].each { |x| FileUtils.rm_rf(File.join(repository.root.to_s, x)) }
1261
- end
1262
- if @format == false
1263
- # TODO validate existing repo
1264
- log.info("attached to existing repository for task #{tag} in #{repository.root}")
1265
- else
1266
- begin
1267
- execute(*common_options, 'init')
1268
- log.info("initialized repository for task #{tag} in #{repository.root}")
1269
- rescue
1270
- log.fatal("failed to initialize repository for task #{tag} in #{repository.root}")
1271
- raise
1272
- end
1273
- end
1274
- end
1275
- @state = :intact
1276
- end
1277
-
1278
-
1279
- end
1280
-
1281
-
1282
- class Restore < Task
1283
-
1284
-
1285
- PROCESS = {
1286
- default: ['--no-cache', '--sparse']
1287
- }
1288
- PROCESS[nil] = PROCESS[:default]
1289
-
1290
-
1291
- def create(*args, process: nil, **opts)
1292
- super(*args, **opts)
1293
- @process_options = Bitferry.optional(process, PROCESS)
1294
- end
1295
-
1296
-
1297
- def exclude_filters = exclude.collect { |x| ['--exclude', x] }.flatten
1298
-
1299
-
1300
- def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
1301
-
1302
-
1303
- def show_operation = 'decrypt+restore'
1304
-
1305
-
1306
- def show_direction = '-->'
1307
-
1308
-
1309
- alias :destination :directory
1310
- alias :source :repository
1311
-
1312
-
1313
- def externalize
1314
- restic = {
1315
- process: process_options
1316
- }.compact
1317
- super.merge({
1318
- operation: :restore,
1319
- restic: restic.empty? ? nil : restic
1320
- }.compact)
1321
- end
1322
-
1323
-
1324
- def restore(hash)
1325
- super
1326
- opts = hash.fetch(:rclone, {})
1327
- @process_options = opts[:process]
1328
- end
1329
-
1330
-
1331
- def process
1332
- log.info("processing task #{tag}")
1333
- begin
1334
- # FIXME restore specifically tagged latest snapshot
1335
- execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
1336
- true
1337
- rescue
1338
- false
1339
- end
1340
- end
1341
-
1342
-
1343
- end
1344
-
1345
-
1346
-
1347
- end
1348
-
1349
-
1350
- Task::ROUTE = {
1351
- copy: Rclone::Copy,
1352
- update: Rclone::Update,
1353
- synchronize: Rclone::Synchronize,
1354
- equalize: Rclone::Equalize,
1355
- backup: Restic::Backup,
1356
- restore: Restic::Restore
1357
- }
1358
-
1359
-
1360
- class Endpoint
1361
-
1362
-
1363
- def self.restore(hash)
1364
- endpoint = allocate
1365
- endpoint.send(:restore, hash)
1366
- endpoint
1367
- end
1368
-
1369
-
1370
- class Local < Endpoint
1371
-
1372
-
1373
- attr_reader :root
1374
-
1375
-
1376
- def initialize(root) = @root = Pathname.new(root).realdirpath
1377
-
1378
-
1379
- def restore(hash) = initialize(hash.fetch(:root))
1380
-
1381
-
1382
- def externalize
1383
- {
1384
- endpoint: :local,
1385
- root: root
1386
- }
1387
- end
1388
-
1389
-
1390
- def show_status = root.to_s
1391
-
1392
-
1393
- def intact? = true
1394
-
1395
-
1396
- def refers?(volume) = false
1397
-
1398
-
1399
- def generation = 0
1400
-
1401
-
1402
- end
1403
-
1404
-
1405
- class Rclone < Endpoint
1406
- # TODO
1407
- end
1408
-
1409
-
1410
- class Bitferry < Endpoint
1411
-
1412
-
1413
- attr_reader :volume_tag
1414
-
1415
-
1416
- attr_reader :path
1417
-
1418
-
1419
- def root = Volume[volume_tag].root.join(path)
1420
-
1421
-
1422
- def initialize(volume, path)
1423
- @volume_tag = volume.tag
1424
- @path = Pathname.new(path)
1425
- raise ArgumentError, "expected relative path but got #{self.path}" unless (/^[\.\/]/ =~ self.path.to_s).nil?
1426
- end
1427
-
1428
-
1429
- def restore(hash)
1430
- @volume_tag = hash.fetch(:volume)
1431
- @path = Pathname.new(hash.fetch(:path, ''))
1432
- end
1433
-
1434
-
1435
- def externalize
1436
- {
1437
- endpoint: :bitferry,
1438
- volume: volume_tag,
1439
- path: path.to_s.empty? ? nil : path
1440
- }.compact
1441
- end
1442
-
1443
-
1444
- def show_status = intact? ? ":#{volume_tag}:#{path}" : ":{#{volume_tag}}:#{path}"
1445
-
1446
-
1447
- def intact? = !Volume[volume_tag].nil?
1448
-
1449
-
1450
- def refers?(volume) = volume.tag == volume_tag
1451
-
1452
-
1453
- def generation
1454
- v = Volume[volume_tag]
1455
- v ? v.generation : 0
1456
- end
1457
-
1458
-
1459
- end
1460
-
1461
-
1462
- ROUTE = {
1463
- local: Local,
1464
- rclone: Rclone,
1465
- bitferry: Bitferry
1466
- }
1467
-
1468
-
1469
- end
1470
-
1471
-
1472
- reset
1473
-
1474
-
1475
- end
1
+ require 'json'
2
+ require 'date'
3
+ require 'open3'
4
+ require 'base64'
5
+ require 'logger'
6
+ require 'openssl'
7
+ require 'pathname'
8
+ require 'rbconfig'
9
+ require 'fileutils'
10
+ require 'shellwords'
11
+
12
+
13
+ module Bitferry
14
+
15
+
16
+ VERSION = '0.0.8'
17
+
18
+
19
+ module Logging
20
+ def self.log
21
+ unless @log
22
+ @log = Logger.new($stderr)
23
+ @log.level = Logger::WARN
24
+ @log.progname = :bitferry
25
+ end
26
+ @log
27
+ end
28
+ def self.log=(log) @log = log end
29
+ def log = Logging.log
30
+ end
31
+
32
+
33
+ include Logging
34
+ extend Logging
35
+
36
+
37
+ def self.tag = format('%08x', 2**32*rand)
38
+
39
+
40
+ def self.restore
41
+ reset
42
+ log.info('restoring volumes')
43
+ result = true
44
+ roots = (environment_mounts + system_mounts + [Dir.home]).uniq
45
+ log.info("distilled volume search path: #{roots.join(', ')}")
46
+ roots.each do |root|
47
+ if File.exist?(File.join(root, Volume::STORAGE))
48
+ log.info("trying to restore volume from #{root}")
49
+ Volume.restore(root) rescue result = false
50
+ end
51
+ end
52
+ if result
53
+ log.info('volumes restored')
54
+ else
55
+ log.warn('volume restore failure(s) reported')
56
+ end
57
+ result
58
+ end
59
+
60
+
61
+ def self.commit
62
+ log.info('committing changes')
63
+ result = true
64
+ modified = false
65
+ Volume.registered.each do |volume|
66
+ begin
67
+ modified = true if volume.modified?
68
+ volume.commit
69
+ rescue IOError => e
70
+ log.error(e.message)
71
+ result = false
72
+ end
73
+ end
74
+ if result
75
+ log.info(modified ? 'changes committed' : 'commits skipped (no changes)')
76
+ else
77
+ log.warn('commit failure(s) reported')
78
+ end
79
+ result
80
+ end
81
+
82
+
83
+ def self.reset
84
+ log.info('resetting state')
85
+ Volume.reset
86
+ Task.reset
87
+ end
88
+
89
+
90
+ def self.intact_tasks = Volume.intact.collect { |volume| volume.intact_tasks }.flatten.uniq
91
+
92
+ def self.process(*tags, &block)
93
+ log.info('processing tasks')
94
+ tasks = intact_tasks
95
+ if tags.empty?
96
+ process = tasks
97
+ else
98
+ process = []
99
+ tags.each do |tag|
100
+ case (tasks = Task.match([tag], tasks)).size
101
+ when 0 then log.warn("no tasks matching (partial) tag #{tag}")
102
+ when 1 then process += tasks
103
+ else
104
+ tags = tasks.collect { |v| v.tag }.join(', ')
105
+ raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
106
+ end
107
+ end
108
+ end
109
+ tasks = process.uniq
110
+ total = tasks.size
111
+ processed = 0
112
+ failed = 0
113
+ result = tasks.all? do |task|
114
+ r = task.process
115
+ processed += 1
116
+ failed += 1 unless r
117
+ yield(total, processed, failed) if block_given?
118
+ r
119
+ end
120
+ result ? log.info('tasks processed') : log.warn('task process failure(s) reported')
121
+ result
122
+ end
123
+
124
+
125
+ def self.endpoint(root)
126
+ case root
127
+ when /^:(\w+):(.*)/
128
+ volumes = Volume.lookup($1)
129
+ volume = case volumes.size
130
+ when 0 then raise ArgumentError, "no intact volume matching (partial) tag #{$1}"
131
+ when 1 then volumes.first
132
+ else
133
+ tags = volumes.collect { |v| v.tag }.join(', ')
134
+ raise ArgumentError, "multiple intact volumes matching (partial) tag #{$1}: #{tags}"
135
+ end
136
+ Endpoint::Bitferry.new(volume, $2)
137
+ when /^(?:local)?:(.*)/ then Endpoint::Local.new($1)
138
+ when /^(\w{2,}):(.*)/ then Endpoint::Rclone.new($1, $2)
139
+ else Volume.endpoint(root)
140
+ end
141
+ end
142
+
143
+
144
+ @simulate = false
145
+ def self.simulate? = @simulate
146
+ def self.simulate=(mode) @simulate = mode end
147
+
148
+
149
+ @verbosity = :default
150
+ def self.verbosity = @verbosity
151
+ def self.verbosity=(mode) @verbosity = mode end
152
+
153
+
154
+ @ui = :cli
155
+ def self.ui = @ui
156
+ def self.ui=(ui) @ui = ui end
157
+
158
+
159
+ # Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
160
+ def self.windows?
161
+ @windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
162
+ end
163
+
164
+ # Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
165
+ # Look for the $BITFERRY_PATH environment variable
166
+ def self.environment_mounts
167
+ ENV['BITFERRY_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
168
+ end
169
+
170
+
171
+ # Specify OS-specific path name list separator (such as in the $PATH environment variable)
172
+ PATH_LIST_SEPARATOR = windows? ? ';' : ':'
173
+
174
+
175
+ # Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Bitferry voulmes
176
+ UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc|efi)!
177
+
178
+
179
+ # Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
180
+ if RUBY_PLATFORM =~ /java/
181
+ require 'java'
182
+ def self.system_mounts
183
+ java.nio.file.FileSystems.getDefault.getFileStores.collect {|x| /^(.*)\s+\(.*\)$/.match(x.to_s)[1]}
184
+ end
185
+ else
186
+ case RbConfig::CONFIG['target_os']
187
+ when 'linux'
188
+ # Linux OS
189
+ def self.system_mounts
190
+ # Query /proc for currently mounted file systems
191
+ IO.readlines('/proc/mounts').collect do |line|
192
+ mount = line.split[1]
193
+ UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
194
+ end.compact
195
+ end
196
+ # TODO handle Windows variants
197
+ when /^mingw/ # RubyInstaller's MRI
198
+ module Kernel32
199
+ require 'fiddle'
200
+ require 'fiddle/types'
201
+ require 'fiddle/import'
202
+ extend Fiddle::Importer
203
+ dlload('kernel32')
204
+ include Fiddle::Win32Types
205
+ extern 'DWORD WINAPI GetLogicalDrives()'
206
+ end
207
+ def self.system_mounts
208
+ mounts = []
209
+ mask = Kernel32.GetLogicalDrives
210
+ ('A'..'Z').each do |x|
211
+ mounts << "#{x}:/" if mask & 1 == 1
212
+ mask >>= 1
213
+ end
214
+ mounts
215
+ end
216
+ else
217
+ # Generic *NIX-like OS, including Cygwin & MSYS2
218
+ def self.system_mounts
219
+ # Use $(mount) system utility to obtain currently mounted file systems
220
+ %x(mount).split("\n").collect do |line|
221
+ mount = line.split[2]
222
+ UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
223
+ end.compact
224
+ end
225
+ end
226
+ end
227
+
228
+
229
+ class Volume
230
+
231
+
232
+ include Logging
233
+ extend Logging
234
+
235
+
236
+ STORAGE = '.bitferry'
237
+ STORAGE_ = '.bitferry~'
238
+ STORAGE_MASK = '.bitferry*'
239
+
240
+
241
+ attr_reader :tag
242
+
243
+
244
+ attr_reader :generation
245
+
246
+
247
+ attr_reader :root
248
+
249
+
250
+ attr_reader :vault
251
+
252
+
253
+ def self.[](tag)
254
+ @@registry.each_value { |volume| return volume if volume.tag == tag }
255
+ nil
256
+ end
257
+
258
+
259
+ # Return list of registered volumes whose tags match at least one specified partial
260
+ def self.lookup(*tags) = match(tags, registered)
261
+
262
+
263
+ def self.match(tags, volumes)
264
+ rxs = tags.collect { |x| Regexp.new(x) }
265
+ volumes.filter do |volume|
266
+ rxs.any? { |rx| !(rx =~ volume.tag).nil? }
267
+ end
268
+ end
269
+
270
+
271
+ def self.new(root, **opts)
272
+ volume = allocate
273
+ volume.send(:create, root, **opts)
274
+ register(volume)
275
+ end
276
+
277
+
278
+ def self.restore(root)
279
+ begin
280
+ volume = allocate
281
+ volume.send(:restore, root)
282
+ volume = register(volume)
283
+ log.info("restored volume #{volume.tag} from #{root}")
284
+ volume
285
+ rescue => e
286
+ log.error("failed to restore volume from #{root}")
287
+ log.error(e.message) if $DEBUG
288
+ raise
289
+ end
290
+ end
291
+
292
+
293
+ def self.delete(*tags, wipe: false)
294
+ process = []
295
+ tags.each do |tag|
296
+ case (volumes = Volume.lookup(tag)).size
297
+ when 0 then log.warn("no volumes matching (partial) tag #{tag}")
298
+ when 1 then process += volumes
299
+ else
300
+ tags = volumes.collect { |v| v.tag }.join(', ')
301
+ raise ArgumentError, "multiple volumes matching (partial) tag #{tag}: #{tags}"
302
+ end
303
+ end
304
+ process.each { |volume| volume.delete(wipe: wipe) }
305
+ end
306
+
307
+
308
+ def initialize(root, tag: Bitferry.tag, modified: nil, overwrite: false)
309
+ @tag = tag
310
+ @generation = 0
311
+ @vault = {}
312
+ @modified = case modified
313
+ when nil then DateTime.now
314
+ when DateTime then modified
315
+ else DateTime.parse(modified)
316
+ end
317
+ @overwrite = overwrite
318
+ @root = Pathname.new(root).realdirpath
319
+ end
320
+
321
+
322
+ def create(*args, **opts)
323
+ initialize(*args, **opts)
324
+ @state = :pristine
325
+ @modified = true
326
+ end
327
+
328
+
329
+ def restore(root)
330
+ hash = JSON.load_file(storage = Pathname(root).join(STORAGE), { symbolize_names: true })
331
+ raise IOError, "bad volume storage #{storage}" unless hash.fetch(:bitferry) == "0"
332
+ volume = hash.fetch(:volume)
333
+ begin
334
+ modified = DateTime.parse(hash.fetch(:modified))
335
+ @modified = false
336
+ rescue
337
+ modified = nil
338
+ @modified = true
339
+ log.warn("modified key missing - flagging volume #{volume} out of date")
340
+ end
341
+ initialize(root, tag: volume, modified: modified)
342
+ hash.fetch(:tasks, []).each { |hash| Task::ROUTE.fetch(hash.fetch(:operation).intern).restore(hash) }
343
+ @vault = hash.fetch(:vault, {}).transform_keys { |key| key.to_s }
344
+ @state = :intact
345
+ end
346
+
347
+
348
+ def storage = @storage ||= root.join(STORAGE)
349
+ def storage_ = @storage_ ||= root.join(STORAGE_)
350
+
351
+
352
+ def commit
353
+ if modified?
354
+ log.info("commit volume #{tag} (modified)")
355
+ case @state
356
+ when :pristine
357
+ format
358
+ store
359
+ when :intact
360
+ store
361
+ when :removing
362
+ remove
363
+ else
364
+ raise
365
+ end
366
+ committed
367
+ else
368
+ log.info("skipped committing volume #{tag} (unmodified)")
369
+ end
370
+ end
371
+
372
+
373
+ ENCOMPASSING_PATH_PREFIX = /^(?!(\.\.$|\.\.\/))/ # not .. by itself and not starting with ../
374
+
375
+
376
+ def self.endpoint(root)
377
+ path = Pathname.new(root).realdirpath
378
+ intact.sort { |v1, v2| v2.root.to_s.size <=> v1.root.to_s.size }.each do |volume|
379
+ begin
380
+ stem = path.relative_path_from(volume.root).to_s #.chomp('/')
381
+ case stem
382
+ when '.' then return volume.endpoint
383
+ when ENCOMPASSING_PATH_PREFIX then return volume.endpoint(stem)
384
+ end
385
+ rescue ArgumentError
386
+ # Catch different prefix error on Windows
387
+ end
388
+ end
389
+ raise ArgumentError, "no intact volume encompasses path #{root}"
390
+ end
391
+
392
+
393
+ def endpoint(path = String.new) = Endpoint::Bitferry.new(self, path)
394
+
395
+
396
+ def modified? = @modified || tasks.any? { |t| t.generation > generation }
397
+
398
+
399
+ def intact? = @state != :removing
400
+
401
+
402
+ def touch
403
+ x = tasks.collect { |t| t.generation }.max
404
+ @generation = x ? x + 1 : 0
405
+ @modified = true
406
+ end
407
+
408
+
409
+ def delete(wipe: false)
410
+ touch
411
+ @wipe = wipe
412
+ @state = :removing
413
+ log.info("marked volume #{tag} for deletion")
414
+ end
415
+
416
+
417
+ def committed
418
+ x = tasks.collect { |t| t.generation }.min
419
+ @generation = x ? x : 0
420
+ @modified = false
421
+ end
422
+
423
+
424
+ def store
425
+ require 'neatjson'
426
+ tasks.each(&:commit)
427
+ hash = JSON.neat_generate(externalize, short: false, wrap: 200, afterColon: 1, afterComma: 1)
428
+ if Bitferry.simulate?
429
+ log.info("skipped volume #{tag} storage modification (simulation)")
430
+ else
431
+ begin
432
+ File.write(storage_, hash)
433
+ FileUtils.mv(storage_, storage)
434
+ log.info("written volume #{tag} storage #{storage}")
435
+ ensure
436
+ FileUtils.rm_f(storage_)
437
+ end
438
+ end
439
+ @state = :intact
440
+ end
441
+
442
+
443
+ def format
444
+ raise IOError, "refuse to overwrite existing volume storage #{storage}" if !@overwrite && File.exist?(storage)
445
+ if Bitferry.simulate?
446
+ log.info("skipped storage formatting (simulation)")
447
+ else
448
+ FileUtils.mkdir_p(root)
449
+ FileUtils.rm_f [storage, storage_]
450
+ log.info("formatted volume #{tag} in #{root}")
451
+ end
452
+ @state = nil
453
+ end
454
+
455
+
456
+ def remove
457
+ unless Bitferry.simulate?
458
+ if @wipe
459
+ FileUtils.rm_rf(Dir[File.join(root, '*'), File.join(root, '.*')])
460
+ log.info("wiped entire volume directory #{root}")
461
+ else
462
+ FileUtils.rm_f [storage, storage_]
463
+ log.info("deleted volume #{tag} storage files #{File.join(root, STORAGE_MASK)}")
464
+ end
465
+ end
466
+ @@registry.delete(root)
467
+ @state = nil
468
+ end
469
+
470
+
471
+ def externalize
472
+ tasks = live_tasks
473
+ v = vault.filter { |t| !Task[t].nil? && Task[t].live? } # Purge entries from non-existing (deleted) tasks
474
+ {
475
+ bitferry: "0",
476
+ volume: tag,
477
+ modified: (@modified = DateTime.now),
478
+ tasks: tasks.empty? ? nil : tasks.collect(&:externalize),
479
+ vault: v.empty? ? nil : v
480
+ }.compact
481
+ end
482
+
483
+
484
+ def tasks = Task.registered.filter { |task| task.refers?(self) }
485
+
486
+
487
+ def live_tasks = Task.live.filter { |task| task.refers?(self) }
488
+
489
+
490
+ def intact_tasks = live_tasks.filter { |task| task.intact? }
491
+
492
+
493
+ def self.reset = @@registry = {}
494
+
495
+
496
+ def self.register(volume) = @@registry[volume.root] = volume
497
+
498
+
499
+ def self.registered = @@registry.values
500
+
501
+
502
+ def self.intact = registered.filter { |volume| volume.intact? }
503
+
504
+
505
+ end
506
+
507
+
508
+ def self.optional(option, route)
509
+ case option
510
+ when Array then option # Array is passed verbatim
511
+ when '-' then nil # Disable adding any options with -
512
+ when /^-/ then option.split(',') # Split comma-separated string into array --foo,bar --> [--foo, bar]
513
+ else route.fetch(option.nil? ? nil : option.to_sym) # Obtain options from the profile database
514
+ end
515
+ end
516
+
517
+
518
+ class Task
519
+
520
+
521
+ include Logging
522
+ extend Logging
523
+
524
+
525
+ attr_reader :tag
526
+
527
+
528
+ attr_reader :generation
529
+
530
+
531
+ attr_reader :modified
532
+
533
+
534
+ attr_reader :include, :exclude
535
+
536
+
537
+ def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
538
+
539
+
540
+ def self.new(*args, **opts)
541
+ task = allocate
542
+ task.send(:create, *args, **opts)
543
+ register(task)
544
+ end
545
+
546
+
547
+ def self.restore(hash)
548
+ task = allocate
549
+ task.send(:restore, hash)
550
+ register(task)
551
+ end
552
+
553
+
554
+ def self.delete(*tags)
555
+ process = []
556
+ tags.each do |tag|
557
+ case (tasks = Task.lookup(tag)).size
558
+ when 0 then log.warn("no tasks matching (partial) tag #{tag}")
559
+ when 1 then process += tasks
560
+ else
561
+ tags = tasks.collect { |v| v.tag }.join(', ')
562
+ raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
563
+ end
564
+ end
565
+ process.each { |task| task.delete }
566
+ end
567
+
568
+
569
+ def initialize(tag: Bitferry.tag, modified: nil, include: [], exclude: [])
570
+ @tag = tag
571
+ @generation = 0
572
+ @include = include
573
+ @exclude = exclude
574
+ @modified = case modified
575
+ when nil then DateTime.now
576
+ when DateTime then modified
577
+ else DateTime.parse(modified)
578
+ end
579
+ # FIXME handle process_options at this level
580
+ end
581
+
582
+
583
+ def create(*args, **opts)
584
+ initialize(*args, **opts)
585
+ @state = :pristine
586
+ touch
587
+ end
588
+
589
+
590
+ def restore(hash)
591
+ @include = hash.fetch(:include, [])
592
+ @exclude = hash.fetch(:exclude, [])
593
+ @state = :intact
594
+ log.info("restored task #{tag}")
595
+ end
596
+
597
+
598
+ # FIXME move to Endpoint#restore
599
+ def restore_endpoint(x) = Endpoint::ROUTE.fetch(x.fetch(:endpoint).intern).restore(x)
600
+
601
+
602
+ def externalize
603
+ {
604
+ task: tag,
605
+ modified: modified,
606
+ include: include.empty? ? nil : include,
607
+ exclude: exclude.empty? ? nil : exclude
608
+ }.compact
609
+ end
610
+
611
+
612
+ def live? = !@state.nil? && @state != :removing
613
+
614
+
615
+ def touch = @modified = DateTime.now
616
+
617
+
618
+ def delete
619
+ touch
620
+ @state = :removing
621
+ log.info("marked task #{tag} for removal")
622
+ end
623
+
624
+
625
+ def commit
626
+ case @state
627
+ when :pristine then format
628
+ when :removing then @state = nil
629
+ end
630
+ end
631
+
632
+
633
+ def show_filters
634
+ xs = []
635
+ xs << 'include: ' + include.join(',') unless include.empty?
636
+ xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
637
+ xs.join(' ').to_s
638
+ end
639
+
640
+
641
+ def self.[](tag) = @@registry[tag]
642
+
643
+
644
+ # Return list of registered tasks whose tags match at least one of specified partial tags
645
+ def self.lookup(*tags) = match(tags, registered)
646
+
647
+
648
+ # Return list of specified tasks whose tags match at least one of specified partial tags
649
+ def self.match(tags, tasks)
650
+ rxs = tags.collect { |x| Regexp.new(x) }
651
+ tasks.filter do |task|
652
+ rxs.any? { |rx| !(rx =~ task.tag).nil? }
653
+ end
654
+ end
655
+
656
+
657
+ def self.registered = @@registry.values
658
+
659
+
660
+ def self.live = registered.filter { |task| task.live? }
661
+
662
+
663
+ def self.reset = @@registry = {}
664
+
665
+
666
+ def self.register(task)
667
+ # Task with newer timestamp replaces already registered task, if any
668
+ if (xtag = @@registry[task.tag]).nil?
669
+ @@registry[task.tag] = task
670
+ elsif xtag.modified < task.modified
671
+ @@registry[task.tag] = task
672
+ else
673
+ xtag
674
+ end
675
+ end
676
+
677
+ def self.intact = live.filter { |task| task.intact? }
678
+
679
+
680
+ def self.stale = live.filter { |task| !task.intact? }
681
+
682
+
683
+ end
684
+
685
+
686
+ module Rclone
687
+
688
+
689
+ include Logging
690
+ extend Logging
691
+
692
+
693
+ def self.executable = @executable ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
694
+
695
+
696
+ def self.exec(*args)
697
+ cmd = [executable] + args
698
+ log.debug(cmd.collect(&:shellescape).join(' '))
699
+ stdout, status = Open3.capture2e(*cmd)
700
+ unless status.success?
701
+ msg = "rclone exit code #{status.exitstatus}"
702
+ log.error(msg)
703
+ raise RuntimeError, msg
704
+ end
705
+ stdout.strip
706
+ end
707
+
708
+
709
+ # https://github.com/rclone/rclone/blob/master/fs/config/obscure/obscure.go
710
+ SECRET = "\x9c\x93\x5b\x48\x73\x0a\x55\x4d\x6b\xfd\x7c\x63\xc8\x86\xa9\x2b\xd3\x90\x19\x8e\xb8\x12\x8a\xfb\xf4\xde\x16\x2b\x8b\x95\xf6\x38"
711
+
712
+
713
+ def self.obscure(plain)
714
+ cipher = OpenSSL::Cipher.new('AES-256-CTR')
715
+ cipher.encrypt
716
+ cipher.key = SECRET
717
+ Base64.urlsafe_encode64(cipher.random_iv + cipher.update(plain) + cipher.final, padding: false)
718
+ end
719
+
720
+
721
+ def self.reveal(token)
722
+ data = Base64.urlsafe_decode64(token)
723
+ cipher = OpenSSL::Cipher.new('AES-256-CTR')
724
+ cipher.decrypt
725
+ cipher.key = SECRET
726
+ cipher.iv = data[0...cipher.iv_len]
727
+ cipher.update(data[cipher.iv_len..-1]) + cipher.final
728
+ end
729
+
730
+
731
+ class Encryption
732
+
733
+
734
+ PROCESS = {
735
+ default: ['--crypt-filename-encoding', :base32, '--crypt-filename-encryption', :standard],
736
+ extended: ['--crypt-filename-encoding', :base32768, '--crypt-filename-encryption', :standard]
737
+ }
738
+ PROCESS[nil] = PROCESS[:default]
739
+
740
+
741
+ def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
742
+
743
+
744
+ def initialize(token, process: nil)
745
+ @process_options = Bitferry.optional(process, PROCESS)
746
+ @token = token
747
+ end
748
+
749
+
750
+ def create(password, **opts) = initialize(Rclone.obscure(password), **opts)
751
+
752
+
753
+ def restore(hash) = @process_options = hash[:rclone]
754
+
755
+
756
+ def externalize = process_options.empty? ? {} : { rclone: process_options }
757
+
758
+
759
+ def configure(task) = install_token(task)
760
+
761
+
762
+ def process(task) = ENV['RCLONE_CRYPT_PASSWORD'] = obtain_token(task)
763
+
764
+
765
+ def arguments(task) = process_options + ['--crypt-remote', encrypted(task).root.to_s]
766
+
767
+
768
+ def install_token(task)
769
+ x = decrypted(task)
770
+ raise TypeError, 'unsupported unencrypted endpoint type' unless x.is_a?(Endpoint::Bitferry)
771
+ Volume[x.volume_tag].vault[task.tag] = @token # Token is stored on the decrypted end only
772
+ end
773
+
774
+
775
+ def obtain_token(task) = Volume[decrypted(task).volume_tag].vault.fetch(task.tag)
776
+
777
+
778
+ def self.new(*args, **opts)
779
+ obj = allocate
780
+ obj.send(:create, *args, **opts)
781
+ obj
782
+ end
783
+
784
+
785
+ def self.restore(hash)
786
+ obj = ROUTE.fetch(hash.fetch(:operation).intern).allocate
787
+ obj.send(:restore, hash)
788
+ obj
789
+ end
790
+
791
+
792
+ end
793
+
794
+
795
+ class Encrypt < Encryption
796
+
797
+
798
+ def encrypted(task) = task.destination
799
+
800
+
801
+ def decrypted(task) = task.source
802
+
803
+
804
+ def externalize = super.merge(operation: :encrypt)
805
+
806
+
807
+ def show_operation = 'encrypt+'
808
+
809
+
810
+ def arguments(task) = super + [decrypted(task).root.to_s, ':crypt:']
811
+
812
+
813
+ end
814
+
815
+
816
+ class Decrypt < Encryption
817
+
818
+
819
+ def encrypted(task) = task.source
820
+
821
+
822
+ def decrypted(task) = task.destination
823
+
824
+
825
+ def externalize = super.merge(operation: :decrypt)
826
+
827
+
828
+ def show_operation = 'decrypt+'
829
+
830
+
831
+ def arguments(task) = super + [':crypt:', decrypted(task).root.to_s]
832
+
833
+
834
+ end
835
+
836
+
837
+ ROUTE = {
838
+ encrypt: Encrypt,
839
+ decrypt: Decrypt
840
+ }
841
+
842
+
843
+ class Task < Bitferry::Task
844
+
845
+
846
+ attr_reader :source, :destination
847
+
848
+
849
+ attr_reader :encryption
850
+
851
+
852
+ attr_reader :token
853
+
854
+
855
+ PROCESS = {
856
+ default: ['--metadata']
857
+ }
858
+ PROCESS[nil] = PROCESS[:default]
859
+
860
+
861
+ def initialize(source, destination, encryption: nil, process: nil, **opts)
862
+ super(**opts)
863
+ @process_options = Bitferry.optional(process, PROCESS)
864
+ @source = source.is_a?(Endpoint) ? source : Bitferry.endpoint(source)
865
+ @destination = destination.is_a?(Endpoint) ? destination : Bitferry.endpoint(destination)
866
+ @encryption = encryption
867
+ end
868
+
869
+
870
+ def create(*args, process: nil, **opts)
871
+ super(*args, process: process, **opts)
872
+ encryption.configure(self) unless encryption.nil?
873
+ end
874
+
875
+
876
+ def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
877
+
878
+
879
+ def show_operation = encryption.nil? ? '' : encryption.show_operation
880
+
881
+
882
+ def show_direction = '-->'
883
+
884
+
885
+ def intact? = live? && source.intact? && destination.intact?
886
+
887
+
888
+ def refers?(volume) = source.refers?(volume) || destination.refers?(volume)
889
+
890
+
891
+ def touch
892
+ @generation = [source.generation, destination.generation].max + 1
893
+ super
894
+ end
895
+
896
+
897
+ def format = nil
898
+
899
+
900
+ def common_options
901
+ [
902
+ '--config', Bitferry.windows? ? 'NUL' : '/dev/null',
903
+ case Bitferry.verbosity
904
+ when :verbose then '--verbose'
905
+ when :quiet then '--quiet'
906
+ else nil
907
+ end,
908
+ Bitferry.verbosity == :verbose ? '--progress' : nil,
909
+ Bitferry.simulate? ? '--dry-run' : nil,
910
+ ].compact
911
+ end
912
+
913
+
914
+ def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
915
+
916
+
917
+ def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
918
+
919
+
920
+ def process_arguments
921
+ include_filters + exclude_filters + common_options + process_options + (
922
+ encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
923
+ )
924
+ end
925
+
926
+
927
+ def execute(*args)
928
+ cmd = [Rclone.executable] + args
929
+ cms = cmd.collect(&:shellescape).join(' ')
930
+ $stdout.puts cms if Bitferry.verbosity == :verbose
931
+ log.info(cms)
932
+ if Bitferry.ui == :gui
933
+ t = nil
934
+ Open3.popen2e(*cmd) do |i, oe, thr|
935
+ while x = oe.gets; $stdout.puts(x) end
936
+ t = thr
937
+ end
938
+ status = t.value
939
+ else
940
+ status = Open3.pipeline(cmd).first
941
+ end
942
+ raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
943
+ status.success?
944
+ end
945
+
946
+
947
+ def process
948
+ log.info("processing task #{tag}")
949
+ encryption.process(self) unless encryption.nil?
950
+ execute(*process_arguments)
951
+ end
952
+
953
+
954
+ def externalize
955
+ super.merge(
956
+ source: source.externalize,
957
+ destination: destination.externalize,
958
+ encryption: encryption.nil? ? nil : encryption.externalize,
959
+ rclone: process_options.empty? ? nil : process_options
960
+ ).compact
961
+ end
962
+
963
+
964
+ def restore(hash)
965
+ task = hash.fetch(:task)
966
+ begin
967
+ modified = hash.fetch(:modified)
968
+ rescue
969
+ modified = nil
970
+ log.warn("modified key missing - flagging task #{task} out of date")
971
+ end
972
+ initialize(
973
+ restore_endpoint(hash.fetch(:source)),
974
+ restore_endpoint(hash.fetch(:destination)),
975
+ tag: task,
976
+ modified: modified,
977
+ process: hash[:rclone],
978
+ encryption: hash[:encryption].nil? ? nil : Rclone::Encryption.restore(hash[:encryption])
979
+ )
980
+ super(hash)
981
+ touch if modified.nil?
982
+ end
983
+
984
+
985
+ end
986
+
987
+
988
+ class Copy < Task
989
+
990
+
991
+ def process_arguments = ['copy'] + super
992
+
993
+
994
+ def externalize = super.merge(operation: :copy)
995
+
996
+
997
+ def show_operation = super + 'copy'
998
+
999
+
1000
+ end
1001
+
1002
+
1003
+ class Update < Task
1004
+
1005
+
1006
+ def process_arguments = ['copy', '--update'] + super
1007
+
1008
+
1009
+ def externalize = super.merge(operation: :update)
1010
+
1011
+
1012
+ def show_operation = super + 'update'
1013
+
1014
+
1015
+ end
1016
+
1017
+
1018
+ class Synchronize < Task
1019
+
1020
+
1021
+ def process_arguments = ['sync'] + super
1022
+
1023
+
1024
+ def externalize = super.merge(operation: :synchronize)
1025
+
1026
+
1027
+ def show_operation = super + 'synchronize'
1028
+
1029
+
1030
+ end
1031
+
1032
+
1033
+ class Equalize < Task
1034
+
1035
+
1036
+ def process_arguments = ['bisync', '--resync'] + super
1037
+
1038
+
1039
+ def externalize = super.merge(operation: :equalize)
1040
+
1041
+
1042
+ def show_operation = super + 'equalize'
1043
+
1044
+
1045
+ def show_direction = '<->'
1046
+
1047
+
1048
+ end
1049
+
1050
+
1051
+ end
1052
+
1053
+
1054
+ module Restic
1055
+
1056
+
1057
+ include Logging
1058
+ extend Logging
1059
+
1060
+
1061
+ def self.executable = @executable ||= (restic = ENV['RESTIC']).nil? ? 'restic' : restic
1062
+
1063
+
1064
+ def self.exec(*args)
1065
+ cmd = [executable] + args
1066
+ log.debug(cmd.collect(&:shellescape).join(' '))
1067
+ stdout, status = Open3.capture2(*cmd)
1068
+ unless status.success?
1069
+ msg = "restic exit code #{status.to_i}"
1070
+ log.error(msg)
1071
+ raise RuntimeError, msg
1072
+ end
1073
+ stdout.strip
1074
+ end
1075
+
1076
+
1077
+ class Task < Bitferry::Task
1078
+
1079
+
1080
+ attr_reader :directory, :repository
1081
+
1082
+
1083
+ def initialize(directory, repository, **opts)
1084
+ super(**opts)
1085
+ @directory = directory.is_a?(Endpoint) ? directory : Bitferry.endpoint(directory)
1086
+ @repository = repository.is_a?(Endpoint) ? repository : Bitferry.endpoint(repository)
1087
+ end
1088
+
1089
+
1090
+ def create(directory, repository, password, **opts)
1091
+ super(directory, repository, **opts)
1092
+ raise TypeError, 'unsupported unencrypted endpoint type' unless self.directory.is_a?(Endpoint::Bitferry)
1093
+ Volume[self.directory.volume_tag].vault[tag] = Rclone.obscure(@password = password) # Token is stored on the decrypted end only
1094
+ end
1095
+
1096
+
1097
+ def password = @password ||= Rclone.reveal(Volume[directory.volume_tag].vault.fetch(tag))
1098
+
1099
+
1100
+ def intact? = live? && directory.intact? && repository.intact?
1101
+
1102
+
1103
+ def refers?(volume) = directory.refers?(volume) || repository.refers?(volume)
1104
+
1105
+
1106
+ def touch
1107
+ @generation = [directory.generation, repository.generation].max + 1
1108
+ super
1109
+ end
1110
+
1111
+
1112
+ def format = nil
1113
+
1114
+
1115
+ def include_filters = include.collect { |x| ['--include', x]}.flatten
1116
+
1117
+
1118
+ def common_options
1119
+ [
1120
+ case Bitferry.verbosity
1121
+ when :verbose then '--verbose'
1122
+ when :quiet then '--quiet'
1123
+ else nil
1124
+ end,
1125
+ '-r', repository.root.to_s
1126
+ ].compact
1127
+ end
1128
+
1129
+
1130
+ def execute(*args, simulate: false, chdir: nil)
1131
+ cmd = [Restic.executable] + args
1132
+ ENV['RESTIC_PASSWORD'] = password
1133
+ ENV['RESTIC_PROGRESS_FPS'] = 1.to_s if Bitferry.verbosity == :verbose && Bitferry.ui == :gui
1134
+ cms = cmd.collect(&:shellescape).join(' ')
1135
+ $stdout.puts cms if Bitferry.verbosity == :verbose
1136
+ log.info(cms)
1137
+ if simulate
1138
+ log.info('(simulated)')
1139
+ true
1140
+ else
1141
+ wd = Dir.getwd unless chdir.nil?
1142
+ begin
1143
+ Dir.chdir(chdir) unless chdir.nil?
1144
+ if Bitferry.ui == :gui
1145
+ t = nil
1146
+ Open3.popen2e(*cmd) do |i, oe, thr|
1147
+ while x = oe.gets; $stdout.puts(x) end
1148
+ t = thr
1149
+ end
1150
+ status = t.value
1151
+ else
1152
+ status = Open3.pipeline(cmd).first
1153
+ end
1154
+ raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
1155
+ status.success?
1156
+ ensure
1157
+ Dir.chdir(wd) unless chdir.nil?
1158
+ end
1159
+ end
1160
+ end
1161
+
1162
+
1163
+ def externalize
1164
+ super.merge(
1165
+ directory: directory.externalize,
1166
+ repository: repository.externalize,
1167
+ ).compact
1168
+ end
1169
+
1170
+
1171
+ def restore(hash)
1172
+ initialize(
1173
+ restore_endpoint(hash.fetch(:directory)),
1174
+ restore_endpoint(hash.fetch(:repository)),
1175
+ tag: hash.fetch(:task),
1176
+ modified: hash.fetch(:modified, DateTime.now)
1177
+ )
1178
+ super(hash)
1179
+ end
1180
+
1181
+
1182
+ end
1183
+
1184
+
1185
+ class Backup < Task
1186
+
1187
+
1188
+ PROCESS = {
1189
+ default: ['--no-cache']
1190
+ }
1191
+ PROCESS[nil] = PROCESS[:default]
1192
+
1193
+
1194
+ FORGET = {
1195
+ default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
1196
+ }
1197
+ FORGET[nil] = nil # Skip processing retention policy by default
1198
+
1199
+
1200
+ CHECK = {
1201
+ default: ['--no-cache'],
1202
+ full: ['--no-cache', '--read-data']
1203
+ }
1204
+ CHECK[nil] = nil # Skip integrity checking by default
1205
+
1206
+
1207
+ attr_reader :forget_options
1208
+ attr_reader :check_options
1209
+
1210
+
1211
+ def create(*args, format: nil, process: nil, forget: nil, check: nil, **opts)
1212
+ super(*args, **opts)
1213
+ @format = format
1214
+ @process_options = Bitferry.optional(process, PROCESS)
1215
+ @forget_options = Bitferry.optional(forget, FORGET)
1216
+ @check_options = Bitferry.optional(check, CHECK)
1217
+ end
1218
+
1219
+
1220
+ def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
1221
+
1222
+
1223
+ def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
1224
+
1225
+
1226
+ def show_operation = 'encrypt+backup'
1227
+
1228
+
1229
+ def show_direction = '-->'
1230
+
1231
+
1232
+ alias :source :directory
1233
+ alias :destination :repository
1234
+
1235
+
1236
+ def process
1237
+ begin
1238
+ log.info("processing task #{tag}")
1239
+ execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
1240
+ unless check_options.nil?
1241
+ log.info("checking repository in #{repository.root}")
1242
+ execute('check', *check_options, *common_options)
1243
+ end
1244
+ unless forget_options.nil?
1245
+ log.info("performing repository maintenance tasks in #{repository.root}")
1246
+ execute('forget', '--tag', "bitferry,#{tag}", *forget_options.collect(&:to_s), *common_options_simulate)
1247
+ end
1248
+ true
1249
+ rescue
1250
+ false
1251
+ end
1252
+ end
1253
+
1254
+
1255
+ def common_options_simulate = common_options + [Bitferry.simulate? ? '--dry-run' : nil].compact
1256
+
1257
+
1258
+ def externalize
1259
+ restic = {
1260
+ process: process_options,
1261
+ forget: forget_options,
1262
+ check: check_options
1263
+ }.compact
1264
+ super.merge({
1265
+ operation: :backup,
1266
+ restic: restic.empty? ? nil : restic
1267
+ }.compact)
1268
+ end
1269
+
1270
+
1271
+ def restore(hash)
1272
+ super
1273
+ opts = hash.fetch(:restic, {})
1274
+ @process_options = opts[:process]
1275
+ @forget_options = opts[:forget]
1276
+ @check_options = opts[:check]
1277
+ end
1278
+
1279
+
1280
+ def format
1281
+ if Bitferry.simulate?
1282
+ log.info('skipped repository initialization (simulation)')
1283
+ else
1284
+ log.info("initializing repository for task #{tag}")
1285
+ if @format == true
1286
+ log.debug("wiping repository in #{repository.root}")
1287
+ ['config', 'data', 'index', 'keys', 'locks', 'snapshots'].each { |x| FileUtils.rm_rf(File.join(repository.root.to_s, x)) }
1288
+ end
1289
+ if @format == false
1290
+ # TODO validate existing repo
1291
+ log.info("attached to existing repository for task #{tag} in #{repository.root}")
1292
+ else
1293
+ begin
1294
+ execute(*common_options, 'init')
1295
+ log.info("initialized repository for task #{tag} in #{repository.root}")
1296
+ rescue
1297
+ log.fatal("failed to initialize repository for task #{tag} in #{repository.root}")
1298
+ raise
1299
+ end
1300
+ end
1301
+ end
1302
+ @state = :intact
1303
+ end
1304
+
1305
+
1306
+ end
1307
+
1308
+
1309
+ class Restore < Task
1310
+
1311
+
1312
+ PROCESS = {
1313
+ default: ['--no-cache', '--sparse']
1314
+ }
1315
+ PROCESS[nil] = PROCESS[:default]
1316
+
1317
+
1318
+ def create(*args, process: nil, **opts)
1319
+ super(*args, **opts)
1320
+ @process_options = Bitferry.optional(process, PROCESS)
1321
+ end
1322
+
1323
+
1324
+ def exclude_filters = exclude.collect { |x| ['--exclude', x] }.flatten
1325
+
1326
+
1327
+ def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
1328
+
1329
+
1330
+ def show_operation = 'decrypt+restore'
1331
+
1332
+
1333
+ def show_direction = '-->'
1334
+
1335
+
1336
+ alias :destination :directory
1337
+ alias :source :repository
1338
+
1339
+
1340
+ def externalize
1341
+ restic = {
1342
+ process: process_options
1343
+ }.compact
1344
+ super.merge({
1345
+ operation: :restore,
1346
+ restic: restic.empty? ? nil : restic
1347
+ }.compact)
1348
+ end
1349
+
1350
+
1351
+ def restore(hash)
1352
+ super
1353
+ opts = hash.fetch(:rclone, {})
1354
+ @process_options = opts[:process]
1355
+ end
1356
+
1357
+
1358
+ def process
1359
+ log.info("processing task #{tag}")
1360
+ begin
1361
+ # FIXME restore specifically tagged latest snapshot
1362
+ execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
1363
+ true
1364
+ rescue
1365
+ false
1366
+ end
1367
+ end
1368
+
1369
+
1370
+ end
1371
+
1372
+
1373
+
1374
+ end
1375
+
1376
+
1377
+ Task::ROUTE = {
1378
+ copy: Rclone::Copy,
1379
+ update: Rclone::Update,
1380
+ synchronize: Rclone::Synchronize,
1381
+ equalize: Rclone::Equalize,
1382
+ backup: Restic::Backup,
1383
+ restore: Restic::Restore
1384
+ }
1385
+
1386
+
1387
+ class Endpoint
1388
+
1389
+
1390
+ def self.restore(hash)
1391
+ endpoint = allocate
1392
+ endpoint.send(:restore, hash)
1393
+ endpoint
1394
+ end
1395
+
1396
+
1397
+ class Local < Endpoint
1398
+
1399
+
1400
+ attr_reader :root
1401
+
1402
+
1403
+ def initialize(root) = @root = Pathname.new(root).realdirpath
1404
+
1405
+
1406
+ def restore(hash) = initialize(hash.fetch(:root))
1407
+
1408
+
1409
+ def externalize
1410
+ {
1411
+ endpoint: :local,
1412
+ root: root
1413
+ }
1414
+ end
1415
+
1416
+
1417
+ def show_status = root.to_s
1418
+
1419
+
1420
+ def intact? = true
1421
+
1422
+
1423
+ def refers?(volume) = false
1424
+
1425
+
1426
+ def generation = 0
1427
+
1428
+
1429
+ end
1430
+
1431
+
1432
+ class Rclone < Endpoint
1433
+ # TODO
1434
+ end
1435
+
1436
+
1437
+ class Bitferry < Endpoint
1438
+
1439
+
1440
+ attr_reader :volume_tag
1441
+
1442
+
1443
+ attr_reader :path
1444
+
1445
+
1446
+ def root = Volume[volume_tag].root.join(path)
1447
+
1448
+
1449
+ def initialize(volume, path)
1450
+ @volume_tag = volume.tag
1451
+ @path = Pathname.new(path)
1452
+ raise ArgumentError, "expected relative path but got #{self.path}" if (Volume::ENCOMPASSING_PATH_PREFIX =~ self.path.to_s).nil?
1453
+ end
1454
+
1455
+
1456
+ def restore(hash)
1457
+ @volume_tag = hash.fetch(:volume)
1458
+ @path = Pathname.new(hash.fetch(:path, ''))
1459
+ end
1460
+
1461
+
1462
+ def externalize
1463
+ {
1464
+ endpoint: :bitferry,
1465
+ volume: volume_tag,
1466
+ path: path.to_s.empty? ? nil : path
1467
+ }.compact
1468
+ end
1469
+
1470
+
1471
+ def show_status = intact? ? ":#{volume_tag}:#{path}" : ":{#{volume_tag}}:#{path}"
1472
+
1473
+
1474
+ def intact? = !Volume[volume_tag].nil?
1475
+
1476
+
1477
+ def refers?(volume) = volume.tag == volume_tag
1478
+
1479
+
1480
+ def generation
1481
+ v = Volume[volume_tag]
1482
+ v ? v.generation : 0
1483
+ end
1484
+
1485
+
1486
+ end
1487
+
1488
+
1489
+ ROUTE = {
1490
+ local: Local,
1491
+ rclone: Rclone,
1492
+ bitferry: Bitferry
1493
+ }
1494
+
1495
+
1496
+ end
1497
+
1498
+
1499
+ reset
1500
+
1501
+
1502
+ end