bitferry 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +39 -28
- data/README.md +231 -231
- data/bin/bitferry +2 -2
- data/bin/bitferryfx +2 -2
- data/lib/bitferry/cli.rb +376 -375
- data/lib/bitferry/fx.rb +131 -63
- data/lib/bitferry.rb +1502 -1424
- metadata +38 -14
data/lib/bitferry.rb
CHANGED
@@ -1,1424 +1,1502 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'date'
|
3
|
-
require 'open3'
|
4
|
-
require '
|
5
|
-
require '
|
6
|
-
require '
|
7
|
-
require '
|
8
|
-
require '
|
9
|
-
require '
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
@log
|
23
|
-
@log.
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
roots
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
log.
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
log.
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
def self.
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
process =
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
r
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
#
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
def self.
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
end
|
269
|
-
|
270
|
-
|
271
|
-
def self.
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
@state = :
|
325
|
-
@modified =
|
326
|
-
end
|
327
|
-
|
328
|
-
|
329
|
-
def
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
end
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
def
|
403
|
-
tasks.
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
FileUtils.
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
@state = :
|
594
|
-
log.info("
|
595
|
-
end
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
def
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
def
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
def self.
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
end
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
784
|
-
|
785
|
-
def
|
786
|
-
|
787
|
-
|
788
|
-
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
|
815
|
-
|
816
|
-
|
817
|
-
|
818
|
-
|
819
|
-
|
820
|
-
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
|
857
|
-
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
|
881
|
-
|
882
|
-
|
883
|
-
|
884
|
-
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
|
921
|
-
|
922
|
-
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
|
941
|
-
|
942
|
-
|
943
|
-
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
|
951
|
-
|
952
|
-
|
953
|
-
|
954
|
-
|
955
|
-
|
956
|
-
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
|
961
|
-
|
962
|
-
|
963
|
-
|
964
|
-
def
|
965
|
-
|
966
|
-
|
967
|
-
|
968
|
-
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
|
973
|
-
|
974
|
-
|
975
|
-
|
976
|
-
|
977
|
-
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
983
|
-
|
984
|
-
|
985
|
-
|
986
|
-
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
|
996
|
-
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
|
1001
|
-
|
1002
|
-
|
1003
|
-
|
1004
|
-
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1008
|
-
|
1009
|
-
|
1010
|
-
|
1011
|
-
|
1012
|
-
|
1013
|
-
|
1014
|
-
|
1015
|
-
|
1016
|
-
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1022
|
-
|
1023
|
-
|
1024
|
-
|
1025
|
-
|
1026
|
-
|
1027
|
-
|
1028
|
-
|
1029
|
-
|
1030
|
-
|
1031
|
-
|
1032
|
-
|
1033
|
-
|
1034
|
-
|
1035
|
-
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1039
|
-
|
1040
|
-
|
1041
|
-
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
|
1046
|
-
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1050
|
-
|
1051
|
-
|
1052
|
-
|
1053
|
-
|
1054
|
-
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1059
|
-
|
1060
|
-
|
1061
|
-
|
1062
|
-
|
1063
|
-
|
1064
|
-
|
1065
|
-
|
1066
|
-
|
1067
|
-
|
1068
|
-
|
1069
|
-
|
1070
|
-
|
1071
|
-
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
1084
|
-
|
1085
|
-
|
1086
|
-
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
1093
|
-
|
1094
|
-
|
1095
|
-
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
super
|
1109
|
-
end
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1165
|
-
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1181
|
-
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1188
|
-
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1192
|
-
|
1193
|
-
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
1197
|
-
|
1198
|
-
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1216
|
-
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
|
1231
|
-
|
1232
|
-
|
1233
|
-
|
1234
|
-
|
1235
|
-
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1245
|
-
|
1246
|
-
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1251
|
-
|
1252
|
-
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1265
|
-
|
1266
|
-
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1270
|
-
|
1271
|
-
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1275
|
-
|
1276
|
-
@
|
1277
|
-
end
|
1278
|
-
|
1279
|
-
|
1280
|
-
def
|
1281
|
-
|
1282
|
-
|
1283
|
-
|
1284
|
-
|
1285
|
-
true
|
1286
|
-
|
1287
|
-
|
1288
|
-
|
1289
|
-
|
1290
|
-
|
1291
|
-
|
1292
|
-
|
1293
|
-
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
1301
|
-
|
1302
|
-
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1308
|
-
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1313
|
-
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1317
|
-
|
1318
|
-
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
|
1323
|
-
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1329
|
-
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1343
|
-
|
1344
|
-
|
1345
|
-
|
1346
|
-
|
1347
|
-
|
1348
|
-
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1353
|
-
|
1354
|
-
|
1355
|
-
|
1356
|
-
|
1357
|
-
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1365
|
-
|
1366
|
-
|
1367
|
-
|
1368
|
-
|
1369
|
-
|
1370
|
-
|
1371
|
-
|
1372
|
-
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1376
|
-
|
1377
|
-
|
1378
|
-
|
1379
|
-
|
1380
|
-
|
1381
|
-
|
1382
|
-
|
1383
|
-
|
1384
|
-
|
1385
|
-
|
1386
|
-
|
1387
|
-
|
1388
|
-
|
1389
|
-
|
1390
|
-
|
1391
|
-
|
1392
|
-
|
1393
|
-
|
1394
|
-
|
1395
|
-
|
1396
|
-
|
1397
|
-
|
1398
|
-
|
1399
|
-
|
1400
|
-
|
1401
|
-
|
1402
|
-
|
1403
|
-
|
1404
|
-
|
1405
|
-
|
1406
|
-
|
1407
|
-
|
1408
|
-
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1419
|
-
|
1420
|
-
|
1421
|
-
|
1422
|
-
|
1423
|
-
|
1424
|
-
|
1
|
+
require 'json'
|
2
|
+
require 'date'
|
3
|
+
require 'open3'
|
4
|
+
require 'base64'
|
5
|
+
require 'logger'
|
6
|
+
require 'openssl'
|
7
|
+
require 'pathname'
|
8
|
+
require 'rbconfig'
|
9
|
+
require 'fileutils'
|
10
|
+
require 'shellwords'
|
11
|
+
|
12
|
+
|
13
|
+
module Bitferry
|
14
|
+
|
15
|
+
|
16
|
+
VERSION = '0.0.8'
|
17
|
+
|
18
|
+
|
19
|
+
module Logging
|
20
|
+
def self.log
|
21
|
+
unless @log
|
22
|
+
@log = Logger.new($stderr)
|
23
|
+
@log.level = Logger::WARN
|
24
|
+
@log.progname = :bitferry
|
25
|
+
end
|
26
|
+
@log
|
27
|
+
end
|
28
|
+
def self.log=(log) @log = log end
|
29
|
+
def log = Logging.log
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
include Logging
|
34
|
+
extend Logging
|
35
|
+
|
36
|
+
|
37
|
+
def self.tag = format('%08x', 2**32*rand)
|
38
|
+
|
39
|
+
|
40
|
+
def self.restore
|
41
|
+
reset
|
42
|
+
log.info('restoring volumes')
|
43
|
+
result = true
|
44
|
+
roots = (environment_mounts + system_mounts + [Dir.home]).uniq
|
45
|
+
log.info("distilled volume search path: #{roots.join(', ')}")
|
46
|
+
roots.each do |root|
|
47
|
+
if File.exist?(File.join(root, Volume::STORAGE))
|
48
|
+
log.info("trying to restore volume from #{root}")
|
49
|
+
Volume.restore(root) rescue result = false
|
50
|
+
end
|
51
|
+
end
|
52
|
+
if result
|
53
|
+
log.info('volumes restored')
|
54
|
+
else
|
55
|
+
log.warn('volume restore failure(s) reported')
|
56
|
+
end
|
57
|
+
result
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
def self.commit
|
62
|
+
log.info('committing changes')
|
63
|
+
result = true
|
64
|
+
modified = false
|
65
|
+
Volume.registered.each do |volume|
|
66
|
+
begin
|
67
|
+
modified = true if volume.modified?
|
68
|
+
volume.commit
|
69
|
+
rescue IOError => e
|
70
|
+
log.error(e.message)
|
71
|
+
result = false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
if result
|
75
|
+
log.info(modified ? 'changes committed' : 'commits skipped (no changes)')
|
76
|
+
else
|
77
|
+
log.warn('commit failure(s) reported')
|
78
|
+
end
|
79
|
+
result
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
def self.reset
|
84
|
+
log.info('resetting state')
|
85
|
+
Volume.reset
|
86
|
+
Task.reset
|
87
|
+
end
|
88
|
+
|
89
|
+
|
90
|
+
def self.intact_tasks = Volume.intact.collect { |volume| volume.intact_tasks }.flatten.uniq
|
91
|
+
|
92
|
+
def self.process(*tags, &block)
|
93
|
+
log.info('processing tasks')
|
94
|
+
tasks = intact_tasks
|
95
|
+
if tags.empty?
|
96
|
+
process = tasks
|
97
|
+
else
|
98
|
+
process = []
|
99
|
+
tags.each do |tag|
|
100
|
+
case (tasks = Task.match([tag], tasks)).size
|
101
|
+
when 0 then log.warn("no tasks matching (partial) tag #{tag}")
|
102
|
+
when 1 then process += tasks
|
103
|
+
else
|
104
|
+
tags = tasks.collect { |v| v.tag }.join(', ')
|
105
|
+
raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
tasks = process.uniq
|
110
|
+
total = tasks.size
|
111
|
+
processed = 0
|
112
|
+
failed = 0
|
113
|
+
result = tasks.all? do |task|
|
114
|
+
r = task.process
|
115
|
+
processed += 1
|
116
|
+
failed += 1 unless r
|
117
|
+
yield(total, processed, failed) if block_given?
|
118
|
+
r
|
119
|
+
end
|
120
|
+
result ? log.info('tasks processed') : log.warn('task process failure(s) reported')
|
121
|
+
result
|
122
|
+
end
|
123
|
+
|
124
|
+
|
125
|
+
def self.endpoint(root)
|
126
|
+
case root
|
127
|
+
when /^:(\w+):(.*)/
|
128
|
+
volumes = Volume.lookup($1)
|
129
|
+
volume = case volumes.size
|
130
|
+
when 0 then raise ArgumentError, "no intact volume matching (partial) tag #{$1}"
|
131
|
+
when 1 then volumes.first
|
132
|
+
else
|
133
|
+
tags = volumes.collect { |v| v.tag }.join(', ')
|
134
|
+
raise ArgumentError, "multiple intact volumes matching (partial) tag #{$1}: #{tags}"
|
135
|
+
end
|
136
|
+
Endpoint::Bitferry.new(volume, $2)
|
137
|
+
when /^(?:local)?:(.*)/ then Endpoint::Local.new($1)
|
138
|
+
when /^(\w{2,}):(.*)/ then Endpoint::Rclone.new($1, $2)
|
139
|
+
else Volume.endpoint(root)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
|
144
|
+
@simulate = false
|
145
|
+
def self.simulate? = @simulate
|
146
|
+
def self.simulate=(mode) @simulate = mode end
|
147
|
+
|
148
|
+
|
149
|
+
@verbosity = :default
|
150
|
+
def self.verbosity = @verbosity
|
151
|
+
def self.verbosity=(mode) @verbosity = mode end
|
152
|
+
|
153
|
+
|
154
|
+
@ui = :cli
|
155
|
+
def self.ui = @ui
|
156
|
+
def self.ui=(ui) @ui = ui end
|
157
|
+
|
158
|
+
|
159
|
+
# Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
|
160
|
+
def self.windows?
|
161
|
+
@windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
|
162
|
+
end
|
163
|
+
|
164
|
+
# Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
|
165
|
+
# Look for the $BITFERRY_PATH environment variable
|
166
|
+
def self.environment_mounts
|
167
|
+
ENV['BITFERRY_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
|
168
|
+
end
|
169
|
+
|
170
|
+
|
171
|
+
# Specify OS-specific path name list separator (such as in the $PATH environment variable)
|
172
|
+
PATH_LIST_SEPARATOR = windows? ? ';' : ':'
|
173
|
+
|
174
|
+
|
175
|
+
# Match OS-specific system mount points (/dev /proc etc.) which normally should be omitted when scanning for Bitferry voulmes
|
176
|
+
UNIX_SYSTEM_MOUNTS = %r!^/(dev|sys|proc|efi)!
|
177
|
+
|
178
|
+
|
179
|
+
# Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Bitferry volumes
|
180
|
+
if RUBY_PLATFORM =~ /java/
|
181
|
+
require 'java'
|
182
|
+
def self.system_mounts
|
183
|
+
java.nio.file.FileSystems.getDefault.getFileStores.collect {|x| /^(.*)\s+\(.*\)$/.match(x.to_s)[1]}
|
184
|
+
end
|
185
|
+
else
|
186
|
+
case RbConfig::CONFIG['target_os']
|
187
|
+
when 'linux'
|
188
|
+
# Linux OS
|
189
|
+
def self.system_mounts
|
190
|
+
# Query /proc for currently mounted file systems
|
191
|
+
IO.readlines('/proc/mounts').collect do |line|
|
192
|
+
mount = line.split[1]
|
193
|
+
UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
|
194
|
+
end.compact
|
195
|
+
end
|
196
|
+
# TODO handle Windows variants
|
197
|
+
when /^mingw/ # RubyInstaller's MRI
|
198
|
+
module Kernel32
|
199
|
+
require 'fiddle'
|
200
|
+
require 'fiddle/types'
|
201
|
+
require 'fiddle/import'
|
202
|
+
extend Fiddle::Importer
|
203
|
+
dlload('kernel32')
|
204
|
+
include Fiddle::Win32Types
|
205
|
+
extern 'DWORD WINAPI GetLogicalDrives()'
|
206
|
+
end
|
207
|
+
def self.system_mounts
|
208
|
+
mounts = []
|
209
|
+
mask = Kernel32.GetLogicalDrives
|
210
|
+
('A'..'Z').each do |x|
|
211
|
+
mounts << "#{x}:/" if mask & 1 == 1
|
212
|
+
mask >>= 1
|
213
|
+
end
|
214
|
+
mounts
|
215
|
+
end
|
216
|
+
else
|
217
|
+
# Generic *NIX-like OS, including Cygwin & MSYS2
|
218
|
+
def self.system_mounts
|
219
|
+
# Use $(mount) system utility to obtain currently mounted file systems
|
220
|
+
%x(mount).split("\n").collect do |line|
|
221
|
+
mount = line.split[2]
|
222
|
+
UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount
|
223
|
+
end.compact
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
|
229
|
+
class Volume
|
230
|
+
|
231
|
+
|
232
|
+
include Logging
|
233
|
+
extend Logging
|
234
|
+
|
235
|
+
|
236
|
+
STORAGE = '.bitferry'
|
237
|
+
STORAGE_ = '.bitferry~'
|
238
|
+
STORAGE_MASK = '.bitferry*'
|
239
|
+
|
240
|
+
|
241
|
+
attr_reader :tag
|
242
|
+
|
243
|
+
|
244
|
+
attr_reader :generation
|
245
|
+
|
246
|
+
|
247
|
+
attr_reader :root
|
248
|
+
|
249
|
+
|
250
|
+
attr_reader :vault
|
251
|
+
|
252
|
+
|
253
|
+
def self.[](tag)
|
254
|
+
@@registry.each_value { |volume| return volume if volume.tag == tag }
|
255
|
+
nil
|
256
|
+
end
|
257
|
+
|
258
|
+
|
259
|
+
# Return list of registered volumes whose tags match at least one specified partial
|
260
|
+
def self.lookup(*tags) = match(tags, registered)
|
261
|
+
|
262
|
+
|
263
|
+
def self.match(tags, volumes)
|
264
|
+
rxs = tags.collect { |x| Regexp.new(x) }
|
265
|
+
volumes.filter do |volume|
|
266
|
+
rxs.any? { |rx| !(rx =~ volume.tag).nil? }
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
|
271
|
+
def self.new(root, **opts)
|
272
|
+
volume = allocate
|
273
|
+
volume.send(:create, root, **opts)
|
274
|
+
register(volume)
|
275
|
+
end
|
276
|
+
|
277
|
+
|
278
|
+
def self.restore(root)
|
279
|
+
begin
|
280
|
+
volume = allocate
|
281
|
+
volume.send(:restore, root)
|
282
|
+
volume = register(volume)
|
283
|
+
log.info("restored volume #{volume.tag} from #{root}")
|
284
|
+
volume
|
285
|
+
rescue => e
|
286
|
+
log.error("failed to restore volume from #{root}")
|
287
|
+
log.error(e.message) if $DEBUG
|
288
|
+
raise
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
|
293
|
+
def self.delete(*tags, wipe: false)
|
294
|
+
process = []
|
295
|
+
tags.each do |tag|
|
296
|
+
case (volumes = Volume.lookup(tag)).size
|
297
|
+
when 0 then log.warn("no volumes matching (partial) tag #{tag}")
|
298
|
+
when 1 then process += volumes
|
299
|
+
else
|
300
|
+
tags = volumes.collect { |v| v.tag }.join(', ')
|
301
|
+
raise ArgumentError, "multiple volumes matching (partial) tag #{tag}: #{tags}"
|
302
|
+
end
|
303
|
+
end
|
304
|
+
process.each { |volume| volume.delete(wipe: wipe) }
|
305
|
+
end
|
306
|
+
|
307
|
+
|
308
|
+
def initialize(root, tag: Bitferry.tag, modified: nil, overwrite: false)
|
309
|
+
@tag = tag
|
310
|
+
@generation = 0
|
311
|
+
@vault = {}
|
312
|
+
@modified = case modified
|
313
|
+
when nil then DateTime.now
|
314
|
+
when DateTime then modified
|
315
|
+
else DateTime.parse(modified)
|
316
|
+
end
|
317
|
+
@overwrite = overwrite
|
318
|
+
@root = Pathname.new(root).realdirpath
|
319
|
+
end
|
320
|
+
|
321
|
+
|
322
|
+
def create(*args, **opts)
|
323
|
+
initialize(*args, **opts)
|
324
|
+
@state = :pristine
|
325
|
+
@modified = true
|
326
|
+
end
|
327
|
+
|
328
|
+
|
329
|
+
def restore(root)
|
330
|
+
hash = JSON.load_file(storage = Pathname(root).join(STORAGE), { symbolize_names: true })
|
331
|
+
raise IOError, "bad volume storage #{storage}" unless hash.fetch(:bitferry) == "0"
|
332
|
+
volume = hash.fetch(:volume)
|
333
|
+
begin
|
334
|
+
modified = DateTime.parse(hash.fetch(:modified))
|
335
|
+
@modified = false
|
336
|
+
rescue
|
337
|
+
modified = nil
|
338
|
+
@modified = true
|
339
|
+
log.warn("modified key missing - flagging volume #{volume} out of date")
|
340
|
+
end
|
341
|
+
initialize(root, tag: volume, modified: modified)
|
342
|
+
hash.fetch(:tasks, []).each { |hash| Task::ROUTE.fetch(hash.fetch(:operation).intern).restore(hash) }
|
343
|
+
@vault = hash.fetch(:vault, {}).transform_keys { |key| key.to_s }
|
344
|
+
@state = :intact
|
345
|
+
end
|
346
|
+
|
347
|
+
|
348
|
+
def storage = @storage ||= root.join(STORAGE)
|
349
|
+
def storage_ = @storage_ ||= root.join(STORAGE_)
|
350
|
+
|
351
|
+
|
352
|
+
def commit
|
353
|
+
if modified?
|
354
|
+
log.info("commit volume #{tag} (modified)")
|
355
|
+
case @state
|
356
|
+
when :pristine
|
357
|
+
format
|
358
|
+
store
|
359
|
+
when :intact
|
360
|
+
store
|
361
|
+
when :removing
|
362
|
+
remove
|
363
|
+
else
|
364
|
+
raise
|
365
|
+
end
|
366
|
+
committed
|
367
|
+
else
|
368
|
+
log.info("skipped committing volume #{tag} (unmodified)")
|
369
|
+
end
|
370
|
+
end
|
371
|
+
|
372
|
+
|
373
|
+
ENCOMPASSING_PATH_PREFIX = /^(?!(\.\.$|\.\.\/))/ # not .. by itself and not starting with ../
|
374
|
+
|
375
|
+
|
376
|
+
def self.endpoint(root)
|
377
|
+
path = Pathname.new(root).realdirpath
|
378
|
+
intact.sort { |v1, v2| v2.root.to_s.size <=> v1.root.to_s.size }.each do |volume|
|
379
|
+
begin
|
380
|
+
stem = path.relative_path_from(volume.root).to_s #.chomp('/')
|
381
|
+
case stem
|
382
|
+
when '.' then return volume.endpoint
|
383
|
+
when ENCOMPASSING_PATH_PREFIX then return volume.endpoint(stem)
|
384
|
+
end
|
385
|
+
rescue ArgumentError
|
386
|
+
# Catch different prefix error on Windows
|
387
|
+
end
|
388
|
+
end
|
389
|
+
raise ArgumentError, "no intact volume encompasses path #{root}"
|
390
|
+
end
|
391
|
+
|
392
|
+
|
393
|
+
def endpoint(path = String.new) = Endpoint::Bitferry.new(self, path)
|
394
|
+
|
395
|
+
|
396
|
+
def modified? = @modified || tasks.any? { |t| t.generation > generation }
|
397
|
+
|
398
|
+
|
399
|
+
def intact? = @state != :removing
|
400
|
+
|
401
|
+
|
402
|
+
def touch
|
403
|
+
x = tasks.collect { |t| t.generation }.max
|
404
|
+
@generation = x ? x + 1 : 0
|
405
|
+
@modified = true
|
406
|
+
end
|
407
|
+
|
408
|
+
|
409
|
+
def delete(wipe: false)
|
410
|
+
touch
|
411
|
+
@wipe = wipe
|
412
|
+
@state = :removing
|
413
|
+
log.info("marked volume #{tag} for deletion")
|
414
|
+
end
|
415
|
+
|
416
|
+
|
417
|
+
def committed
|
418
|
+
x = tasks.collect { |t| t.generation }.min
|
419
|
+
@generation = x ? x : 0
|
420
|
+
@modified = false
|
421
|
+
end
|
422
|
+
|
423
|
+
|
424
|
+
def store
|
425
|
+
require 'neatjson'
|
426
|
+
tasks.each(&:commit)
|
427
|
+
hash = JSON.neat_generate(externalize, short: false, wrap: 200, afterColon: 1, afterComma: 1)
|
428
|
+
if Bitferry.simulate?
|
429
|
+
log.info("skipped volume #{tag} storage modification (simulation)")
|
430
|
+
else
|
431
|
+
begin
|
432
|
+
File.write(storage_, hash)
|
433
|
+
FileUtils.mv(storage_, storage)
|
434
|
+
log.info("written volume #{tag} storage #{storage}")
|
435
|
+
ensure
|
436
|
+
FileUtils.rm_f(storage_)
|
437
|
+
end
|
438
|
+
end
|
439
|
+
@state = :intact
|
440
|
+
end
|
441
|
+
|
442
|
+
|
443
|
+
def format
|
444
|
+
raise IOError, "refuse to overwrite existing volume storage #{storage}" if !@overwrite && File.exist?(storage)
|
445
|
+
if Bitferry.simulate?
|
446
|
+
log.info("skipped storage formatting (simulation)")
|
447
|
+
else
|
448
|
+
FileUtils.mkdir_p(root)
|
449
|
+
FileUtils.rm_f [storage, storage_]
|
450
|
+
log.info("formatted volume #{tag} in #{root}")
|
451
|
+
end
|
452
|
+
@state = nil
|
453
|
+
end
|
454
|
+
|
455
|
+
|
456
|
+
def remove
|
457
|
+
unless Bitferry.simulate?
|
458
|
+
if @wipe
|
459
|
+
FileUtils.rm_rf(Dir[File.join(root, '*'), File.join(root, '.*')])
|
460
|
+
log.info("wiped entire volume directory #{root}")
|
461
|
+
else
|
462
|
+
FileUtils.rm_f [storage, storage_]
|
463
|
+
log.info("deleted volume #{tag} storage files #{File.join(root, STORAGE_MASK)}")
|
464
|
+
end
|
465
|
+
end
|
466
|
+
@@registry.delete(root)
|
467
|
+
@state = nil
|
468
|
+
end
|
469
|
+
|
470
|
+
|
471
|
+
def externalize
|
472
|
+
tasks = live_tasks
|
473
|
+
v = vault.filter { |t| !Task[t].nil? && Task[t].live? } # Purge entries from non-existing (deleted) tasks
|
474
|
+
{
|
475
|
+
bitferry: "0",
|
476
|
+
volume: tag,
|
477
|
+
modified: (@modified = DateTime.now),
|
478
|
+
tasks: tasks.empty? ? nil : tasks.collect(&:externalize),
|
479
|
+
vault: v.empty? ? nil : v
|
480
|
+
}.compact
|
481
|
+
end
|
482
|
+
|
483
|
+
|
484
|
+
def tasks = Task.registered.filter { |task| task.refers?(self) }
|
485
|
+
|
486
|
+
|
487
|
+
def live_tasks = Task.live.filter { |task| task.refers?(self) }
|
488
|
+
|
489
|
+
|
490
|
+
def intact_tasks = live_tasks.filter { |task| task.intact? }
|
491
|
+
|
492
|
+
|
493
|
+
def self.reset = @@registry = {}
|
494
|
+
|
495
|
+
|
496
|
+
def self.register(volume) = @@registry[volume.root] = volume
|
497
|
+
|
498
|
+
|
499
|
+
def self.registered = @@registry.values
|
500
|
+
|
501
|
+
|
502
|
+
def self.intact = registered.filter { |volume| volume.intact? }
|
503
|
+
|
504
|
+
|
505
|
+
end
|
506
|
+
|
507
|
+
|
508
|
+
def self.optional(option, route)
|
509
|
+
case option
|
510
|
+
when Array then option # Array is passed verbatim
|
511
|
+
when '-' then nil # Disable adding any options with -
|
512
|
+
when /^-/ then option.split(',') # Split comma-separated string into array --foo,bar --> [--foo, bar]
|
513
|
+
else route.fetch(option.nil? ? nil : option.to_sym) # Obtain options from the profile database
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
517
|
+
|
518
|
+
class Task
|
519
|
+
|
520
|
+
|
521
|
+
include Logging
|
522
|
+
extend Logging
|
523
|
+
|
524
|
+
|
525
|
+
attr_reader :tag
|
526
|
+
|
527
|
+
|
528
|
+
attr_reader :generation
|
529
|
+
|
530
|
+
|
531
|
+
attr_reader :modified
|
532
|
+
|
533
|
+
|
534
|
+
attr_reader :include, :exclude
|
535
|
+
|
536
|
+
|
537
|
+
def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
|
538
|
+
|
539
|
+
|
540
|
+
def self.new(*args, **opts)
|
541
|
+
task = allocate
|
542
|
+
task.send(:create, *args, **opts)
|
543
|
+
register(task)
|
544
|
+
end
|
545
|
+
|
546
|
+
|
547
|
+
def self.restore(hash)
|
548
|
+
task = allocate
|
549
|
+
task.send(:restore, hash)
|
550
|
+
register(task)
|
551
|
+
end
|
552
|
+
|
553
|
+
|
554
|
+
def self.delete(*tags)
|
555
|
+
process = []
|
556
|
+
tags.each do |tag|
|
557
|
+
case (tasks = Task.lookup(tag)).size
|
558
|
+
when 0 then log.warn("no tasks matching (partial) tag #{tag}")
|
559
|
+
when 1 then process += tasks
|
560
|
+
else
|
561
|
+
tags = tasks.collect { |v| v.tag }.join(', ')
|
562
|
+
raise ArgumentError, "multiple tasks matching (partial) tag #{tag}: #{tags}"
|
563
|
+
end
|
564
|
+
end
|
565
|
+
process.each { |task| task.delete }
|
566
|
+
end
|
567
|
+
|
568
|
+
|
569
|
+
def initialize(tag: Bitferry.tag, modified: nil, include: [], exclude: [])
|
570
|
+
@tag = tag
|
571
|
+
@generation = 0
|
572
|
+
@include = include
|
573
|
+
@exclude = exclude
|
574
|
+
@modified = case modified
|
575
|
+
when nil then DateTime.now
|
576
|
+
when DateTime then modified
|
577
|
+
else DateTime.parse(modified)
|
578
|
+
end
|
579
|
+
# FIXME handle process_options at this level
|
580
|
+
end
|
581
|
+
|
582
|
+
|
583
|
+
def create(*args, **opts)
|
584
|
+
initialize(*args, **opts)
|
585
|
+
@state = :pristine
|
586
|
+
touch
|
587
|
+
end
|
588
|
+
|
589
|
+
|
590
|
+
def restore(hash)
|
591
|
+
@include = hash.fetch(:include, [])
|
592
|
+
@exclude = hash.fetch(:exclude, [])
|
593
|
+
@state = :intact
|
594
|
+
log.info("restored task #{tag}")
|
595
|
+
end
|
596
|
+
|
597
|
+
|
598
|
+
# FIXME move to Endpoint#restore
|
599
|
+
def restore_endpoint(x) = Endpoint::ROUTE.fetch(x.fetch(:endpoint).intern).restore(x)
|
600
|
+
|
601
|
+
|
602
|
+
def externalize
|
603
|
+
{
|
604
|
+
task: tag,
|
605
|
+
modified: modified,
|
606
|
+
include: include.empty? ? nil : include,
|
607
|
+
exclude: exclude.empty? ? nil : exclude
|
608
|
+
}.compact
|
609
|
+
end
|
610
|
+
|
611
|
+
|
612
|
+
def live? = !@state.nil? && @state != :removing
|
613
|
+
|
614
|
+
|
615
|
+
def touch = @modified = DateTime.now
|
616
|
+
|
617
|
+
|
618
|
+
def delete
|
619
|
+
touch
|
620
|
+
@state = :removing
|
621
|
+
log.info("marked task #{tag} for removal")
|
622
|
+
end
|
623
|
+
|
624
|
+
|
625
|
+
def commit
|
626
|
+
case @state
|
627
|
+
when :pristine then format
|
628
|
+
when :removing then @state = nil
|
629
|
+
end
|
630
|
+
end
|
631
|
+
|
632
|
+
|
633
|
+
def show_filters
|
634
|
+
xs = []
|
635
|
+
xs << 'include: ' + include.join(',') unless include.empty?
|
636
|
+
xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
|
637
|
+
xs.join(' ').to_s
|
638
|
+
end
|
639
|
+
|
640
|
+
|
641
|
+
def self.[](tag) = @@registry[tag]
|
642
|
+
|
643
|
+
|
644
|
+
# Return list of registered tasks whose tags match at least one of specified partial tags
|
645
|
+
def self.lookup(*tags) = match(tags, registered)
|
646
|
+
|
647
|
+
|
648
|
+
# Return list of specified tasks whose tags match at least one of specified partial tags
|
649
|
+
def self.match(tags, tasks)
|
650
|
+
rxs = tags.collect { |x| Regexp.new(x) }
|
651
|
+
tasks.filter do |task|
|
652
|
+
rxs.any? { |rx| !(rx =~ task.tag).nil? }
|
653
|
+
end
|
654
|
+
end
|
655
|
+
|
656
|
+
|
657
|
+
def self.registered = @@registry.values
|
658
|
+
|
659
|
+
|
660
|
+
def self.live = registered.filter { |task| task.live? }
|
661
|
+
|
662
|
+
|
663
|
+
def self.reset = @@registry = {}
|
664
|
+
|
665
|
+
|
666
|
+
def self.register(task)
|
667
|
+
# Task with newer timestamp replaces already registered task, if any
|
668
|
+
if (xtag = @@registry[task.tag]).nil?
|
669
|
+
@@registry[task.tag] = task
|
670
|
+
elsif xtag.modified < task.modified
|
671
|
+
@@registry[task.tag] = task
|
672
|
+
else
|
673
|
+
xtag
|
674
|
+
end
|
675
|
+
end
|
676
|
+
|
677
|
+
def self.intact = live.filter { |task| task.intact? }
|
678
|
+
|
679
|
+
|
680
|
+
def self.stale = live.filter { |task| !task.intact? }
|
681
|
+
|
682
|
+
|
683
|
+
end
|
684
|
+
|
685
|
+
|
686
|
+
module Rclone
|
687
|
+
|
688
|
+
|
689
|
+
include Logging
|
690
|
+
extend Logging
|
691
|
+
|
692
|
+
|
693
|
+
def self.executable = @executable ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
|
694
|
+
|
695
|
+
|
696
|
+
def self.exec(*args)
|
697
|
+
cmd = [executable] + args
|
698
|
+
log.debug(cmd.collect(&:shellescape).join(' '))
|
699
|
+
stdout, status = Open3.capture2e(*cmd)
|
700
|
+
unless status.success?
|
701
|
+
msg = "rclone exit code #{status.exitstatus}"
|
702
|
+
log.error(msg)
|
703
|
+
raise RuntimeError, msg
|
704
|
+
end
|
705
|
+
stdout.strip
|
706
|
+
end
|
707
|
+
|
708
|
+
|
709
|
+
# https://github.com/rclone/rclone/blob/master/fs/config/obscure/obscure.go
|
710
|
+
SECRET = "\x9c\x93\x5b\x48\x73\x0a\x55\x4d\x6b\xfd\x7c\x63\xc8\x86\xa9\x2b\xd3\x90\x19\x8e\xb8\x12\x8a\xfb\xf4\xde\x16\x2b\x8b\x95\xf6\x38"
|
711
|
+
|
712
|
+
|
713
|
+
def self.obscure(plain)
|
714
|
+
cipher = OpenSSL::Cipher.new('AES-256-CTR')
|
715
|
+
cipher.encrypt
|
716
|
+
cipher.key = SECRET
|
717
|
+
Base64.urlsafe_encode64(cipher.random_iv + cipher.update(plain) + cipher.final, padding: false)
|
718
|
+
end
|
719
|
+
|
720
|
+
|
721
|
+
def self.reveal(token)
|
722
|
+
data = Base64.urlsafe_decode64(token)
|
723
|
+
cipher = OpenSSL::Cipher.new('AES-256-CTR')
|
724
|
+
cipher.decrypt
|
725
|
+
cipher.key = SECRET
|
726
|
+
cipher.iv = data[0...cipher.iv_len]
|
727
|
+
cipher.update(data[cipher.iv_len..-1]) + cipher.final
|
728
|
+
end
|
729
|
+
|
730
|
+
|
731
|
+
class Encryption
|
732
|
+
|
733
|
+
|
734
|
+
PROCESS = {
|
735
|
+
default: ['--crypt-filename-encoding', :base32, '--crypt-filename-encryption', :standard],
|
736
|
+
extended: ['--crypt-filename-encoding', :base32768, '--crypt-filename-encryption', :standard]
|
737
|
+
}
|
738
|
+
PROCESS[nil] = PROCESS[:default]
|
739
|
+
|
740
|
+
|
741
|
+
def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
|
742
|
+
|
743
|
+
|
744
|
+
def initialize(token, process: nil)
|
745
|
+
@process_options = Bitferry.optional(process, PROCESS)
|
746
|
+
@token = token
|
747
|
+
end
|
748
|
+
|
749
|
+
|
750
|
+
def create(password, **opts) = initialize(Rclone.obscure(password), **opts)
|
751
|
+
|
752
|
+
|
753
|
+
def restore(hash) = @process_options = hash[:rclone]
|
754
|
+
|
755
|
+
|
756
|
+
def externalize = process_options.empty? ? {} : { rclone: process_options }
|
757
|
+
|
758
|
+
|
759
|
+
def configure(task) = install_token(task)
|
760
|
+
|
761
|
+
|
762
|
+
def process(task) = ENV['RCLONE_CRYPT_PASSWORD'] = obtain_token(task)
|
763
|
+
|
764
|
+
|
765
|
+
def arguments(task) = process_options + ['--crypt-remote', encrypted(task).root.to_s]
|
766
|
+
|
767
|
+
|
768
|
+
def install_token(task)
|
769
|
+
x = decrypted(task)
|
770
|
+
raise TypeError, 'unsupported unencrypted endpoint type' unless x.is_a?(Endpoint::Bitferry)
|
771
|
+
Volume[x.volume_tag].vault[task.tag] = @token # Token is stored on the decrypted end only
|
772
|
+
end
|
773
|
+
|
774
|
+
|
775
|
+
def obtain_token(task) = Volume[decrypted(task).volume_tag].vault.fetch(task.tag)
|
776
|
+
|
777
|
+
|
778
|
+
def self.new(*args, **opts)
|
779
|
+
obj = allocate
|
780
|
+
obj.send(:create, *args, **opts)
|
781
|
+
obj
|
782
|
+
end
|
783
|
+
|
784
|
+
|
785
|
+
def self.restore(hash)
|
786
|
+
obj = ROUTE.fetch(hash.fetch(:operation).intern).allocate
|
787
|
+
obj.send(:restore, hash)
|
788
|
+
obj
|
789
|
+
end
|
790
|
+
|
791
|
+
|
792
|
+
end
|
793
|
+
|
794
|
+
|
795
|
+
class Encrypt < Encryption
|
796
|
+
|
797
|
+
|
798
|
+
def encrypted(task) = task.destination
|
799
|
+
|
800
|
+
|
801
|
+
def decrypted(task) = task.source
|
802
|
+
|
803
|
+
|
804
|
+
def externalize = super.merge(operation: :encrypt)
|
805
|
+
|
806
|
+
|
807
|
+
def show_operation = 'encrypt+'
|
808
|
+
|
809
|
+
|
810
|
+
def arguments(task) = super + [decrypted(task).root.to_s, ':crypt:']
|
811
|
+
|
812
|
+
|
813
|
+
end
|
814
|
+
|
815
|
+
|
816
|
+
class Decrypt < Encryption
|
817
|
+
|
818
|
+
|
819
|
+
def encrypted(task) = task.source
|
820
|
+
|
821
|
+
|
822
|
+
def decrypted(task) = task.destination
|
823
|
+
|
824
|
+
|
825
|
+
def externalize = super.merge(operation: :decrypt)
|
826
|
+
|
827
|
+
|
828
|
+
def show_operation = 'decrypt+'
|
829
|
+
|
830
|
+
|
831
|
+
def arguments(task) = super + [':crypt:', decrypted(task).root.to_s]
|
832
|
+
|
833
|
+
|
834
|
+
end
|
835
|
+
|
836
|
+
|
837
|
+
ROUTE = {
|
838
|
+
encrypt: Encrypt,
|
839
|
+
decrypt: Decrypt
|
840
|
+
}
|
841
|
+
|
842
|
+
|
843
|
+
class Task < Bitferry::Task
|
844
|
+
|
845
|
+
|
846
|
+
attr_reader :source, :destination
|
847
|
+
|
848
|
+
|
849
|
+
attr_reader :encryption
|
850
|
+
|
851
|
+
|
852
|
+
attr_reader :token
|
853
|
+
|
854
|
+
|
855
|
+
PROCESS = {
|
856
|
+
default: ['--metadata']
|
857
|
+
}
|
858
|
+
PROCESS[nil] = PROCESS[:default]
|
859
|
+
|
860
|
+
|
861
|
+
def initialize(source, destination, encryption: nil, process: nil, **opts)
|
862
|
+
super(**opts)
|
863
|
+
@process_options = Bitferry.optional(process, PROCESS)
|
864
|
+
@source = source.is_a?(Endpoint) ? source : Bitferry.endpoint(source)
|
865
|
+
@destination = destination.is_a?(Endpoint) ? destination : Bitferry.endpoint(destination)
|
866
|
+
@encryption = encryption
|
867
|
+
end
|
868
|
+
|
869
|
+
|
870
|
+
def create(*args, process: nil, **opts)
|
871
|
+
super(*args, process: process, **opts)
|
872
|
+
encryption.configure(self) unless encryption.nil?
|
873
|
+
end
|
874
|
+
|
875
|
+
|
876
|
+
def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
|
877
|
+
|
878
|
+
|
879
|
+
def show_operation = encryption.nil? ? '' : encryption.show_operation
|
880
|
+
|
881
|
+
|
882
|
+
def show_direction = '-->'
|
883
|
+
|
884
|
+
|
885
|
+
def intact? = live? && source.intact? && destination.intact?
|
886
|
+
|
887
|
+
|
888
|
+
def refers?(volume) = source.refers?(volume) || destination.refers?(volume)
|
889
|
+
|
890
|
+
|
891
|
+
def touch
|
892
|
+
@generation = [source.generation, destination.generation].max + 1
|
893
|
+
super
|
894
|
+
end
|
895
|
+
|
896
|
+
|
897
|
+
def format = nil
|
898
|
+
|
899
|
+
|
900
|
+
def common_options
|
901
|
+
[
|
902
|
+
'--config', Bitferry.windows? ? 'NUL' : '/dev/null',
|
903
|
+
case Bitferry.verbosity
|
904
|
+
when :verbose then '--verbose'
|
905
|
+
when :quiet then '--quiet'
|
906
|
+
else nil
|
907
|
+
end,
|
908
|
+
Bitferry.verbosity == :verbose ? '--progress' : nil,
|
909
|
+
Bitferry.simulate? ? '--dry-run' : nil,
|
910
|
+
].compact
|
911
|
+
end
|
912
|
+
|
913
|
+
|
914
|
+
def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
|
915
|
+
|
916
|
+
|
917
|
+
def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
|
918
|
+
|
919
|
+
|
920
|
+
def process_arguments
|
921
|
+
include_filters + exclude_filters + common_options + process_options + (
|
922
|
+
encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
|
923
|
+
)
|
924
|
+
end
|
925
|
+
|
926
|
+
|
927
|
+
def execute(*args)
|
928
|
+
cmd = [Rclone.executable] + args
|
929
|
+
cms = cmd.collect(&:shellescape).join(' ')
|
930
|
+
$stdout.puts cms if Bitferry.verbosity == :verbose
|
931
|
+
log.info(cms)
|
932
|
+
if Bitferry.ui == :gui
|
933
|
+
t = nil
|
934
|
+
Open3.popen2e(*cmd) do |i, oe, thr|
|
935
|
+
while x = oe.gets; $stdout.puts(x) end
|
936
|
+
t = thr
|
937
|
+
end
|
938
|
+
status = t.value
|
939
|
+
else
|
940
|
+
status = Open3.pipeline(cmd).first
|
941
|
+
end
|
942
|
+
raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
|
943
|
+
status.success?
|
944
|
+
end
|
945
|
+
|
946
|
+
|
947
|
+
def process
|
948
|
+
log.info("processing task #{tag}")
|
949
|
+
encryption.process(self) unless encryption.nil?
|
950
|
+
execute(*process_arguments)
|
951
|
+
end
|
952
|
+
|
953
|
+
|
954
|
+
def externalize
|
955
|
+
super.merge(
|
956
|
+
source: source.externalize,
|
957
|
+
destination: destination.externalize,
|
958
|
+
encryption: encryption.nil? ? nil : encryption.externalize,
|
959
|
+
rclone: process_options.empty? ? nil : process_options
|
960
|
+
).compact
|
961
|
+
end
|
962
|
+
|
963
|
+
|
964
|
+
def restore(hash)
|
965
|
+
task = hash.fetch(:task)
|
966
|
+
begin
|
967
|
+
modified = hash.fetch(:modified)
|
968
|
+
rescue
|
969
|
+
modified = nil
|
970
|
+
log.warn("modified key missing - flagging task #{task} out of date")
|
971
|
+
end
|
972
|
+
initialize(
|
973
|
+
restore_endpoint(hash.fetch(:source)),
|
974
|
+
restore_endpoint(hash.fetch(:destination)),
|
975
|
+
tag: task,
|
976
|
+
modified: modified,
|
977
|
+
process: hash[:rclone],
|
978
|
+
encryption: hash[:encryption].nil? ? nil : Rclone::Encryption.restore(hash[:encryption])
|
979
|
+
)
|
980
|
+
super(hash)
|
981
|
+
touch if modified.nil?
|
982
|
+
end
|
983
|
+
|
984
|
+
|
985
|
+
end
|
986
|
+
|
987
|
+
|
988
|
+
class Copy < Task
|
989
|
+
|
990
|
+
|
991
|
+
def process_arguments = ['copy'] + super
|
992
|
+
|
993
|
+
|
994
|
+
def externalize = super.merge(operation: :copy)
|
995
|
+
|
996
|
+
|
997
|
+
def show_operation = super + 'copy'
|
998
|
+
|
999
|
+
|
1000
|
+
end
|
1001
|
+
|
1002
|
+
|
1003
|
+
class Update < Task
|
1004
|
+
|
1005
|
+
|
1006
|
+
def process_arguments = ['copy', '--update'] + super
|
1007
|
+
|
1008
|
+
|
1009
|
+
def externalize = super.merge(operation: :update)
|
1010
|
+
|
1011
|
+
|
1012
|
+
def show_operation = super + 'update'
|
1013
|
+
|
1014
|
+
|
1015
|
+
end
|
1016
|
+
|
1017
|
+
|
1018
|
+
class Synchronize < Task
|
1019
|
+
|
1020
|
+
|
1021
|
+
def process_arguments = ['sync'] + super
|
1022
|
+
|
1023
|
+
|
1024
|
+
def externalize = super.merge(operation: :synchronize)
|
1025
|
+
|
1026
|
+
|
1027
|
+
def show_operation = super + 'synchronize'
|
1028
|
+
|
1029
|
+
|
1030
|
+
end
|
1031
|
+
|
1032
|
+
|
1033
|
+
class Equalize < Task
|
1034
|
+
|
1035
|
+
|
1036
|
+
def process_arguments = ['bisync', '--resync'] + super
|
1037
|
+
|
1038
|
+
|
1039
|
+
def externalize = super.merge(operation: :equalize)
|
1040
|
+
|
1041
|
+
|
1042
|
+
def show_operation = super + 'equalize'
|
1043
|
+
|
1044
|
+
|
1045
|
+
def show_direction = '<->'
|
1046
|
+
|
1047
|
+
|
1048
|
+
end
|
1049
|
+
|
1050
|
+
|
1051
|
+
end
|
1052
|
+
|
1053
|
+
|
1054
|
+
module Restic
|
1055
|
+
|
1056
|
+
|
1057
|
+
include Logging
|
1058
|
+
extend Logging
|
1059
|
+
|
1060
|
+
|
1061
|
+
def self.executable = @executable ||= (restic = ENV['RESTIC']).nil? ? 'restic' : restic
|
1062
|
+
|
1063
|
+
|
1064
|
+
def self.exec(*args)
|
1065
|
+
cmd = [executable] + args
|
1066
|
+
log.debug(cmd.collect(&:shellescape).join(' '))
|
1067
|
+
stdout, status = Open3.capture2(*cmd)
|
1068
|
+
unless status.success?
|
1069
|
+
msg = "restic exit code #{status.to_i}"
|
1070
|
+
log.error(msg)
|
1071
|
+
raise RuntimeError, msg
|
1072
|
+
end
|
1073
|
+
stdout.strip
|
1074
|
+
end
|
1075
|
+
|
1076
|
+
|
1077
|
+
class Task < Bitferry::Task
|
1078
|
+
|
1079
|
+
|
1080
|
+
attr_reader :directory, :repository
|
1081
|
+
|
1082
|
+
|
1083
|
+
def initialize(directory, repository, **opts)
|
1084
|
+
super(**opts)
|
1085
|
+
@directory = directory.is_a?(Endpoint) ? directory : Bitferry.endpoint(directory)
|
1086
|
+
@repository = repository.is_a?(Endpoint) ? repository : Bitferry.endpoint(repository)
|
1087
|
+
end
|
1088
|
+
|
1089
|
+
|
1090
|
+
def create(directory, repository, password, **opts)
|
1091
|
+
super(directory, repository, **opts)
|
1092
|
+
raise TypeError, 'unsupported unencrypted endpoint type' unless self.directory.is_a?(Endpoint::Bitferry)
|
1093
|
+
Volume[self.directory.volume_tag].vault[tag] = Rclone.obscure(@password = password) # Token is stored on the decrypted end only
|
1094
|
+
end
|
1095
|
+
|
1096
|
+
|
1097
|
+
def password = @password ||= Rclone.reveal(Volume[directory.volume_tag].vault.fetch(tag))
|
1098
|
+
|
1099
|
+
|
1100
|
+
def intact? = live? && directory.intact? && repository.intact?
|
1101
|
+
|
1102
|
+
|
1103
|
+
def refers?(volume) = directory.refers?(volume) || repository.refers?(volume)
|
1104
|
+
|
1105
|
+
|
1106
|
+
def touch
|
1107
|
+
@generation = [directory.generation, repository.generation].max + 1
|
1108
|
+
super
|
1109
|
+
end
|
1110
|
+
|
1111
|
+
|
1112
|
+
def format = nil
|
1113
|
+
|
1114
|
+
|
1115
|
+
def include_filters = include.collect { |x| ['--include', x]}.flatten
|
1116
|
+
|
1117
|
+
|
1118
|
+
def common_options
|
1119
|
+
[
|
1120
|
+
case Bitferry.verbosity
|
1121
|
+
when :verbose then '--verbose'
|
1122
|
+
when :quiet then '--quiet'
|
1123
|
+
else nil
|
1124
|
+
end,
|
1125
|
+
'-r', repository.root.to_s
|
1126
|
+
].compact
|
1127
|
+
end
|
1128
|
+
|
1129
|
+
|
1130
|
+
def execute(*args, simulate: false, chdir: nil)
|
1131
|
+
cmd = [Restic.executable] + args
|
1132
|
+
ENV['RESTIC_PASSWORD'] = password
|
1133
|
+
ENV['RESTIC_PROGRESS_FPS'] = 1.to_s if Bitferry.verbosity == :verbose && Bitferry.ui == :gui
|
1134
|
+
cms = cmd.collect(&:shellescape).join(' ')
|
1135
|
+
$stdout.puts cms if Bitferry.verbosity == :verbose
|
1136
|
+
log.info(cms)
|
1137
|
+
if simulate
|
1138
|
+
log.info('(simulated)')
|
1139
|
+
true
|
1140
|
+
else
|
1141
|
+
wd = Dir.getwd unless chdir.nil?
|
1142
|
+
begin
|
1143
|
+
Dir.chdir(chdir) unless chdir.nil?
|
1144
|
+
if Bitferry.ui == :gui
|
1145
|
+
t = nil
|
1146
|
+
Open3.popen2e(*cmd) do |i, oe, thr|
|
1147
|
+
while x = oe.gets; $stdout.puts(x) end
|
1148
|
+
t = thr
|
1149
|
+
end
|
1150
|
+
status = t.value
|
1151
|
+
else
|
1152
|
+
status = Open3.pipeline(cmd).first
|
1153
|
+
end
|
1154
|
+
raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
|
1155
|
+
status.success?
|
1156
|
+
ensure
|
1157
|
+
Dir.chdir(wd) unless chdir.nil?
|
1158
|
+
end
|
1159
|
+
end
|
1160
|
+
end
|
1161
|
+
|
1162
|
+
|
1163
|
+
def externalize
|
1164
|
+
super.merge(
|
1165
|
+
directory: directory.externalize,
|
1166
|
+
repository: repository.externalize,
|
1167
|
+
).compact
|
1168
|
+
end
|
1169
|
+
|
1170
|
+
|
1171
|
+
def restore(hash)
|
1172
|
+
initialize(
|
1173
|
+
restore_endpoint(hash.fetch(:directory)),
|
1174
|
+
restore_endpoint(hash.fetch(:repository)),
|
1175
|
+
tag: hash.fetch(:task),
|
1176
|
+
modified: hash.fetch(:modified, DateTime.now)
|
1177
|
+
)
|
1178
|
+
super(hash)
|
1179
|
+
end
|
1180
|
+
|
1181
|
+
|
1182
|
+
end
|
1183
|
+
|
1184
|
+
|
1185
|
+
class Backup < Task
|
1186
|
+
|
1187
|
+
|
1188
|
+
PROCESS = {
|
1189
|
+
default: ['--no-cache']
|
1190
|
+
}
|
1191
|
+
PROCESS[nil] = PROCESS[:default]
|
1192
|
+
|
1193
|
+
|
1194
|
+
FORGET = {
|
1195
|
+
default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
|
1196
|
+
}
|
1197
|
+
FORGET[nil] = nil # Skip processing retention policy by default
|
1198
|
+
|
1199
|
+
|
1200
|
+
CHECK = {
|
1201
|
+
default: ['--no-cache'],
|
1202
|
+
full: ['--no-cache', '--read-data']
|
1203
|
+
}
|
1204
|
+
CHECK[nil] = nil # Skip integrity checking by default
|
1205
|
+
|
1206
|
+
|
1207
|
+
attr_reader :forget_options
|
1208
|
+
attr_reader :check_options
|
1209
|
+
|
1210
|
+
|
1211
|
+
def create(*args, format: nil, process: nil, forget: nil, check: nil, **opts)
|
1212
|
+
super(*args, **opts)
|
1213
|
+
@format = format
|
1214
|
+
@process_options = Bitferry.optional(process, PROCESS)
|
1215
|
+
@forget_options = Bitferry.optional(forget, FORGET)
|
1216
|
+
@check_options = Bitferry.optional(check, CHECK)
|
1217
|
+
end
|
1218
|
+
|
1219
|
+
|
1220
|
+
def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
|
1221
|
+
|
1222
|
+
|
1223
|
+
def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
|
1224
|
+
|
1225
|
+
|
1226
|
+
def show_operation = 'encrypt+backup'
|
1227
|
+
|
1228
|
+
|
1229
|
+
def show_direction = '-->'
|
1230
|
+
|
1231
|
+
|
1232
|
+
alias :source :directory
|
1233
|
+
alias :destination :repository
|
1234
|
+
|
1235
|
+
|
1236
|
+
def process
|
1237
|
+
begin
|
1238
|
+
log.info("processing task #{tag}")
|
1239
|
+
execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
|
1240
|
+
unless check_options.nil?
|
1241
|
+
log.info("checking repository in #{repository.root}")
|
1242
|
+
execute('check', *check_options, *common_options)
|
1243
|
+
end
|
1244
|
+
unless forget_options.nil?
|
1245
|
+
log.info("performing repository maintenance tasks in #{repository.root}")
|
1246
|
+
execute('forget', '--tag', "bitferry,#{tag}", *forget_options.collect(&:to_s), *common_options_simulate)
|
1247
|
+
end
|
1248
|
+
true
|
1249
|
+
rescue
|
1250
|
+
false
|
1251
|
+
end
|
1252
|
+
end
|
1253
|
+
|
1254
|
+
|
1255
|
+
def common_options_simulate = common_options + [Bitferry.simulate? ? '--dry-run' : nil].compact
|
1256
|
+
|
1257
|
+
|
1258
|
+
def externalize
|
1259
|
+
restic = {
|
1260
|
+
process: process_options,
|
1261
|
+
forget: forget_options,
|
1262
|
+
check: check_options
|
1263
|
+
}.compact
|
1264
|
+
super.merge({
|
1265
|
+
operation: :backup,
|
1266
|
+
restic: restic.empty? ? nil : restic
|
1267
|
+
}.compact)
|
1268
|
+
end
|
1269
|
+
|
1270
|
+
|
1271
|
+
def restore(hash)
|
1272
|
+
super
|
1273
|
+
opts = hash.fetch(:restic, {})
|
1274
|
+
@process_options = opts[:process]
|
1275
|
+
@forget_options = opts[:forget]
|
1276
|
+
@check_options = opts[:check]
|
1277
|
+
end
|
1278
|
+
|
1279
|
+
|
1280
|
+
def format
|
1281
|
+
if Bitferry.simulate?
|
1282
|
+
log.info('skipped repository initialization (simulation)')
|
1283
|
+
else
|
1284
|
+
log.info("initializing repository for task #{tag}")
|
1285
|
+
if @format == true
|
1286
|
+
log.debug("wiping repository in #{repository.root}")
|
1287
|
+
['config', 'data', 'index', 'keys', 'locks', 'snapshots'].each { |x| FileUtils.rm_rf(File.join(repository.root.to_s, x)) }
|
1288
|
+
end
|
1289
|
+
if @format == false
|
1290
|
+
# TODO validate existing repo
|
1291
|
+
log.info("attached to existing repository for task #{tag} in #{repository.root}")
|
1292
|
+
else
|
1293
|
+
begin
|
1294
|
+
execute(*common_options, 'init')
|
1295
|
+
log.info("initialized repository for task #{tag} in #{repository.root}")
|
1296
|
+
rescue
|
1297
|
+
log.fatal("failed to initialize repository for task #{tag} in #{repository.root}")
|
1298
|
+
raise
|
1299
|
+
end
|
1300
|
+
end
|
1301
|
+
end
|
1302
|
+
@state = :intact
|
1303
|
+
end
|
1304
|
+
|
1305
|
+
|
1306
|
+
end
|
1307
|
+
|
1308
|
+
|
1309
|
+
class Restore < Task
|
1310
|
+
|
1311
|
+
|
1312
|
+
PROCESS = {
|
1313
|
+
default: ['--no-cache', '--sparse']
|
1314
|
+
}
|
1315
|
+
PROCESS[nil] = PROCESS[:default]
|
1316
|
+
|
1317
|
+
|
1318
|
+
def create(*args, process: nil, **opts)
|
1319
|
+
super(*args, **opts)
|
1320
|
+
@process_options = Bitferry.optional(process, PROCESS)
|
1321
|
+
end
|
1322
|
+
|
1323
|
+
|
1324
|
+
def exclude_filters = exclude.collect { |x| ['--exclude', x] }.flatten
|
1325
|
+
|
1326
|
+
|
1327
|
+
def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
|
1328
|
+
|
1329
|
+
|
1330
|
+
def show_operation = 'decrypt+restore'
|
1331
|
+
|
1332
|
+
|
1333
|
+
def show_direction = '-->'
|
1334
|
+
|
1335
|
+
|
1336
|
+
alias :destination :directory
|
1337
|
+
alias :source :repository
|
1338
|
+
|
1339
|
+
|
1340
|
+
def externalize
|
1341
|
+
restic = {
|
1342
|
+
process: process_options
|
1343
|
+
}.compact
|
1344
|
+
super.merge({
|
1345
|
+
operation: :restore,
|
1346
|
+
restic: restic.empty? ? nil : restic
|
1347
|
+
}.compact)
|
1348
|
+
end
|
1349
|
+
|
1350
|
+
|
1351
|
+
def restore(hash)
|
1352
|
+
super
|
1353
|
+
opts = hash.fetch(:rclone, {})
|
1354
|
+
@process_options = opts[:process]
|
1355
|
+
end
|
1356
|
+
|
1357
|
+
|
1358
|
+
def process
|
1359
|
+
log.info("processing task #{tag}")
|
1360
|
+
begin
|
1361
|
+
# FIXME restore specifically tagged latest snapshot
|
1362
|
+
execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
|
1363
|
+
true
|
1364
|
+
rescue
|
1365
|
+
false
|
1366
|
+
end
|
1367
|
+
end
|
1368
|
+
|
1369
|
+
|
1370
|
+
end
|
1371
|
+
|
1372
|
+
|
1373
|
+
|
1374
|
+
end
|
1375
|
+
|
1376
|
+
|
1377
|
+
Task::ROUTE = {
|
1378
|
+
copy: Rclone::Copy,
|
1379
|
+
update: Rclone::Update,
|
1380
|
+
synchronize: Rclone::Synchronize,
|
1381
|
+
equalize: Rclone::Equalize,
|
1382
|
+
backup: Restic::Backup,
|
1383
|
+
restore: Restic::Restore
|
1384
|
+
}
|
1385
|
+
|
1386
|
+
|
1387
|
+
class Endpoint
|
1388
|
+
|
1389
|
+
|
1390
|
+
def self.restore(hash)
|
1391
|
+
endpoint = allocate
|
1392
|
+
endpoint.send(:restore, hash)
|
1393
|
+
endpoint
|
1394
|
+
end
|
1395
|
+
|
1396
|
+
|
1397
|
+
class Local < Endpoint
|
1398
|
+
|
1399
|
+
|
1400
|
+
attr_reader :root
|
1401
|
+
|
1402
|
+
|
1403
|
+
def initialize(root) = @root = Pathname.new(root).realdirpath
|
1404
|
+
|
1405
|
+
|
1406
|
+
def restore(hash) = initialize(hash.fetch(:root))
|
1407
|
+
|
1408
|
+
|
1409
|
+
def externalize
|
1410
|
+
{
|
1411
|
+
endpoint: :local,
|
1412
|
+
root: root
|
1413
|
+
}
|
1414
|
+
end
|
1415
|
+
|
1416
|
+
|
1417
|
+
def show_status = root.to_s
|
1418
|
+
|
1419
|
+
|
1420
|
+
def intact? = true
|
1421
|
+
|
1422
|
+
|
1423
|
+
def refers?(volume) = false
|
1424
|
+
|
1425
|
+
|
1426
|
+
def generation = 0
|
1427
|
+
|
1428
|
+
|
1429
|
+
end
|
1430
|
+
|
1431
|
+
|
1432
|
+
class Rclone < Endpoint
|
1433
|
+
# TODO
|
1434
|
+
end
|
1435
|
+
|
1436
|
+
|
1437
|
+
class Bitferry < Endpoint
|
1438
|
+
|
1439
|
+
|
1440
|
+
attr_reader :volume_tag
|
1441
|
+
|
1442
|
+
|
1443
|
+
attr_reader :path
|
1444
|
+
|
1445
|
+
|
1446
|
+
def root = Volume[volume_tag].root.join(path)
|
1447
|
+
|
1448
|
+
|
1449
|
+
def initialize(volume, path)
|
1450
|
+
@volume_tag = volume.tag
|
1451
|
+
@path = Pathname.new(path)
|
1452
|
+
raise ArgumentError, "expected relative path but got #{self.path}" if (Volume::ENCOMPASSING_PATH_PREFIX =~ self.path.to_s).nil?
|
1453
|
+
end
|
1454
|
+
|
1455
|
+
|
1456
|
+
def restore(hash)
|
1457
|
+
@volume_tag = hash.fetch(:volume)
|
1458
|
+
@path = Pathname.new(hash.fetch(:path, ''))
|
1459
|
+
end
|
1460
|
+
|
1461
|
+
|
1462
|
+
def externalize
|
1463
|
+
{
|
1464
|
+
endpoint: :bitferry,
|
1465
|
+
volume: volume_tag,
|
1466
|
+
path: path.to_s.empty? ? nil : path
|
1467
|
+
}.compact
|
1468
|
+
end
|
1469
|
+
|
1470
|
+
|
1471
|
+
def show_status = intact? ? ":#{volume_tag}:#{path}" : ":{#{volume_tag}}:#{path}"
|
1472
|
+
|
1473
|
+
|
1474
|
+
def intact? = !Volume[volume_tag].nil?
|
1475
|
+
|
1476
|
+
|
1477
|
+
def refers?(volume) = volume.tag == volume_tag
|
1478
|
+
|
1479
|
+
|
1480
|
+
def generation
|
1481
|
+
v = Volume[volume_tag]
|
1482
|
+
v ? v.generation : 0
|
1483
|
+
end
|
1484
|
+
|
1485
|
+
|
1486
|
+
end
|
1487
|
+
|
1488
|
+
|
1489
|
+
ROUTE = {
|
1490
|
+
local: Local,
|
1491
|
+
rclone: Rclone,
|
1492
|
+
bitferry: Bitferry
|
1493
|
+
}
|
1494
|
+
|
1495
|
+
|
1496
|
+
end
|
1497
|
+
|
1498
|
+
|
1499
|
+
reset
|
1500
|
+
|
1501
|
+
|
1502
|
+
end
|