synqa 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -5
- data/Gemfile +16 -16
- data/Gemfile.lock +4 -3
- data/README.rdoc +43 -43
- data/Rakefile +53 -53
- data/VERSION +1 -1
- data/_project.el +9 -9
- data/examples/synqa-useage.rb +37 -37
- data/lib/based.rb +177 -177
- data/lib/synqa.rb +1052 -1005
- data/test/helper.rb +18 -18
- data/test/test_based.rb +114 -114
- data/test/test_synqa.rb +37 -37
- metadata +22 -23
data/lib/synqa.rb
CHANGED
@@ -1,1005 +1,1052 @@
|
|
1
|
-
require 'time'
|
2
|
-
require 'net/ssh'
|
3
|
-
require 'net/scp'
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
@
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
#
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
@
|
69
|
-
@
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
#
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
#
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
#
|
112
|
-
# to list all
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
@
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
#
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
contentTree
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
#
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
end
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
def
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
#
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
def
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
end
|
311
|
-
|
312
|
-
#
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
sshAndScp
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
def
|
337
|
-
sshAndScp.
|
338
|
-
end
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
#
|
359
|
-
def
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
end
|
367
|
-
|
368
|
-
#
|
369
|
-
def
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
def
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
#
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
def
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
# the
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
end
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
end
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
end
|
629
|
-
|
630
|
-
#
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
return
|
666
|
-
end
|
667
|
-
|
668
|
-
#
|
669
|
-
#
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
end
|
685
|
-
end
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
#
|
690
|
-
#
|
691
|
-
def
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
#
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
788
|
-
|
789
|
-
#
|
790
|
-
#
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
|
815
|
-
|
816
|
-
|
817
|
-
|
818
|
-
|
819
|
-
|
820
|
-
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
|
857
|
-
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
def
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
|
881
|
-
|
882
|
-
|
883
|
-
|
884
|
-
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
900
|
-
|
901
|
-
|
902
|
-
end
|
903
|
-
|
904
|
-
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
|
921
|
-
|
922
|
-
|
923
|
-
end
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
|
941
|
-
|
942
|
-
|
943
|
-
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
|
951
|
-
|
952
|
-
|
953
|
-
|
954
|
-
|
955
|
-
|
956
|
-
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
|
961
|
-
|
962
|
-
|
963
|
-
|
964
|
-
|
965
|
-
|
966
|
-
|
967
|
-
|
968
|
-
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
|
973
|
-
|
974
|
-
|
975
|
-
|
976
|
-
|
977
|
-
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
983
|
-
|
984
|
-
|
985
|
-
|
986
|
-
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
|
996
|
-
|
997
|
-
|
998
|
-
|
999
|
-
|
1000
|
-
|
1001
|
-
|
1002
|
-
|
1003
|
-
|
1004
|
-
|
1005
|
-
end
|
1
|
+
require 'time'
|
2
|
+
require 'net/ssh'
|
3
|
+
require 'net/scp'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Synqa
|
7
|
+
|
8
|
+
# ensure that a directory exists
|
9
|
+
def ensureDirectoryExists(directoryName)
|
10
|
+
if File.exist? directoryName
|
11
|
+
if not File.directory? directoryName
|
12
|
+
raise "#{directoryName} is a non-directory file"
|
13
|
+
end
|
14
|
+
else
|
15
|
+
FileUtils.makedirs(directoryName)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# Return the enumerated lines of the command's output
|
20
|
+
def getCommandOutput(command)
|
21
|
+
puts "#{command.inspect} ..."
|
22
|
+
return IO.popen(command)
|
23
|
+
end
|
24
|
+
|
25
|
+
# Check if the last executed process exited with status 0, if not, raise an exception
|
26
|
+
def checkProcessStatus(description)
|
27
|
+
processStatus = $?
|
28
|
+
if not processStatus.exited?
|
29
|
+
raise "#{description}: process did not exit normally"
|
30
|
+
end
|
31
|
+
exitStatus = processStatus.exitstatus
|
32
|
+
if exitStatus != 0
|
33
|
+
raise "#{description}: exit status = #{exitStatus}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# An object representing a file path relative to a base directory, and a hash string
|
38
|
+
class RelativePathWithHash
|
39
|
+
# The relative file path (e.g. c:/dir/subdir/file.txt relative to c:/dir would be subdir/file.txt)
|
40
|
+
attr_reader :relativePath
|
41
|
+
|
42
|
+
# The hash code, e.g. a1c5b67fdb3cf0df8f1d29ae90561f9ad099bada44aeb6b2574ad9e15f2a84ed
|
43
|
+
attr_reader :hash
|
44
|
+
|
45
|
+
def initialize(relativePath, hash)
|
46
|
+
@relativePath = relativePath
|
47
|
+
@hash = hash
|
48
|
+
end
|
49
|
+
|
50
|
+
def inspect
|
51
|
+
return "RelativePathWithHash[#{relativePath}, #{hash}]"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# A command to be executed on the remote system which calculates a hash value for
|
56
|
+
# a file (of a given length), in the format: *hexadecimal-hash* *a-fixed-number-of-characters* *file-name*
|
57
|
+
class HashCommand
|
58
|
+
# The command - a string or array of strings e.g. "sha256sum" or ["sha256", "-r"]
|
59
|
+
attr_reader :command
|
60
|
+
|
61
|
+
# The length of the calculated hash value e.g. 64 for sha256
|
62
|
+
attr_reader :length
|
63
|
+
|
64
|
+
# The number of characters between the hash value and the file name (usually 1 or 2)
|
65
|
+
attr_reader :spacerLen
|
66
|
+
|
67
|
+
def initialize(command, length, spacerLen)
|
68
|
+
@command = command
|
69
|
+
@length = length
|
70
|
+
@spacerLen = spacerLen
|
71
|
+
end
|
72
|
+
|
73
|
+
# Parse a hash line relative to a base directory, returning a RelativePathWithHash
|
74
|
+
def parseFileHashLine(baseDir, fileHashLine)
|
75
|
+
hash = fileHashLine[0...length]
|
76
|
+
fullPath = fileHashLine[(length + spacerLen)..-1]
|
77
|
+
if fullPath.start_with?(baseDir)
|
78
|
+
return RelativePathWithHash.new(fullPath[baseDir.length..-1], hash)
|
79
|
+
else
|
80
|
+
raise "File #{fullPath} from hash line is not in base dir #{baseDir}"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def to_s
|
85
|
+
return command.join(" ")
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Hash command for sha256sum, which generates a 64 hexadecimal digit hash, and outputs two characters between
|
90
|
+
# the hash and the file name.
|
91
|
+
class Sha256SumCommand<HashCommand
|
92
|
+
def initialize
|
93
|
+
super(["sha256sum"], 64, 2)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
# Hash command for sha256, which generates a 64 hexadecimal digit hash, and outputs one character between
|
98
|
+
# the hash and the file name, and which requires a "-r" argument to put the hash value first.
|
99
|
+
class Sha256Command<HashCommand
|
100
|
+
def initialize
|
101
|
+
super(["sha256", "-r"], 64, 1)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# Put "/" at the end of a directory name if it is not already there.
|
106
|
+
def normalisedDir(baseDir)
|
107
|
+
return baseDir.end_with?("/") ? baseDir : baseDir + "/"
|
108
|
+
end
|
109
|
+
|
110
|
+
|
111
|
+
# Base class for an object representing a remote system where the contents of a directory
|
112
|
+
# on the system are enumerated by one command to list all sub-directories and another command
|
113
|
+
# to list all files in the directory and their hash values.
|
114
|
+
class DirContentHost
|
115
|
+
|
116
|
+
# The HashCommand object used to calculate and parse hash values of files
|
117
|
+
attr_reader :hashCommand
|
118
|
+
|
119
|
+
# Prefix required for *find* command (usually nothing, since it should be on the system path)
|
120
|
+
attr_reader :pathPrefix
|
121
|
+
|
122
|
+
def initialize(hashCommand, pathPrefix = "")
|
123
|
+
@hashCommand = hashCommand
|
124
|
+
@pathPrefix = pathPrefix
|
125
|
+
end
|
126
|
+
|
127
|
+
# Generate the *find* command which will list all the sub-directories of the base directory
|
128
|
+
def findDirectoriesCommand(baseDir)
|
129
|
+
return ["#{@pathPrefix}find", baseDir, "-type", "d", "-print"]
|
130
|
+
end
|
131
|
+
|
132
|
+
# Return the list of sub-directories relative to the base directory
|
133
|
+
def listDirectories(baseDir)
|
134
|
+
baseDir = normalisedDir(baseDir)
|
135
|
+
command = findDirectoriesCommand(baseDir)
|
136
|
+
output = getCommandOutput(command)
|
137
|
+
directories = []
|
138
|
+
baseDirLen = baseDir.length
|
139
|
+
puts "Listing directories ..."
|
140
|
+
while (line = output.gets)
|
141
|
+
line = line.chomp
|
142
|
+
puts " #{line}"
|
143
|
+
if line.start_with?(baseDir)
|
144
|
+
directories << line[baseDirLen..-1]
|
145
|
+
else
|
146
|
+
raise "Directory #{line} is not a sub-directory of base directory #{baseDir}"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
output.close()
|
150
|
+
checkProcessStatus(command)
|
151
|
+
return directories
|
152
|
+
end
|
153
|
+
|
154
|
+
# Generate the *find* command which will list all the files within the base directory
|
155
|
+
def findFilesCommand(baseDir)
|
156
|
+
return ["#{@pathPrefix}find", baseDir, "-type", "f", "-print"]
|
157
|
+
end
|
158
|
+
|
159
|
+
# List file hashes by executing the command to hash each file on the output of the
|
160
|
+
# *find* command which lists all files, and parse the output.
|
161
|
+
def listFileHashes(baseDir)
|
162
|
+
baseDir = normalisedDir(baseDir)
|
163
|
+
fileHashes = []
|
164
|
+
listFileHashLines(baseDir) do |fileHashLine|
|
165
|
+
fileHash = self.hashCommand.parseFileHashLine(baseDir, fileHashLine)
|
166
|
+
if fileHash != nil
|
167
|
+
fileHashes << fileHash
|
168
|
+
end
|
169
|
+
end
|
170
|
+
return fileHashes
|
171
|
+
end
|
172
|
+
|
173
|
+
# Construct the ContentTree for the given base directory
|
174
|
+
def getContentTree(baseDir)
|
175
|
+
contentTree = ContentTree.new()
|
176
|
+
contentTree.time = Time.now.utc
|
177
|
+
for dir in listDirectories(baseDir)
|
178
|
+
contentTree.addDir(dir)
|
179
|
+
end
|
180
|
+
for fileHash in listFileHashes(baseDir)
|
181
|
+
contentTree.addFile(fileHash.relativePath, fileHash.hash)
|
182
|
+
end
|
183
|
+
return contentTree
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# Execute a (local) command, or, if dryRun, just pretend to execute it.
|
188
|
+
# Raise an exception if the process exit status is not 0.
|
189
|
+
def executeCommand(command, dryRun)
|
190
|
+
puts "EXECUTE: #{command}"
|
191
|
+
if not dryRun
|
192
|
+
system(command)
|
193
|
+
checkProcessStatus(command)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# Base SSH/SCP implementation
|
198
|
+
class BaseSshScp
|
199
|
+
attr_reader :userAtHost, :user, :host
|
200
|
+
|
201
|
+
def setUserAtHost(userAtHost)
|
202
|
+
@userAtHost = userAtHost
|
203
|
+
@user, @host = @userAtHost.split("@")
|
204
|
+
end
|
205
|
+
|
206
|
+
def close
|
207
|
+
# by default do nothing - close any cached connections
|
208
|
+
end
|
209
|
+
|
210
|
+
# delete remote directory (if dryRun is false) using "rm -r"
|
211
|
+
def deleteDirectory(dirPath, dryRun)
|
212
|
+
ssh("rm -r #{dirPath}", dryRun)
|
213
|
+
end
|
214
|
+
|
215
|
+
# delete remote file (if dryRun is false) using "rm"
|
216
|
+
def deleteFile(filePath, dryRun)
|
217
|
+
ssh("rm #{filePath}", dryRun)
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
# SSH/SCP using Ruby Net::SSH & Net::SCP
|
222
|
+
class InternalSshScp<BaseSshScp
|
223
|
+
|
224
|
+
def initialize
|
225
|
+
@connection = nil
|
226
|
+
end
|
227
|
+
|
228
|
+
def connection
|
229
|
+
if @connection == nil
|
230
|
+
puts "Opening SSH connection to #{user}@#{host} ..."
|
231
|
+
@connection = Net::SSH.start(host, user)
|
232
|
+
end
|
233
|
+
return @connection
|
234
|
+
end
|
235
|
+
|
236
|
+
def scpConnection
|
237
|
+
return connection.scp
|
238
|
+
end
|
239
|
+
|
240
|
+
def close()
|
241
|
+
if @connection != nil
|
242
|
+
puts "Closing SSH connection to #{user}@#{host} ..."
|
243
|
+
@connection.close()
|
244
|
+
@connection = nil
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
# execute command on remote host (if dryRun is false), yielding lines of output
|
249
|
+
def ssh(commandString, dryRun)
|
250
|
+
description = "SSH #{user}@#{host}: executing #{commandString}"
|
251
|
+
puts description
|
252
|
+
if not dryRun
|
253
|
+
outputText = connection.exec!(commandString)
|
254
|
+
if outputText != nil then
|
255
|
+
for line in outputText.split("\n") do
|
256
|
+
yield line
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
# copy a local directory to a remote directory (if dryRun is false)
|
263
|
+
def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
264
|
+
description = "SCP: copy directory #{sourcePath} to #{user}@#{host}:#{destinationPath}"
|
265
|
+
puts description
|
266
|
+
if not dryRun
|
267
|
+
scpConnection.upload!(sourcePath, destinationPath, :recursive => true)
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
# copy a local file to a remote directory (if dryRun is false)
|
272
|
+
def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
273
|
+
description = "SCP: copy file #{sourcePath} to #{user}@#{host}:#{destinationPath}"
|
274
|
+
puts description
|
275
|
+
if not dryRun
|
276
|
+
scpConnection.upload!(sourcePath, destinationPath)
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
end
|
281
|
+
|
282
|
+
# SSH/SCP using external commands, such as "plink" and "pscp"
|
283
|
+
class ExternalSshScp<BaseSshScp
|
284
|
+
# The SSH client, e.g. ["ssh"] or ["plink","-pw","mysecretpassword"] (i.e. command + args as an array)
|
285
|
+
attr_reader :shell
|
286
|
+
|
287
|
+
# The SCP client, e.g. ["scp"] or ["pscp","-pw","mysecretpassword"] (i.e. command + args as an array)
|
288
|
+
attr_reader :scpProgram
|
289
|
+
|
290
|
+
# The SCP command as a string
|
291
|
+
attr_reader :scpCommandString
|
292
|
+
|
293
|
+
def initialize(shell, scpProgram)
|
294
|
+
@shell = shell.is_a?(String) ? [shell] : shell
|
295
|
+
@scpProgram = scpProgram.is_a?(String) ? [scpProgram] : scpProgram
|
296
|
+
@scpCommandString = @scpProgram.join(" ")
|
297
|
+
end
|
298
|
+
|
299
|
+
# execute command on remote host (if dryRun is false), yielding lines of output
|
300
|
+
def ssh(commandString, dryRun)
|
301
|
+
puts "SSH #{userAtHost} (#{shell.join(" ")}): executing #{commandString}"
|
302
|
+
if not dryRun
|
303
|
+
output = getCommandOutput(shell + [userAtHost, commandString])
|
304
|
+
while (line = output.gets)
|
305
|
+
yield line.chomp
|
306
|
+
end
|
307
|
+
output.close()
|
308
|
+
checkProcessStatus("SSH #{userAtHost} #{commandString}")
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
312
|
+
# copy a local directory to a remote directory (if dryRun is false)
|
313
|
+
def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
314
|
+
executeCommand("#{@scpCommandString} -r #{sourcePath} #{userAtHost}:#{destinationPath}", dryRun)
|
315
|
+
end
|
316
|
+
|
317
|
+
# copy a local file to a remote directory (if dryRun is false)
|
318
|
+
def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
319
|
+
executeCommand("#{@scpCommandString} #{sourcePath} #{userAtHost}:#{destinationPath}", dryRun)
|
320
|
+
end
|
321
|
+
|
322
|
+
end
|
323
|
+
|
324
|
+
# Representation of a remote system accessible via SSH
|
325
|
+
class SshContentHost<DirContentHost
|
326
|
+
|
327
|
+
# The remote SSH/SCP login, e.g. SSH via "username@host.example.com"
|
328
|
+
attr_reader :sshAndScp
|
329
|
+
|
330
|
+
def initialize(userAtHost, hashCommand, sshAndScp = nil)
|
331
|
+
super(hashCommand)
|
332
|
+
@sshAndScp = sshAndScp != nil ? sshAndScp : InternalSshScp.new()
|
333
|
+
@sshAndScp.setUserAtHost(userAtHost)
|
334
|
+
end
|
335
|
+
|
336
|
+
def userAtHost
|
337
|
+
return @sshAndScp.userAtHost
|
338
|
+
end
|
339
|
+
|
340
|
+
def closeConnections()
|
341
|
+
@sshAndScp.close()
|
342
|
+
end
|
343
|
+
|
344
|
+
# Return readable description of base directory on remote system
|
345
|
+
def locationDescriptor(baseDir)
|
346
|
+
baseDir = normalisedDir(baseDir)
|
347
|
+
return "#{userAtHost}:#{baseDir} (connect = #{shell}/#{scpProgram}, hashCommand = #{hashCommand})"
|
348
|
+
end
|
349
|
+
|
350
|
+
# execute an SSH command on the remote system, yielding lines of output
|
351
|
+
# (or don't actually execute, if dryRun is false)
|
352
|
+
def ssh(commandString, dryRun = false)
|
353
|
+
sshAndScp.ssh(commandString, dryRun) do |line|
|
354
|
+
yield line
|
355
|
+
end
|
356
|
+
end
|
357
|
+
|
358
|
+
# delete a remote directory, if dryRun is false
|
359
|
+
def deleteDirectory(dirPath, dryRun)
|
360
|
+
sshAndScp.deleteDirectory(dirPath, dryRun)
|
361
|
+
end
|
362
|
+
|
363
|
+
# delete a remote file, if dryRun is false
|
364
|
+
def deleteFile(filePath, dryRun)
|
365
|
+
sshAndScp.deleteFile(filePath, dryRun)
|
366
|
+
end
|
367
|
+
|
368
|
+
# copy a local directory to a remote directory, if dryRun is false
|
369
|
+
def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
370
|
+
sshAndScp.copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
371
|
+
end
|
372
|
+
|
373
|
+
# copy a local file to a remote directory, if dryRun is false
|
374
|
+
def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
375
|
+
sshAndScp.copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
376
|
+
end
|
377
|
+
|
378
|
+
# Return a list of all subdirectories of the base directory (as paths relative to the base directory)
|
379
|
+
def listDirectories(baseDir)
|
380
|
+
baseDir = normalisedDir(baseDir)
|
381
|
+
puts "Listing directories ..."
|
382
|
+
directories = []
|
383
|
+
baseDirLen = baseDir.length
|
384
|
+
ssh(findDirectoriesCommand(baseDir).join(" ")) do |line|
|
385
|
+
puts " #{line}"
|
386
|
+
if line.start_with?(baseDir)
|
387
|
+
directories << line[baseDirLen..-1]
|
388
|
+
else
|
389
|
+
raise "Directory #{line} is not a sub-directory of base directory #{baseDir}"
|
390
|
+
end
|
391
|
+
end
|
392
|
+
return directories
|
393
|
+
end
|
394
|
+
|
395
|
+
# Yield lines of output from the command to display hash values and file names
|
396
|
+
# of all files within the base directory
|
397
|
+
def listFileHashLines(baseDir)
|
398
|
+
baseDir = normalisedDir(baseDir)
|
399
|
+
remoteFileHashLinesCommand = findFilesCommand(baseDir) + ["|", "xargs", "-r"] + @hashCommand.command
|
400
|
+
ssh(remoteFileHashLinesCommand.join(" ")) do |line|
|
401
|
+
puts " #{line}"
|
402
|
+
yield line
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
# List all files within the base directory to stdout
|
407
|
+
def listFiles(baseDir)
|
408
|
+
baseDir = normalisedDir(baseDir)
|
409
|
+
ssh(findFilesCommand(baseDir).join(" ")) do |line|
|
410
|
+
puts " #{line}"
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
end
|
415
|
+
|
416
|
+
# An object representing the content of a file within a ContentTree.
|
417
|
+
# The file may be marked for copying (if it's in a source ContentTree)
|
418
|
+
# or for deletion (if it's in a destination ContentTree)
|
419
|
+
class FileContent
|
420
|
+
# The name of the file
|
421
|
+
attr_reader :name
|
422
|
+
|
423
|
+
# The hash value of the file's contents
|
424
|
+
attr_reader :hash
|
425
|
+
|
426
|
+
# The components of the relative path where the file is found
|
427
|
+
attr_reader :parentPathElements
|
428
|
+
|
429
|
+
# The destination to which the file should be copied
|
430
|
+
attr_reader :copyDestination
|
431
|
+
|
432
|
+
# Should this file be deleted
|
433
|
+
attr_reader :toBeDeleted
|
434
|
+
|
435
|
+
def initialize(name, hash, parentPathElements)
|
436
|
+
@name = name
|
437
|
+
@hash = hash
|
438
|
+
@parentPathElements = parentPathElements
|
439
|
+
@copyDestination = nil
|
440
|
+
@toBeDeleted = false
|
441
|
+
end
|
442
|
+
|
443
|
+
# Mark this file to be copied to a destination directory (from a destination content tree)
|
444
|
+
def markToCopy(destinationDirectory)
|
445
|
+
@copyDestination = destinationDirectory
|
446
|
+
end
|
447
|
+
|
448
|
+
# Mark this file to be deleted
|
449
|
+
def markToDelete
|
450
|
+
@toBeDeleted = true
|
451
|
+
end
|
452
|
+
|
453
|
+
def to_s
|
454
|
+
return "#{name} (#{hash})"
|
455
|
+
end
|
456
|
+
|
457
|
+
# The relative name of this file in the content tree (relative to the base dir)
|
458
|
+
def relativePath
|
459
|
+
return (parentPathElements + [name]).join("/")
|
460
|
+
end
|
461
|
+
end
|
462
|
+
|
463
|
+
# A "content tree" consisting of a description of the contents of files and
|
464
|
+
# sub-directories within a base directory. The file contents are described via
|
465
|
+
# cryptographic hash values.
|
466
|
+
# Each sub-directory within a content tree is also represented as a ContentTree.
|
467
|
+
class ContentTree
|
468
|
+
# name of the sub-directory within the containing directory (or nil if this is the base directory)
|
469
|
+
attr_reader :name
|
470
|
+
|
471
|
+
# path elements from base directory leading to this one
|
472
|
+
attr_reader :pathElements
|
473
|
+
|
474
|
+
# files within this sub-directory (as FileContent's)
|
475
|
+
attr_reader :files
|
476
|
+
|
477
|
+
# immediate sub-directories of this directory
|
478
|
+
attr_reader :dirs
|
479
|
+
|
480
|
+
# the files within this sub-directory, indexed by file name
|
481
|
+
attr_reader :fileByName
|
482
|
+
|
483
|
+
# immediate sub-directories of this directory, indexed by name
|
484
|
+
attr_reader :dirByName
|
485
|
+
|
486
|
+
# where this directory should be copied to
|
487
|
+
attr_reader :copyDestination
|
488
|
+
|
489
|
+
# whether this directory should be deleted
|
490
|
+
attr_reader :toBeDeleted
|
491
|
+
|
492
|
+
# the UTC time (on the local system, even if this content tree represents a remote directory)
|
493
|
+
# that this content tree was constructed. Only set for the base directory.
|
494
|
+
attr_accessor :time
|
495
|
+
|
496
|
+
def initialize(name = nil, parentPathElements = nil)
|
497
|
+
@name = name
|
498
|
+
@pathElements = name == nil ? [] : parentPathElements + [name]
|
499
|
+
@files = []
|
500
|
+
@dirs = []
|
501
|
+
@fileByName = {}
|
502
|
+
@dirByName = {}
|
503
|
+
@copyDestination = nil
|
504
|
+
@toBeDeleted = false
|
505
|
+
@time = nil
|
506
|
+
end
|
507
|
+
|
508
|
+
# mark this directory to be copied to a destination directory
|
509
|
+
def markToCopy(destinationDirectory)
|
510
|
+
@copyDestination = destinationDirectory
|
511
|
+
end
|
512
|
+
|
513
|
+
# mark this directory (on a remote system) to be deleted
|
514
|
+
def markToDelete
|
515
|
+
@toBeDeleted = true
|
516
|
+
end
|
517
|
+
|
518
|
+
# the path of the directory that this content tree represents, relative to the base directory
|
519
|
+
def relativePath
|
520
|
+
return @pathElements.join("/")
|
521
|
+
end
|
522
|
+
|
523
|
+
# convert a path string to an array of path elements (or return it as is if it's already an array)
|
524
|
+
def getPathElements(path)
|
525
|
+
return path.is_a?(String) ? (path == "" ? [] : path.split("/")) : path
|
526
|
+
end
|
527
|
+
|
528
|
+
# get the content tree for a sub-directory (creating it if it doesn't yet exist)
|
529
|
+
def getContentTreeForSubDir(subDir)
|
530
|
+
dirContentTree = dirByName.fetch(subDir, nil)
|
531
|
+
if dirContentTree == nil
|
532
|
+
dirContentTree = ContentTree.new(subDir, @pathElements)
|
533
|
+
dirs << dirContentTree
|
534
|
+
dirByName[subDir] = dirContentTree
|
535
|
+
end
|
536
|
+
return dirContentTree
|
537
|
+
end
|
538
|
+
|
539
|
+
# add a sub-directory to this content tree
|
540
|
+
def addDir(dirPath)
|
541
|
+
pathElements = getPathElements(dirPath)
|
542
|
+
if pathElements.length > 0
|
543
|
+
pathStart = pathElements[0]
|
544
|
+
restOfPath = pathElements[1..-1]
|
545
|
+
getContentTreeForSubDir(pathStart).addDir(restOfPath)
|
546
|
+
end
|
547
|
+
end
|
548
|
+
|
549
|
+
# recursively sort the files and sub-directories of this content tree alphabetically
|
550
|
+
def sort!
|
551
|
+
dirs.sort_by! {|dir| dir.name}
|
552
|
+
files.sort_by! {|file| file.name}
|
553
|
+
for dir in dirs
|
554
|
+
dir.sort!
|
555
|
+
end
|
556
|
+
end
|
557
|
+
|
558
|
+
# given a relative path, add a file and hash value to this content tree
|
559
|
+
def addFile(filePath, hash)
|
560
|
+
pathElements = getPathElements(filePath)
|
561
|
+
if pathElements.length == 0
|
562
|
+
raise "Invalid file path: #{filePath.inspect}"
|
563
|
+
end
|
564
|
+
if pathElements.length == 1
|
565
|
+
fileName = pathElements[0]
|
566
|
+
fileContent = FileContent.new(fileName, hash, @pathElements)
|
567
|
+
files << fileContent
|
568
|
+
fileByName[fileName] = fileContent
|
569
|
+
else
|
570
|
+
pathStart = pathElements[0]
|
571
|
+
restOfPath = pathElements[1..-1]
|
572
|
+
getContentTreeForSubDir(pathStart).addFile(restOfPath, hash)
|
573
|
+
end
|
574
|
+
end
|
575
|
+
|
576
|
+
# date-time format for reading and writing times, e.g. "2007-12-23 13:03:99.012 +0000"
|
577
|
+
@@dateTimeFormat = "%Y-%m-%d %H:%M:%S.%L %z"
|
578
|
+
|
579
|
+
# pretty-print this content tree
|
580
|
+
def showIndented(name = "", indent = " ", currentIndent = "")
|
581
|
+
if time != nil
|
582
|
+
puts "#{currentIndent}[TIME: #{time.strftime(@@dateTimeFormat)}]"
|
583
|
+
end
|
584
|
+
if name != ""
|
585
|
+
puts "#{currentIndent}#{name}"
|
586
|
+
end
|
587
|
+
if copyDestination != nil
|
588
|
+
puts "#{currentIndent} [COPY to #{copyDestination.relativePath}]"
|
589
|
+
end
|
590
|
+
if toBeDeleted
|
591
|
+
puts "#{currentIndent} [DELETE]"
|
592
|
+
end
|
593
|
+
nextIndent = currentIndent + indent
|
594
|
+
for dir in dirs
|
595
|
+
dir.showIndented("#{dir.name}/", indent = indent, currentIndent = nextIndent)
|
596
|
+
end
|
597
|
+
for file in files
|
598
|
+
puts "#{nextIndent}#{file.name} - #{file.hash}"
|
599
|
+
if file.copyDestination != nil
|
600
|
+
puts "#{nextIndent} [COPY to #{file.copyDestination.relativePath}]"
|
601
|
+
end
|
602
|
+
if file.toBeDeleted
|
603
|
+
puts "#{nextIndent} [DELETE]"
|
604
|
+
end
|
605
|
+
end
|
606
|
+
end
|
607
|
+
|
608
|
+
# write this content tree to an open file, indented
|
609
|
+
def writeLinesToFile(outFile, prefix = "")
|
610
|
+
if time != nil
|
611
|
+
outFile.puts("T #{time.strftime(@@dateTimeFormat)}\n")
|
612
|
+
end
|
613
|
+
for dir in dirs
|
614
|
+
outFile.puts("D #{prefix}#{dir.name}\n")
|
615
|
+
dir.writeLinesToFile(outFile, "#{prefix}#{dir.name}/")
|
616
|
+
end
|
617
|
+
for file in files
|
618
|
+
outFile.puts("F #{file.hash} #{prefix}#{file.name}\n")
|
619
|
+
end
|
620
|
+
end
|
621
|
+
|
622
|
+
# write this content tree to a file (in a format which readFromFile can read back in)
|
623
|
+
def writeToFile(fileName)
|
624
|
+
puts "Writing content tree to file #{fileName} ..."
|
625
|
+
File.open(fileName, "w") do |outFile|
|
626
|
+
writeLinesToFile(outFile)
|
627
|
+
end
|
628
|
+
end
|
629
|
+
|
630
|
+
# regular expression for directory entries in content tree file
|
631
|
+
@@dirLineRegex = /^D (.*)$/
|
632
|
+
|
633
|
+
# regular expression for file entries in content tree file
|
634
|
+
@@fileLineRegex = /^F ([^ ]*) (.*)$/
|
635
|
+
|
636
|
+
# regular expression for time entry in content tree file
|
637
|
+
@@timeRegex = /^T (.*)$/
|
638
|
+
|
639
|
+
# read a content tree from a file (in format written by writeToFile)
|
640
|
+
def self.readFromFile(fileName)
|
641
|
+
contentTree = ContentTree.new()
|
642
|
+
puts "Reading content tree from #{fileName} ..."
|
643
|
+
IO.foreach(fileName) do |line|
|
644
|
+
dirLineMatch = @@dirLineRegex.match(line)
|
645
|
+
if dirLineMatch
|
646
|
+
dirName = dirLineMatch[1]
|
647
|
+
contentTree.addDir(dirName)
|
648
|
+
else
|
649
|
+
fileLineMatch = @@fileLineRegex.match(line)
|
650
|
+
if fileLineMatch
|
651
|
+
hash = fileLineMatch[1]
|
652
|
+
fileName = fileLineMatch[2]
|
653
|
+
contentTree.addFile(fileName, hash)
|
654
|
+
else
|
655
|
+
timeLineMatch = @@timeRegex.match(line)
|
656
|
+
if timeLineMatch
|
657
|
+
timeString = timeLineMatch[1]
|
658
|
+
contentTree.time = Time.strptime(timeString, @@dateTimeFormat)
|
659
|
+
else
|
660
|
+
raise "Invalid line in content tree file: #{line.inspect}"
|
661
|
+
end
|
662
|
+
end
|
663
|
+
end
|
664
|
+
end
|
665
|
+
return contentTree
|
666
|
+
end
|
667
|
+
|
668
|
+
# read a content tree as a map of hashes, i.e. from relative file path to hash value for the file
|
669
|
+
# Actually returns an array of the time entry (if any) and the map of hashes
|
670
|
+
def self.readMapOfHashesFromFile(fileName)
|
671
|
+
mapOfHashes = {}
|
672
|
+
time = nil
|
673
|
+
File.open(fileName).each_line do |line|
|
674
|
+
fileLineMatch = @@fileLineRegex.match(line)
|
675
|
+
if fileLineMatch
|
676
|
+
hash = fileLineMatch[1]
|
677
|
+
fileName = fileLineMatch[2]
|
678
|
+
mapOfHashes[fileName] = hash
|
679
|
+
end
|
680
|
+
timeLineMatch = @@timeRegex.match(line)
|
681
|
+
if timeLineMatch
|
682
|
+
timeString = timeLineMatch[1]
|
683
|
+
time = Time.strptime(timeString, @@dateTimeFormat)
|
684
|
+
end
|
685
|
+
end
|
686
|
+
return [time, mapOfHashes]
|
687
|
+
end
|
688
|
+
|
689
|
+
# Mark operations for this (source) content tree and the destination content tree
|
690
|
+
# in order to synch the destination content tree with this one
|
691
|
+
def markSyncOperationsForDestination(destination)
|
692
|
+
markCopyOperations(destination)
|
693
|
+
destination.markDeleteOptions(self)
|
694
|
+
end
|
695
|
+
|
696
|
+
# Get the named sub-directory content tree, if it exists
|
697
|
+
def getDir(dir)
|
698
|
+
return dirByName.fetch(dir, nil)
|
699
|
+
end
|
700
|
+
|
701
|
+
# Get the named file & hash value, if it exists
|
702
|
+
def getFile(file)
|
703
|
+
return fileByName.fetch(file, nil)
|
704
|
+
end
|
705
|
+
|
706
|
+
# Mark copy operations, given that the corresponding destination directory already exists.
|
707
|
+
# For files and directories that don't exist in the destination, mark them to be copied.
|
708
|
+
# For sub-directories that do exist, recursively mark the corresponding sub-directory copy operations.
|
709
|
+
def markCopyOperations(destinationDir)
|
710
|
+
for dir in dirs
|
711
|
+
destinationSubDir = destinationDir.getDir(dir.name)
|
712
|
+
if destinationSubDir != nil
|
713
|
+
dir.markCopyOperations(destinationSubDir)
|
714
|
+
else
|
715
|
+
dir.markToCopy(destinationDir)
|
716
|
+
end
|
717
|
+
end
|
718
|
+
for file in files
|
719
|
+
destinationFile = destinationDir.getFile(file.name)
|
720
|
+
if destinationFile == nil or destinationFile.hash != file.hash
|
721
|
+
file.markToCopy(destinationDir)
|
722
|
+
end
|
723
|
+
end
|
724
|
+
end
|
725
|
+
|
726
|
+
# Mark delete operations, given that the corresponding source directory exists.
|
727
|
+
# For files and directories that don't exist in the source, mark them to be deleted.
|
728
|
+
# For sub-directories that do exist, recursively mark the corresponding sub-directory delete operations.
|
729
|
+
def markDeleteOptions(sourceDir)
|
730
|
+
for dir in dirs
|
731
|
+
sourceSubDir = sourceDir.getDir(dir.name)
|
732
|
+
if sourceSubDir == nil
|
733
|
+
dir.markToDelete()
|
734
|
+
else
|
735
|
+
dir.markDeleteOptions(sourceSubDir)
|
736
|
+
end
|
737
|
+
end
|
738
|
+
for file in files
|
739
|
+
sourceFile = sourceDir.getFile(file.name)
|
740
|
+
if sourceFile == nil
|
741
|
+
file.markToDelete()
|
742
|
+
end
|
743
|
+
end
|
744
|
+
end
|
745
|
+
end
|
746
|
+
|
747
|
+
# Base class for a content location which consists of a base directory
|
748
|
+
# on a local or remote system.
|
749
|
+
class ContentLocation
|
750
|
+
|
751
|
+
# The name of a file used to hold a cached content tree for this location (can optionally be specified)
|
752
|
+
attr_reader :cachedContentFile
|
753
|
+
|
754
|
+
def initialize(cachedContentFile)
|
755
|
+
@cachedContentFile = cachedContentFile
|
756
|
+
end
|
757
|
+
|
758
|
+
# Get the cached content file name, if specified, and if the file exists
|
759
|
+
def getExistingCachedContentTreeFile
|
760
|
+
if cachedContentFile == nil
|
761
|
+
puts "No cached content file specified for location"
|
762
|
+
return nil
|
763
|
+
elsif File.exists?(cachedContentFile)
|
764
|
+
return cachedContentFile
|
765
|
+
else
|
766
|
+
puts "Cached content file #{cachedContentFile} does not yet exist."
|
767
|
+
return nil
|
768
|
+
end
|
769
|
+
end
|
770
|
+
|
771
|
+
# Delete any existing cached content file
|
772
|
+
def clearCachedContentFile
|
773
|
+
if cachedContentFile and File.exists?(cachedContentFile)
|
774
|
+
puts " deleting cached content file #{cachedContentFile} ..."
|
775
|
+
File.delete(cachedContentFile)
|
776
|
+
end
|
777
|
+
end
|
778
|
+
|
779
|
+
# Get the cached content tree (if any), read from the specified cached content file.
|
780
|
+
def getCachedContentTree
|
781
|
+
file = getExistingCachedContentTreeFile
|
782
|
+
if file
|
783
|
+
return ContentTree.readFromFile(file)
|
784
|
+
else
|
785
|
+
return nil
|
786
|
+
end
|
787
|
+
end
|
788
|
+
|
789
|
+
# Read a map of file hashes (mapping from relative file name to hash value) from the
|
790
|
+
# specified cached content file
|
791
|
+
def getCachedContentTreeMapOfHashes
|
792
|
+
file = getExistingCachedContentTreeFile
|
793
|
+
if file
|
794
|
+
puts "Reading cached file hashes from #{file} ..."
|
795
|
+
return ContentTree.readMapOfHashesFromFile(file)
|
796
|
+
else
|
797
|
+
return [nil, {}]
|
798
|
+
end
|
799
|
+
end
|
800
|
+
|
801
|
+
end
|
802
|
+
|
803
|
+
# A directory of files on a local system. The corresponding content tree
|
804
|
+
# can be calculated directly using Ruby library functions.
|
805
|
+
class LocalContentLocation<ContentLocation
|
806
|
+
|
807
|
+
# the base directory, for example of type Based::BaseDirectory. Methods invoked are: allFiles, subDirs and fullPath.
|
808
|
+
# For file and dir objects returned by allFiles & subDirs, methods invoked are: relativePath and fullPath
|
809
|
+
attr_reader :baseDirectory
|
810
|
+
# the ruby class that generates the hash, e.g. Digest::SHA256
|
811
|
+
attr_reader :hashClass
|
812
|
+
|
813
|
+
def initialize(baseDirectory, hashClass, cachedContentFile = nil)
|
814
|
+
super(cachedContentFile)
|
815
|
+
@baseDirectory = baseDirectory
|
816
|
+
@hashClass = hashClass
|
817
|
+
end
|
818
|
+
|
819
|
+
# get the full path of a relative path (i.e. of a file/directory within the base directory)
|
820
|
+
def getFullPath(relativePath)
|
821
|
+
return @baseDirectory.fullPath + relativePath
|
822
|
+
end
|
823
|
+
|
824
|
+
# get the content tree for this base directory by iterating over all
|
825
|
+
# sub-directories and files within the base directory (and excluding the excluded files)
|
826
|
+
# and calculating file hashes using the specified Ruby hash class
|
827
|
+
# If there is an existing cached content file, use that to get the hash values
|
828
|
+
# of files whose modification time is earlier than the time value for the cached content tree.
|
829
|
+
# Also, if a cached content file is specified, write the final content tree back out to the cached content file.
|
830
|
+
def getContentTree
|
831
|
+
cachedTimeAndMapOfHashes = getCachedContentTreeMapOfHashes
|
832
|
+
cachedTime = cachedTimeAndMapOfHashes[0]
|
833
|
+
cachedMapOfHashes = cachedTimeAndMapOfHashes[1]
|
834
|
+
contentTree = ContentTree.new()
|
835
|
+
contentTree.time = Time.now.utc
|
836
|
+
for subDir in @baseDirectory.subDirs
|
837
|
+
contentTree.addDir(subDir.relativePath)
|
838
|
+
end
|
839
|
+
for file in @baseDirectory.allFiles
|
840
|
+
cachedDigest = cachedMapOfHashes[file.relativePath]
|
841
|
+
if cachedTime and cachedDigest and File.stat(file.fullPath).mtime < cachedTime
|
842
|
+
digest = cachedDigest
|
843
|
+
else
|
844
|
+
digest = hashClass.file(file.fullPath).hexdigest
|
845
|
+
end
|
846
|
+
contentTree.addFile(file.relativePath, digest)
|
847
|
+
end
|
848
|
+
contentTree.sort!
|
849
|
+
if cachedContentFile != nil
|
850
|
+
contentTree.writeToFile(cachedContentFile)
|
851
|
+
end
|
852
|
+
return contentTree
|
853
|
+
end
|
854
|
+
end
|
855
|
+
|
856
|
+
# A directory of files on a remote system
|
857
|
+
class RemoteContentLocation<ContentLocation
|
858
|
+
# the remote SshContentHost
|
859
|
+
attr_reader :contentHost
|
860
|
+
|
861
|
+
# the base directory on the remote system
|
862
|
+
attr_reader :baseDir
|
863
|
+
|
864
|
+
def initialize(contentHost, baseDir, cachedContentFile = nil)
|
865
|
+
super(cachedContentFile)
|
866
|
+
@contentHost = contentHost
|
867
|
+
@baseDir = normalisedDir(baseDir)
|
868
|
+
end
|
869
|
+
|
870
|
+
def closeConnections
|
871
|
+
@contentHost.closeConnections()
|
872
|
+
end
|
873
|
+
|
874
|
+
# list files within the base directory on the remote contentHost
|
875
|
+
def listFiles()
|
876
|
+
contentHost.listFiles(baseDir)
|
877
|
+
end
|
878
|
+
|
879
|
+
# object required to execute SCP (e.g. "scp" or "pscp", possibly with extra args)
|
880
|
+
def sshAndScp
|
881
|
+
return contentHost.sshAndScp
|
882
|
+
end
|
883
|
+
|
884
|
+
# get the full path of a relative path
|
885
|
+
def getFullPath(relativePath)
|
886
|
+
return baseDir + relativePath
|
887
|
+
end
|
888
|
+
|
889
|
+
# execute an SSH command on the remote host (or just pretend, if dryRun is true)
|
890
|
+
def ssh(commandString, dryRun = false)
|
891
|
+
contentHost.sshAndScp.ssh(commandString, dryRun)
|
892
|
+
end
|
893
|
+
|
894
|
+
# list all sub-directories of the base directory on the remote host
|
895
|
+
def listDirectories
|
896
|
+
return contentHost.listDirectories(baseDir)
|
897
|
+
end
|
898
|
+
|
899
|
+
# list all the file hashes of the files within the base directory
|
900
|
+
def listFileHashes
|
901
|
+
return contentHost.listFileHashes(baseDir)
|
902
|
+
end
|
903
|
+
|
904
|
+
def to_s
|
905
|
+
return contentHost.locationDescriptor(baseDir)
|
906
|
+
end
|
907
|
+
|
908
|
+
# Get the content tree, from the cached content file if it exists,
|
909
|
+
# otherwise get if from listing directories and files and hash values thereof
|
910
|
+
# on the remote host. And also, if the cached content file name is specified,
|
911
|
+
# write the content tree out to that file.
|
912
|
+
def getContentTree
|
913
|
+
if cachedContentFile and File.exists?(cachedContentFile)
|
914
|
+
return ContentTree.readFromFile(cachedContentFile)
|
915
|
+
else
|
916
|
+
contentTree = contentHost.getContentTree(baseDir)
|
917
|
+
contentTree.sort!
|
918
|
+
if cachedContentFile != nil
|
919
|
+
contentTree.writeToFile(cachedContentFile)
|
920
|
+
end
|
921
|
+
return contentTree
|
922
|
+
end
|
923
|
+
end
|
924
|
+
|
925
|
+
end
|
926
|
+
|
927
|
+
# The operation of synchronising files on the remote directory with files on the local directory.
|
928
|
+
class SyncOperation
|
929
|
+
# The source location (presumed to be local)
|
930
|
+
attr_reader :sourceLocation
|
931
|
+
|
932
|
+
# The destination location (presumed to be remote)
|
933
|
+
attr_reader :destinationLocation
|
934
|
+
|
935
|
+
def initialize(sourceLocation, destinationLocation)
|
936
|
+
@sourceLocation = sourceLocation
|
937
|
+
@destinationLocation = destinationLocation
|
938
|
+
end
|
939
|
+
|
940
|
+
# Get the local and remote content trees
|
941
|
+
def getContentTrees
|
942
|
+
@sourceContent = @sourceLocation.getContentTree()
|
943
|
+
@destinationContent = @destinationLocation.getContentTree()
|
944
|
+
end
|
945
|
+
|
946
|
+
# On the local and remote content trees, mark the copy and delete operations required
|
947
|
+
# to sync the remote location to the local location.
|
948
|
+
def markSyncOperations
|
949
|
+
@sourceContent.markSyncOperationsForDestination(@destinationContent)
|
950
|
+
puts " ================================================ "
|
951
|
+
puts "After marking for sync --"
|
952
|
+
puts ""
|
953
|
+
puts "Local:"
|
954
|
+
@sourceContent.showIndented()
|
955
|
+
puts ""
|
956
|
+
puts "Remote:"
|
957
|
+
@destinationContent.showIndented()
|
958
|
+
end
|
959
|
+
|
960
|
+
# Delete the local and remote cached content files (which will force a full recalculation
|
961
|
+
# of both content trees next time)
|
962
|
+
def clearCachedContentFiles
|
963
|
+
@sourceLocation.clearCachedContentFile()
|
964
|
+
@destinationLocation.clearCachedContentFile()
|
965
|
+
end
|
966
|
+
|
967
|
+
# Do the sync. Options: :full = true means clear the cached content files first, :dryRun
|
968
|
+
# means don't do the actual copies and deletes, but just show what they would be.
|
969
|
+
def doSync(options = {})
|
970
|
+
if options[:full]
|
971
|
+
clearCachedContentFiles()
|
972
|
+
end
|
973
|
+
getContentTrees()
|
974
|
+
markSyncOperations()
|
975
|
+
dryRun = options[:dryRun]
|
976
|
+
if not dryRun
|
977
|
+
@destinationLocation.clearCachedContentFile()
|
978
|
+
end
|
979
|
+
doAllCopyOperations(dryRun)
|
980
|
+
doAllDeleteOperations(dryRun)
|
981
|
+
if (not dryRun and @destinationLocation.cachedContentFile and @sourceLocation.cachedContentFile and
|
982
|
+
File.exists?(@sourceLocation.cachedContentFile))
|
983
|
+
FileUtils::Verbose.cp(@sourceLocation.cachedContentFile, @destinationLocation.cachedContentFile)
|
984
|
+
end
|
985
|
+
closeConnections()
|
986
|
+
end
|
987
|
+
|
988
|
+
# Do all the copy operations, copying local directories or files which are missing from the remote location
|
989
|
+
def doAllCopyOperations(dryRun)
|
990
|
+
doCopyOperations(@sourceContent, @destinationContent, dryRun)
|
991
|
+
end
|
992
|
+
|
993
|
+
# Do all delete operations, deleting remote directories or files which do not exist at the local location
|
994
|
+
def doAllDeleteOperations(dryRun)
|
995
|
+
doDeleteOperations(@destinationContent, dryRun)
|
996
|
+
end
|
997
|
+
|
998
|
+
# Execute a (local) command, or, if dryRun, just pretend to execute it.
|
999
|
+
# Raise an exception if the process exit status is not 0.
|
1000
|
+
def executeCommand(command, dryRun)
|
1001
|
+
puts "EXECUTE: #{command}"
|
1002
|
+
if not dryRun
|
1003
|
+
system(command)
|
1004
|
+
checkProcessStatus(command)
|
1005
|
+
end
|
1006
|
+
end
|
1007
|
+
|
1008
|
+
# Recursively perform all marked copy operations from the source content tree to the
|
1009
|
+
# destination content tree, or if dryRun, just pretend to perform them.
|
1010
|
+
def doCopyOperations(sourceContent, destinationContent, dryRun)
|
1011
|
+
for dir in sourceContent.dirs
|
1012
|
+
if dir.copyDestination != nil
|
1013
|
+
sourcePath = sourceLocation.getFullPath(dir.relativePath)
|
1014
|
+
destinationPath = destinationLocation.getFullPath(dir.copyDestination.relativePath)
|
1015
|
+
destinationLocation.contentHost.copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
1016
|
+
else
|
1017
|
+
doCopyOperations(dir, destinationContent.getDir(dir.name), dryRun)
|
1018
|
+
end
|
1019
|
+
end
|
1020
|
+
for file in sourceContent.files
|
1021
|
+
if file.copyDestination != nil
|
1022
|
+
sourcePath = sourceLocation.getFullPath(file.relativePath)
|
1023
|
+
destinationPath = destinationLocation.getFullPath(file.copyDestination.relativePath)
|
1024
|
+
destinationLocation.contentHost.copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
|
1025
|
+
end
|
1026
|
+
end
|
1027
|
+
end
|
1028
|
+
|
1029
|
+
# Recursively perform all marked delete operations on the destination content tree,
|
1030
|
+
# or if dryRun, just pretend to perform them.
|
1031
|
+
def doDeleteOperations(destinationContent, dryRun)
|
1032
|
+
for dir in destinationContent.dirs
|
1033
|
+
if dir.toBeDeleted
|
1034
|
+
dirPath = destinationLocation.getFullPath(dir.relativePath)
|
1035
|
+
destinationLocation.contentHost.deleteDirectory(dirPath, dryRun)
|
1036
|
+
else
|
1037
|
+
doDeleteOperations(dir, dryRun)
|
1038
|
+
end
|
1039
|
+
end
|
1040
|
+
for file in destinationContent.files
|
1041
|
+
if file.toBeDeleted
|
1042
|
+
filePath = destinationLocation.getFullPath(file.relativePath)
|
1043
|
+
destinationLocation.contentHost.deleteFile(filePath, dryRun)
|
1044
|
+
end
|
1045
|
+
end
|
1046
|
+
end
|
1047
|
+
|
1048
|
+
def closeConnections
|
1049
|
+
destinationLocation.closeConnections()
|
1050
|
+
end
|
1051
|
+
end
|
1052
|
+
end
|