synqa 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +1 -1
- data/VERSION +1 -1
- data/lib/synqa.rb +213 -19
- metadata +11 -11
data/Rakefile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
data/lib/synqa.rb
CHANGED
@@ -2,6 +2,7 @@ require 'time'
|
|
2
2
|
|
3
3
|
module Synqa
|
4
4
|
|
5
|
+
# Check if the last executed process exited with status 0, if not, raise an exception
|
5
6
|
def checkProcessStatus(description)
|
6
7
|
processStatus = $?
|
7
8
|
if not processStatus.exited?
|
@@ -13,8 +14,13 @@ module Synqa
|
|
13
14
|
end
|
14
15
|
end
|
15
16
|
|
17
|
+
# An object representing a file path relative to a base directory, and a hash string
|
16
18
|
class RelativePathWithHash
|
17
|
-
|
19
|
+
# The relative file path (e.g. c:/dir/subdir/file.txt relative to c:/dir would be subdir/file.txt)
|
20
|
+
attr_reader :relativePath
|
21
|
+
|
22
|
+
# The hash code, e.g. a1c5b67fdb3cf0df8f1d29ae90561f9ad099bada44aeb6b2574ad9e15f2a84ed
|
23
|
+
attr_reader :hash
|
18
24
|
|
19
25
|
def initialize(relativePath, hash)
|
20
26
|
@relativePath = relativePath
|
@@ -25,17 +31,26 @@ module Synqa
|
|
25
31
|
return "RelativePathWithHash[#{relativePath}, #{hash}]"
|
26
32
|
end
|
27
33
|
end
|
28
|
-
|
34
|
+
|
35
|
+
# A command to be executed on the remote system which calculates a hash value for
|
36
|
+
# a file (of a given length), in the format: *hexadecimal-hash* *a-fixed-number-of-characters* *file-name*
|
29
37
|
class HashCommand
|
38
|
+
# The command - a string or array of strings e.g. "sha256sum" or ["sha256", "-r"]
|
39
|
+
attr_reader :command
|
40
|
+
|
41
|
+
# The length of the calculated hash value e.g. 64 for sha256
|
42
|
+
attr_reader :length
|
30
43
|
|
31
|
-
|
44
|
+
# The number of characters between the hash value and the file name (usually 1 or 2)
|
45
|
+
attr_reader :spacerLen
|
32
46
|
|
33
47
|
def initialize(command, length, spacerLen)
|
34
48
|
@command = command
|
35
49
|
@length = length
|
36
50
|
@spacerLen = spacerLen
|
37
51
|
end
|
38
|
-
|
52
|
+
|
53
|
+
# Parse a hash line relative to a base directory, returning a RelativePathWithHash
|
39
54
|
def parseFileHashLine(baseDir, fileHashLine)
|
40
55
|
hash = fileHashLine[0...length]
|
41
56
|
fullPath = fileHashLine[(length + spacerLen)..-1]
|
@@ -51,35 +66,50 @@ module Synqa
|
|
51
66
|
end
|
52
67
|
end
|
53
68
|
|
69
|
+
# Hash command for sha256sum, which generates a 64 hexadecimal digit hash, and outputs two characters between
|
70
|
+
# the hash and the file name.
|
54
71
|
class Sha256SumCommand<HashCommand
|
55
72
|
def initialize
|
56
73
|
super(["sha256sum"], 64, 2)
|
57
74
|
end
|
58
75
|
end
|
59
76
|
|
77
|
+
# Hash command for sha256, which generates a 64 hexadecimal digit hash, and outputs one character between
|
78
|
+
# the hash and the file name, and which requires a "-r" argument to put the hash value first.
|
60
79
|
class Sha256Command<HashCommand
|
61
80
|
def initialize
|
62
81
|
super(["sha256", "-r"], 64, 1)
|
63
82
|
end
|
64
83
|
end
|
65
84
|
|
85
|
+
# Put "/" at the end of a directory name if it is not already there.
|
66
86
|
def normalisedDir(baseDir)
|
67
87
|
return baseDir.end_with?("/") ? baseDir : baseDir + "/"
|
68
88
|
end
|
69
89
|
|
90
|
+
|
91
|
+
# Base class for an object representing a remote system where the contents of a directory
|
92
|
+
# on the system are enumerated by one command to list all sub-directories and another command
|
93
|
+
# to list all files in the directory and their hash values.
|
70
94
|
class DirContentHost
|
71
95
|
|
72
|
-
|
96
|
+
# The HashCommand object used to calculate and parse hash values of files
|
97
|
+
attr_reader :hashCommand
|
98
|
+
|
99
|
+
# Prefix required for *find* command (usually nothing, since it should be on the system path)
|
100
|
+
attr_reader :pathPrefix
|
73
101
|
|
74
102
|
def initialize(hashCommand, pathPrefix = "")
|
75
103
|
@hashCommand = hashCommand
|
76
104
|
@pathPrefix = pathPrefix
|
77
105
|
end
|
78
106
|
|
107
|
+
# Generate the *find* command which will list all the sub-directories of the base directory
|
79
108
|
def findDirectoriesCommand(baseDir)
|
80
109
|
return ["#{@pathPrefix}find", baseDir, "-type", "d", "-print"]
|
81
110
|
end
|
82
111
|
|
112
|
+
# Return the list of sub-directories relative to the base directory
|
83
113
|
def listDirectories(baseDir)
|
84
114
|
baseDir = normalisedDir(baseDir)
|
85
115
|
command = findDirectoriesCommand(baseDir)
|
@@ -101,10 +131,13 @@ module Synqa
|
|
101
131
|
return directories
|
102
132
|
end
|
103
133
|
|
134
|
+
# Generate the *find* command which will list all the files within the base directory
|
104
135
|
def findFilesCommand(baseDir)
|
105
136
|
return ["#{@pathPrefix}find", baseDir, "-type", "f", "-print"]
|
106
137
|
end
|
107
|
-
|
138
|
+
|
139
|
+
# List file hashes by executing the command to hash each file on the output of the
|
140
|
+
# *find* command which lists all files, and parse the output.
|
108
141
|
def listFileHashes(baseDir)
|
109
142
|
baseDir = normalisedDir(baseDir)
|
110
143
|
fileHashes = []
|
@@ -117,11 +150,13 @@ module Synqa
|
|
117
150
|
return fileHashes
|
118
151
|
end
|
119
152
|
|
153
|
+
# Return the enumerated lines of the command's output
|
120
154
|
def getCommandOutput(command)
|
121
155
|
puts "#{command.inspect} ..."
|
122
156
|
return IO.popen(command)
|
123
157
|
end
|
124
158
|
|
159
|
+
# Construct the ContentTree for the given base directory
|
125
160
|
def getContentTree(baseDir)
|
126
161
|
contentTree = ContentTree.new()
|
127
162
|
contentTree.time = Time.now.utc
|
@@ -135,9 +170,20 @@ module Synqa
|
|
135
170
|
end
|
136
171
|
end
|
137
172
|
|
173
|
+
# Representation of a remote system accessible via SSH
|
138
174
|
class SshContentHost<DirContentHost
|
139
175
|
|
140
|
-
|
176
|
+
# The SSH client, e.g. ["ssh"] or ["plink","-pw","mysecretpassword"] (i.e. command + args as an array)
|
177
|
+
attr_reader :shell
|
178
|
+
|
179
|
+
# The SCP client, e.g. ["scp"] or ["pscp","-pw","mysecretpassword"] (i.e. command + args as an array)
|
180
|
+
attr_reader :scpProgram
|
181
|
+
|
182
|
+
# The remote host, e.g. "username@host.example.com"
|
183
|
+
attr_reader :host
|
184
|
+
|
185
|
+
# The SCP command as a string
|
186
|
+
attr_reader :scpCommandString
|
141
187
|
|
142
188
|
def initialize(host, hashCommand, shell, scpProgram)
|
143
189
|
super(hashCommand)
|
@@ -147,11 +193,14 @@ module Synqa
|
|
147
193
|
@scpCommandString = @scpProgram.join(" ")
|
148
194
|
end
|
149
195
|
|
196
|
+
# Return readable description of base directory on remote system
|
150
197
|
def locationDescriptor(baseDir)
|
151
198
|
baseDir = normalisedDir(baseDir)
|
152
199
|
return "#{host}:#{baseDir} (connect = #{shell}/#{scpProgram}, hashCommand = #{hashCommand})"
|
153
200
|
end
|
154
201
|
|
202
|
+
# execute an SSH command on the remote system, yielding lines of output
|
203
|
+
# (or don't actually execute, if dryRun is true)
|
155
204
|
def executeRemoteCommand(commandString, dryRun = false)
|
156
205
|
puts "SSH #{host} (#{shell.join(" ")}): executing #{commandString}"
|
157
206
|
if not dryRun
|
@@ -164,12 +213,15 @@ module Synqa
|
|
164
213
|
end
|
165
214
|
end
|
166
215
|
|
216
|
+
# execute an SSH command on the remote system, displaying output to stdout,
|
217
|
+
# (or don't actually execute, if dryRun is true)
|
167
218
|
def ssh(commandString, dryRun = false)
|
168
219
|
executeRemoteCommand(commandString, dryRun) do |line|
|
169
220
|
puts line
|
170
221
|
end
|
171
222
|
end
|
172
223
|
|
224
|
+
# Return a list of all subdirectories of the base directory (as paths relative to the base directory)
|
173
225
|
def listDirectories(baseDir)
|
174
226
|
baseDir = normalisedDir(baseDir)
|
175
227
|
puts "Listing directories ..."
|
@@ -186,6 +238,8 @@ module Synqa
|
|
186
238
|
return directories
|
187
239
|
end
|
188
240
|
|
241
|
+
# Yield lines of output from the command to display hash values and file names
|
242
|
+
# of all files within the base directory
|
189
243
|
def listFileHashLines(baseDir)
|
190
244
|
baseDir = normalisedDir(baseDir)
|
191
245
|
remoteFileHashLinesCommand = findFilesCommand(baseDir) + ["|", "xargs", "-r"] + @hashCommand.command
|
@@ -195,6 +249,7 @@ module Synqa
|
|
195
249
|
end
|
196
250
|
end
|
197
251
|
|
252
|
+
# List all files within the base directory to stdout
|
198
253
|
def listFiles(baseDir)
|
199
254
|
baseDir = normalisedDir(baseDir)
|
200
255
|
executeRemoteCommand(findFilesCommand(baseDir).join(" ")) do |line|
|
@@ -202,13 +257,30 @@ module Synqa
|
|
202
257
|
end
|
203
258
|
end
|
204
259
|
|
260
|
+
# Get the remote path of the directory or file on the host, in the format required by SCP
|
205
261
|
def getScpPath(path)
|
206
262
|
return host + ":" + path
|
207
263
|
end
|
208
264
|
end
|
209
265
|
|
266
|
+
# An object representing the content of a file within a ContentTree.
|
267
|
+
# The file may be marked for copying (if it's in a source ContentTree)
|
268
|
+
# or for deletion (if it's in a destination ContentTree)
|
210
269
|
class FileContent
|
211
|
-
|
270
|
+
# The name of the file
|
271
|
+
attr_reader :name
|
272
|
+
|
273
|
+
# The hash value of the file's contents
|
274
|
+
attr_reader :hash
|
275
|
+
|
276
|
+
# The components of the relative path where the file is found
|
277
|
+
attr_reader :parentPathElements
|
278
|
+
|
279
|
+
# The destination to which the file should be copied
|
280
|
+
attr_reader :copyDestination
|
281
|
+
|
282
|
+
# Should this file be deleted
|
283
|
+
attr_reader :toBeDeleted
|
212
284
|
|
213
285
|
def initialize(name, hash, parentPathElements)
|
214
286
|
@name = name
|
@@ -218,10 +290,12 @@ module Synqa
|
|
218
290
|
@toBeDeleted = false
|
219
291
|
end
|
220
292
|
|
293
|
+
# Mark this file to be copied to a destination directory (from a destination content tree)
|
221
294
|
def markToCopy(destinationDirectory)
|
222
295
|
@copyDestination = destinationDirectory
|
223
296
|
end
|
224
297
|
|
298
|
+
# Mark this file to be deleted
|
225
299
|
def markToDelete
|
226
300
|
@toBeDeleted = true
|
227
301
|
end
|
@@ -230,14 +304,43 @@ module Synqa
|
|
230
304
|
return "#{name} (#{hash})"
|
231
305
|
end
|
232
306
|
|
307
|
+
# The full (relative) name of this file in the content tree
|
233
308
|
def fullPath
|
234
309
|
return (parentPathElements + [name]).join("/")
|
235
310
|
end
|
236
311
|
end
|
237
312
|
|
313
|
+
# A "content tree" consisting of a description of the contents of files and
|
314
|
+
# sub-directories within a base directory. The file contents are described via
|
315
|
+
# cryptographic hash values.
|
316
|
+
# Each sub-directory within a content tree is also represented as a ContentTree.
|
238
317
|
class ContentTree
|
239
|
-
|
240
|
-
attr_reader :
|
318
|
+
# name of the sub-directory within the containing directory (or nil if this is the base directory)
|
319
|
+
attr_reader :name
|
320
|
+
|
321
|
+
# path elements from base directory leading to this one
|
322
|
+
attr_reader :pathElements
|
323
|
+
|
324
|
+
# files within this sub-directory (as FileContent's)
|
325
|
+
attr_reader :files
|
326
|
+
|
327
|
+
# immediate sub-directories of this directory
|
328
|
+
attr_reader :dirs
|
329
|
+
|
330
|
+
# the files within this sub-directory, indexed by file name
|
331
|
+
attr_reader :fileByName
|
332
|
+
|
333
|
+
# immediate sub-directories of this directory, indexed by name
|
334
|
+
attr_reader :dirByName
|
335
|
+
|
336
|
+
# where this directory should be copied to
|
337
|
+
attr_reader :copyDestination
|
338
|
+
|
339
|
+
# whether this directory should be deleted
|
340
|
+
attr_reader :toBeDeleted
|
341
|
+
|
342
|
+
# the UTC time (on the local system, even if this content tree represents a remote directory)
|
343
|
+
# that this content tree was constructed. Only set for the base directory.
|
241
344
|
attr_accessor :time
|
242
345
|
|
243
346
|
def initialize(name = nil, parentPathElements = nil)
|
@@ -252,22 +355,27 @@ module Synqa
|
|
252
355
|
@time = nil
|
253
356
|
end
|
254
357
|
|
358
|
+
# mark this directory to be copied to a destination directory
|
255
359
|
def markToCopy(destinationDirectory)
|
256
360
|
@copyDestination = destinationDirectory
|
257
361
|
end
|
258
362
|
|
363
|
+
# mark this directory (on a remote system) to be deleted
|
259
364
|
def markToDelete
|
260
365
|
@toBeDeleted = true
|
261
366
|
end
|
262
367
|
|
368
|
+
# the full path of the directory that this content tree represents (relative to the base directory)
|
263
369
|
def fullPath
|
264
370
|
return @pathElements.join("/")
|
265
371
|
end
|
266
372
|
|
373
|
+
# convert a path string to an array of path elements (or return it as is if it's already an array)
|
267
374
|
def getPathElements(path)
|
268
375
|
return path.is_a?(String) ? (path == "" ? [] : path.split("/")) : path
|
269
376
|
end
|
270
377
|
|
378
|
+
# get the content tree for a sub-directory (creating it if it doesn't yet exist)
|
271
379
|
def getContentTreeForSubDir(subDir)
|
272
380
|
dirContentTree = dirByName.fetch(subDir, nil)
|
273
381
|
if dirContentTree == nil
|
@@ -278,6 +386,7 @@ module Synqa
|
|
278
386
|
return dirContentTree
|
279
387
|
end
|
280
388
|
|
389
|
+
# add a sub-directory to this content tree
|
281
390
|
def addDir(dirPath)
|
282
391
|
pathElements = getPathElements(dirPath)
|
283
392
|
if pathElements.length > 0
|
@@ -287,6 +396,7 @@ module Synqa
|
|
287
396
|
end
|
288
397
|
end
|
289
398
|
|
399
|
+
# recursively sort the files and sub-directories of this content tree alphabetically
|
290
400
|
def sort!
|
291
401
|
dirs.sort_by! {|dir| dir.name}
|
292
402
|
files.sort_by! {|file| file.name}
|
@@ -295,6 +405,7 @@ module Synqa
|
|
295
405
|
end
|
296
406
|
end
|
297
407
|
|
408
|
+
# given a relative path, add a file and hash value to this content tree
|
298
409
|
def addFile(filePath, hash)
|
299
410
|
pathElements = getPathElements(filePath)
|
300
411
|
if pathElements.length == 0
|
@@ -312,8 +423,10 @@ module Synqa
|
|
312
423
|
end
|
313
424
|
end
|
314
425
|
|
426
|
+
# date-time format for reading and writing times, e.g. "2007-12-23 13:03:99.012 +0000"
|
315
427
|
@@dateTimeFormat = "%Y-%m-%d %H:%M:%S.%L %z"
|
316
428
|
|
429
|
+
# pretty-print this content tree
|
317
430
|
def showIndented(name = "", indent = " ", currentIndent = "")
|
318
431
|
if time != nil
|
319
432
|
puts "#{currentIndent}[TIME: #{time.strftime(@@dateTimeFormat)}]"
|
@@ -341,7 +454,8 @@ module Synqa
|
|
341
454
|
end
|
342
455
|
end
|
343
456
|
end
|
344
|
-
|
457
|
+
|
458
|
+
# write this content tree to an open file, indented
|
345
459
|
def writeLinesToFile(outFile, prefix = "")
|
346
460
|
if time != nil
|
347
461
|
outFile.puts("T #{time.strftime(@@dateTimeFormat)}\n")
|
@@ -355,6 +469,7 @@ module Synqa
|
|
355
469
|
end
|
356
470
|
end
|
357
471
|
|
472
|
+
# write this content tree to a file (in a format which readFromFile can read back in)
|
358
473
|
def writeToFile(fileName)
|
359
474
|
puts "Writing content tree to file #{fileName} ..."
|
360
475
|
File.open(fileName, "w") do |outFile|
|
@@ -362,10 +477,16 @@ module Synqa
|
|
362
477
|
end
|
363
478
|
end
|
364
479
|
|
480
|
+
# regular expression for directory entries in content tree file
|
365
481
|
@@dirLineRegex = /^D (.*)$/
|
482
|
+
|
483
|
+
# regular expression for file entries in content tree file
|
366
484
|
@@fileLineRegex = /^F ([^ ]*) (.*)$/
|
485
|
+
|
486
|
+
# regular expression for time entry in content tree file
|
367
487
|
@@timeRegex = /^T (.*)$/
|
368
488
|
|
489
|
+
# read a content tree from a file (in format written by writeToFile)
|
369
490
|
def self.readFromFile(fileName)
|
370
491
|
contentTree = ContentTree.new()
|
371
492
|
puts "Reading content tree from #{fileName} ..."
|
@@ -393,7 +514,9 @@ module Synqa
|
|
393
514
|
end
|
394
515
|
return contentTree
|
395
516
|
end
|
396
|
-
|
517
|
+
|
518
|
+
# read a content tree as a map of hashes, i.e. from relative file path to hash value for the file
|
519
|
+
# Actually returns an array of the time entry (if any) and the map of hashes
|
397
520
|
def self.readMapOfHashesFromFile(fileName)
|
398
521
|
mapOfHashes = {}
|
399
522
|
time = nil
|
@@ -413,19 +536,26 @@ module Synqa
|
|
413
536
|
return [time, mapOfHashes]
|
414
537
|
end
|
415
538
|
|
539
|
+
# Mark operations for this (source) content tree and the destination content tree
|
540
|
+
# in order to synch the destination content tree with this one
|
416
541
|
def markSyncOperationsForDestination(destination)
|
417
542
|
markCopyOperations(destination)
|
418
543
|
destination.markDeleteOptions(self)
|
419
544
|
end
|
420
545
|
|
546
|
+
# Get the named sub-directory content tree, if it exists
|
421
547
|
def getDir(dir)
|
422
548
|
return dirByName.fetch(dir, nil)
|
423
549
|
end
|
424
550
|
|
551
|
+
# Get the named file & hash value, if it exists
|
425
552
|
def getFile(file)
|
426
553
|
return fileByName.fetch(file, nil)
|
427
554
|
end
|
428
555
|
|
556
|
+
# Mark copy operations, given that the corresponding destination directory already exists.
|
557
|
+
# For files and directories that don't exist in the destination, mark them to be copied.
|
558
|
+
# For sub-directories that do exist, recursively mark the corresponding sub-directory copy operations.
|
429
559
|
def markCopyOperations(destinationDir)
|
430
560
|
for dir in dirs
|
431
561
|
destinationSubDir = destinationDir.getDir(dir.name)
|
@@ -443,6 +573,9 @@ module Synqa
|
|
443
573
|
end
|
444
574
|
end
|
445
575
|
|
576
|
+
# Mark delete operations, given that the corresponding source directory exists.
|
577
|
+
# For files and directories that don't exist in the source, mark them to be deleted.
|
578
|
+
# For sub-directories that do exist, recursively mark the corresponding sub-directory delete operations.
|
446
579
|
def markDeleteOptions(sourceDir)
|
447
580
|
for dir in dirs
|
448
581
|
sourceSubDir = sourceDir.getDir(dir.name)
|
@@ -461,13 +594,18 @@ module Synqa
|
|
461
594
|
end
|
462
595
|
end
|
463
596
|
|
597
|
+
# Base class for a content location which consists of a base directory
|
598
|
+
# on a local or remote system.
|
464
599
|
class ContentLocation
|
600
|
+
|
601
|
+
# The name of a file used to hold a cached content tree for this location (can optionally be specified)
|
465
602
|
attr_reader :cachedContentFile
|
466
603
|
|
467
604
|
def initialize(cachedContentFile)
|
468
605
|
@cachedContentFile = cachedContentFile
|
469
606
|
end
|
470
607
|
|
608
|
+
# Get the cached content file name, if specified, and if the file exists
|
471
609
|
def getExistingCachedContentTreeFile
|
472
610
|
if cachedContentFile == nil
|
473
611
|
puts "No cached content file specified for location"
|
@@ -480,6 +618,7 @@ module Synqa
|
|
480
618
|
end
|
481
619
|
end
|
482
620
|
|
621
|
+
# Delete any existing cached content file
|
483
622
|
def clearCachedContentFile
|
484
623
|
if cachedContentFile and File.exists?(cachedContentFile)
|
485
624
|
puts " deleting cached content file #{cachedContentFile} ..."
|
@@ -487,6 +626,7 @@ module Synqa
|
|
487
626
|
end
|
488
627
|
end
|
489
628
|
|
629
|
+
# Get the cached content tree (if any), read from the specified cached content file.
|
490
630
|
def getCachedContentTree
|
491
631
|
file = getExistingCachedContentTreeFile
|
492
632
|
if file
|
@@ -496,6 +636,8 @@ module Synqa
|
|
496
636
|
end
|
497
637
|
end
|
498
638
|
|
639
|
+
# Read a map of file hashes (mapping from relative file name to hash value) from the
|
640
|
+
# specified cached content file
|
499
641
|
def getCachedContentTreeMapOfHashes
|
500
642
|
file = getExistingCachedContentTreeFile
|
501
643
|
if file
|
@@ -508,8 +650,14 @@ module Synqa
|
|
508
650
|
|
509
651
|
end
|
510
652
|
|
653
|
+
# A directory of files on a local system. The corresponding content tree
|
654
|
+
# can be calculated directly using Ruby library functions.
|
511
655
|
class LocalContentLocation<ContentLocation
|
512
|
-
|
656
|
+
|
657
|
+
# the base directory
|
658
|
+
attr_reader :baseDir
|
659
|
+
# the ruby class that generates the hash, e.g. Digest::SHA256
|
660
|
+
attr_reader :hashClass
|
513
661
|
|
514
662
|
def initialize(baseDir, hashClass, cachedContentFile = nil, options = {})
|
515
663
|
super(cachedContentFile)
|
@@ -519,6 +667,7 @@ module Synqa
|
|
519
667
|
@excludeGlobs = options.fetch(:excludes, [])
|
520
668
|
end
|
521
669
|
|
670
|
+
# get the path of a file name relative to the base directory
|
522
671
|
def getRelativePath(fileName)
|
523
672
|
if fileName.start_with? @baseDir
|
524
673
|
return fileName[@baseDirLen..-1]
|
@@ -527,15 +676,18 @@ module Synqa
|
|
527
676
|
end
|
528
677
|
end
|
529
678
|
|
679
|
+
# get the path as required for an SCP command
|
530
680
|
def getScpPath(relativePath)
|
531
681
|
return getFullPath(relativePath)
|
532
682
|
end
|
533
683
|
|
684
|
+
# get the full path of a relative path (i.e. of a file/directory within the base directory)
|
534
685
|
def getFullPath(relativePath)
|
535
686
|
return @baseDir + relativePath
|
536
687
|
end
|
537
688
|
|
538
|
-
|
689
|
+
# is the relative path name excluded by one of the specified exclusion globs?
|
690
|
+
def fileIsExcluded?(relativeFile)
|
539
691
|
for excludeGlob in @excludeGlobs
|
540
692
|
if File.fnmatch(excludeGlob, relativeFile)
|
541
693
|
puts " file #{relativeFile} excluded by glob #{excludeGlob}"
|
@@ -545,6 +697,12 @@ module Synqa
|
|
545
697
|
return false
|
546
698
|
end
|
547
699
|
|
700
|
+
# get the content tree for this base directory by iterating over all
|
701
|
+
# sub-directories and files within the base directory (and excluding the excluded files)
|
702
|
+
# and calculating file hashes using the specified Ruby hash class
|
703
|
+
# If there is an existing cached content file, use that to get the hash values
|
704
|
+
# of files whose modification time is earlier than the time value for the cached content tree.
|
705
|
+
# Also, if a cached content file is specified, write the final content tree back out to the cached content file.
|
548
706
|
def getContentTree
|
549
707
|
cachedTimeAndMapOfHashes = getCachedContentTreeMapOfHashes
|
550
708
|
cachedTime = cachedTimeAndMapOfHashes[0]
|
@@ -559,7 +717,7 @@ module Synqa
|
|
559
717
|
if File.directory? fileOrDir
|
560
718
|
contentTree.addDir(relativePath)
|
561
719
|
else
|
562
|
-
if not fileIsExcluded(relativePath)
|
720
|
+
if not fileIsExcluded?(relativePath)
|
563
721
|
cachedDigest = cachedMapOfHashes[relativePath]
|
564
722
|
if cachedTime and cachedDigest and File.stat(fileOrDir).mtime < cachedTime
|
565
723
|
digest = cachedDigest
|
@@ -579,8 +737,13 @@ module Synqa
|
|
579
737
|
end
|
580
738
|
end
|
581
739
|
|
740
|
+
# A directory of files on a remote system
|
582
741
|
class RemoteContentLocation<ContentLocation
|
583
|
-
|
742
|
+
# the remote username@host value
|
743
|
+
attr_reader :host
|
744
|
+
|
745
|
+
# the base directory on the remote system
|
746
|
+
attr_reader :baseDir
|
584
747
|
|
585
748
|
def initialize(host, baseDir, cachedContentFile = nil)
|
586
749
|
super(cachedContentFile)
|
@@ -588,30 +751,37 @@ module Synqa
|
|
588
751
|
@baseDir = normalisedDir(baseDir)
|
589
752
|
end
|
590
753
|
|
754
|
+
# list files within the base directory on the remote host
|
591
755
|
def listFiles()
|
592
756
|
host.listFiles(baseDir)
|
593
757
|
end
|
594
758
|
|
759
|
+
# the command string required to execute SCP (e.g. "scp" or "pscp", possibly with extra args)
|
595
760
|
def scpCommandString
|
596
761
|
return host.scpCommandString
|
597
762
|
end
|
598
763
|
|
764
|
+
# get the full path of a relative path
|
599
765
|
def getFullPath(relativePath)
|
600
766
|
return baseDir + relativePath
|
601
767
|
end
|
602
768
|
|
769
|
+
# get the full path of a file as required in an SCP command (i.e. with username@host prepended)
|
603
770
|
def getScpPath(relativePath)
|
604
771
|
return host.getScpPath(getFullPath(relativePath))
|
605
772
|
end
|
606
773
|
|
774
|
+
# execute an SSH command on the remote host (or just pretend, if dryRun is true)
|
607
775
|
def ssh(commandString, dryRun = false)
|
608
776
|
host.ssh(commandString, dryRun)
|
609
777
|
end
|
610
778
|
|
779
|
+
# list all sub-directories of the base directory on the remote host
|
611
780
|
def listDirectories
|
612
781
|
return host.listDirectories(baseDir)
|
613
782
|
end
|
614
783
|
|
784
|
+
# list all the file hashes of the files within the base directory
|
615
785
|
def listFileHashes
|
616
786
|
return host.listFileHashes(baseDir)
|
617
787
|
end
|
@@ -619,7 +789,11 @@ module Synqa
|
|
619
789
|
def to_s
|
620
790
|
return host.locationDescriptor(baseDir)
|
621
791
|
end
|
622
|
-
|
792
|
+
|
793
|
+
# Get the content tree, from the cached content file if it exists,
|
794
|
+
# otherwise get if from listing directories and files and hash values thereof
|
795
|
+
# on the remote host. And also, if the cached content file name is specified,
|
796
|
+
# write the content tree out to that file.
|
623
797
|
def getContentTree
|
624
798
|
if cachedContentFile and File.exists?(cachedContentFile)
|
625
799
|
return ContentTree.readFromFile(cachedContentFile)
|
@@ -635,19 +809,27 @@ module Synqa
|
|
635
809
|
|
636
810
|
end
|
637
811
|
|
812
|
+
# The operation of synchronising files on the remote directory with files on the local directory.
|
638
813
|
class SyncOperation
|
639
|
-
|
814
|
+
# The source location (presumed to be local)
|
815
|
+
attr_reader :sourceLocation
|
816
|
+
|
817
|
+
# The destination location (presumed to be remote)
|
818
|
+
attr_reader :destinationLocation
|
640
819
|
|
641
820
|
def initialize(sourceLocation, destinationLocation)
|
642
821
|
@sourceLocation = sourceLocation
|
643
822
|
@destinationLocation = destinationLocation
|
644
823
|
end
|
645
824
|
|
825
|
+
# Get the local and remote content trees
|
646
826
|
def getContentTrees
|
647
827
|
@sourceContent = @sourceLocation.getContentTree()
|
648
828
|
@destinationContent = @destinationLocation.getContentTree()
|
649
829
|
end
|
650
830
|
|
831
|
+
# On the local and remote content trees, mark the copy and delete operations required
|
832
|
+
# to sync the remote location to the local location.
|
651
833
|
def markSyncOperations
|
652
834
|
@sourceContent.markSyncOperationsForDestination(@destinationContent)
|
653
835
|
puts " ================================================ "
|
@@ -660,11 +842,15 @@ module Synqa
|
|
660
842
|
@destinationContent.showIndented()
|
661
843
|
end
|
662
844
|
|
845
|
+
# Delete the local and remote cached content files (which will force a full recalculation
|
846
|
+
# of both content trees next time)
|
663
847
|
def clearCachedContentFiles
|
664
848
|
@sourceLocation.clearCachedContentFile()
|
665
849
|
@destinationLocation.clearCachedContentFile()
|
666
850
|
end
|
667
851
|
|
852
|
+
# Do the sync. Options: :full = true means clear the cached content files first, :dryRun
|
853
|
+
# means don't do the actual copies and deletes, but just show what they would be.
|
668
854
|
def doSync(options = {})
|
669
855
|
if options[:full]
|
670
856
|
clearCachedContentFiles()
|
@@ -682,15 +868,19 @@ module Synqa
|
|
682
868
|
FileUtils::Verbose.cp(@sourceLocation.cachedContentFile, @destinationLocation.cachedContentFile)
|
683
869
|
end
|
684
870
|
end
|
685
|
-
|
871
|
+
|
872
|
+
# Do all the copy operations, copying local directories or files which are missing from the remote location
|
686
873
|
def doAllCopyOperations(dryRun)
|
687
874
|
doCopyOperations(@sourceContent, @destinationContent, dryRun)
|
688
875
|
end
|
689
876
|
|
877
|
+
# Do all delete operations, deleting remote directories or files which do not exist at the local location
|
690
878
|
def doAllDeleteOperations(dryRun)
|
691
879
|
doDeleteOperations(@destinationContent, dryRun)
|
692
880
|
end
|
693
881
|
|
882
|
+
# Execute a (local) command, or, if dryRun, just pretend to execute it.
|
883
|
+
# Raise an exception if the process exit status is not 0.
|
694
884
|
def executeCommand(command, dryRun)
|
695
885
|
puts "EXECUTE: #{command}"
|
696
886
|
if not dryRun
|
@@ -699,6 +889,8 @@ module Synqa
|
|
699
889
|
end
|
700
890
|
end
|
701
891
|
|
892
|
+
# Recursively perform all marked copy operations from the source content tree to the
|
893
|
+
# destination content tree, or if dryRun, just pretend to perform them.
|
702
894
|
def doCopyOperations(sourceContent, destinationContent, dryRun)
|
703
895
|
for dir in sourceContent.dirs do
|
704
896
|
if dir.copyDestination != nil
|
@@ -718,6 +910,8 @@ module Synqa
|
|
718
910
|
end
|
719
911
|
end
|
720
912
|
|
913
|
+
# Recursively perform all marked delete operations on the destination content tree,
|
914
|
+
# or if dryRun, just pretend to perform them.
|
721
915
|
def doDeleteOperations(destinationContent, dryRun)
|
722
916
|
for dir in destinationContent.dirs do
|
723
917
|
if dir.toBeDeleted
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: synqa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,12 +9,12 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-02
|
12
|
+
date: 2011-03-02 00:00:00.000000000 +13:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: shoulda
|
17
|
-
requirement: &
|
17
|
+
requirement: &24960612 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
20
|
- - ! '>='
|
@@ -22,10 +22,10 @@ dependencies:
|
|
22
22
|
version: '0'
|
23
23
|
type: :development
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
25
|
+
version_requirements: *24960612
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
27
|
name: bundler
|
28
|
-
requirement: &
|
28
|
+
requirement: &24959928 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
31
31
|
- - ~>
|
@@ -33,10 +33,10 @@ dependencies:
|
|
33
33
|
version: 1.0.0
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
|
-
version_requirements: *
|
36
|
+
version_requirements: *24959928
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
38
|
name: jeweler
|
39
|
-
requirement: &
|
39
|
+
requirement: &24942648 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ~>
|
@@ -44,10 +44,10 @@ dependencies:
|
|
44
44
|
version: 1.5.2
|
45
45
|
type: :development
|
46
46
|
prerelease: false
|
47
|
-
version_requirements: *
|
47
|
+
version_requirements: *24942648
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
name: rcov
|
50
|
-
requirement: &
|
50
|
+
requirement: &24941832 !ruby/object:Gem::Requirement
|
51
51
|
none: false
|
52
52
|
requirements:
|
53
53
|
- - ! '>='
|
@@ -55,7 +55,7 @@ dependencies:
|
|
55
55
|
version: '0'
|
56
56
|
type: :development
|
57
57
|
prerelease: false
|
58
|
-
version_requirements: *
|
58
|
+
version_requirements: *24941832
|
59
59
|
description: Sync files from a local directory to a remote directory via SSH/SCP
|
60
60
|
email: http://www.1729.com/email.html
|
61
61
|
executables: []
|
@@ -93,7 +93,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
93
93
|
version: '0'
|
94
94
|
segments:
|
95
95
|
- 0
|
96
|
-
hash:
|
96
|
+
hash: 346342291
|
97
97
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
98
98
|
none: false
|
99
99
|
requirements:
|