synqa 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +1 -1
- data/VERSION +1 -1
- data/lib/synqa.rb +213 -19
- metadata +11 -11
data/Rakefile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
data/lib/synqa.rb
CHANGED
@@ -2,6 +2,7 @@ require 'time'
|
|
2
2
|
|
3
3
|
module Synqa
|
4
4
|
|
5
|
+
# Check if the last executed process exited with status 0, if not, raise an exception
|
5
6
|
def checkProcessStatus(description)
|
6
7
|
processStatus = $?
|
7
8
|
if not processStatus.exited?
|
@@ -13,8 +14,13 @@ module Synqa
|
|
13
14
|
end
|
14
15
|
end
|
15
16
|
|
17
|
+
# An object representing a file path relative to a base directory, and a hash string
|
16
18
|
class RelativePathWithHash
|
17
|
-
|
19
|
+
# The relative file path (e.g. c:/dir/subdir/file.txt relative to c:/dir would be subdir/file.txt)
|
20
|
+
attr_reader :relativePath
|
21
|
+
|
22
|
+
# The hash code, e.g. a1c5b67fdb3cf0df8f1d29ae90561f9ad099bada44aeb6b2574ad9e15f2a84ed
|
23
|
+
attr_reader :hash
|
18
24
|
|
19
25
|
def initialize(relativePath, hash)
|
20
26
|
@relativePath = relativePath
|
@@ -25,17 +31,26 @@ module Synqa
|
|
25
31
|
return "RelativePathWithHash[#{relativePath}, #{hash}]"
|
26
32
|
end
|
27
33
|
end
|
28
|
-
|
34
|
+
|
35
|
+
# A command to be executed on the remote system which calculates a hash value for
|
36
|
+
# a file (of a given length), in the format: *hexadecimal-hash* *a-fixed-number-of-characters* *file-name*
|
29
37
|
class HashCommand
|
38
|
+
# The command - a string or array of strings e.g. "sha256sum" or ["sha256", "-r"]
|
39
|
+
attr_reader :command
|
40
|
+
|
41
|
+
# The length of the calculated hash value e.g. 64 for sha256
|
42
|
+
attr_reader :length
|
30
43
|
|
31
|
-
|
44
|
+
# The number of characters between the hash value and the file name (usually 1 or 2)
|
45
|
+
attr_reader :spacerLen
|
32
46
|
|
33
47
|
def initialize(command, length, spacerLen)
|
34
48
|
@command = command
|
35
49
|
@length = length
|
36
50
|
@spacerLen = spacerLen
|
37
51
|
end
|
38
|
-
|
52
|
+
|
53
|
+
# Parse a hash line relative to a base directory, returning a RelativePathWithHash
|
39
54
|
def parseFileHashLine(baseDir, fileHashLine)
|
40
55
|
hash = fileHashLine[0...length]
|
41
56
|
fullPath = fileHashLine[(length + spacerLen)..-1]
|
@@ -51,35 +66,50 @@ module Synqa
|
|
51
66
|
end
|
52
67
|
end
|
53
68
|
|
69
|
+
# Hash command for sha256sum, which generates a 64 hexadecimal digit hash, and outputs two characters between
|
70
|
+
# the hash and the file name.
|
54
71
|
class Sha256SumCommand<HashCommand
|
55
72
|
def initialize
|
56
73
|
super(["sha256sum"], 64, 2)
|
57
74
|
end
|
58
75
|
end
|
59
76
|
|
77
|
+
# Hash command for sha256, which generates a 64 hexadecimal digit hash, and outputs one character between
|
78
|
+
# the hash and the file name, and which requires a "-r" argument to put the hash value first.
|
60
79
|
class Sha256Command<HashCommand
|
61
80
|
def initialize
|
62
81
|
super(["sha256", "-r"], 64, 1)
|
63
82
|
end
|
64
83
|
end
|
65
84
|
|
85
|
+
# Put "/" at the end of a directory name if it is not already there.
|
66
86
|
def normalisedDir(baseDir)
|
67
87
|
return baseDir.end_with?("/") ? baseDir : baseDir + "/"
|
68
88
|
end
|
69
89
|
|
90
|
+
|
91
|
+
# Base class for an object representing a remote system where the contents of a directory
|
92
|
+
# on the system are enumerated by one command to list all sub-directories and another command
|
93
|
+
# to list all files in the directory and their hash values.
|
70
94
|
class DirContentHost
|
71
95
|
|
72
|
-
|
96
|
+
# The HashCommand object used to calculate and parse hash values of files
|
97
|
+
attr_reader :hashCommand
|
98
|
+
|
99
|
+
# Prefix required for *find* command (usually nothing, since it should be on the system path)
|
100
|
+
attr_reader :pathPrefix
|
73
101
|
|
74
102
|
def initialize(hashCommand, pathPrefix = "")
|
75
103
|
@hashCommand = hashCommand
|
76
104
|
@pathPrefix = pathPrefix
|
77
105
|
end
|
78
106
|
|
107
|
+
# Generate the *find* command which will list all the sub-directories of the base directory
|
79
108
|
def findDirectoriesCommand(baseDir)
|
80
109
|
return ["#{@pathPrefix}find", baseDir, "-type", "d", "-print"]
|
81
110
|
end
|
82
111
|
|
112
|
+
# Return the list of sub-directories relative to the base directory
|
83
113
|
def listDirectories(baseDir)
|
84
114
|
baseDir = normalisedDir(baseDir)
|
85
115
|
command = findDirectoriesCommand(baseDir)
|
@@ -101,10 +131,13 @@ module Synqa
|
|
101
131
|
return directories
|
102
132
|
end
|
103
133
|
|
134
|
+
# Generate the *find* command which will list all the files within the base directory
|
104
135
|
def findFilesCommand(baseDir)
|
105
136
|
return ["#{@pathPrefix}find", baseDir, "-type", "f", "-print"]
|
106
137
|
end
|
107
|
-
|
138
|
+
|
139
|
+
# List file hashes by executing the command to hash each file on the output of the
|
140
|
+
# *find* command which lists all files, and parse the output.
|
108
141
|
def listFileHashes(baseDir)
|
109
142
|
baseDir = normalisedDir(baseDir)
|
110
143
|
fileHashes = []
|
@@ -117,11 +150,13 @@ module Synqa
|
|
117
150
|
return fileHashes
|
118
151
|
end
|
119
152
|
|
153
|
+
# Return the enumerated lines of the command's output
|
120
154
|
def getCommandOutput(command)
|
121
155
|
puts "#{command.inspect} ..."
|
122
156
|
return IO.popen(command)
|
123
157
|
end
|
124
158
|
|
159
|
+
# Construct the ContentTree for the given base directory
|
125
160
|
def getContentTree(baseDir)
|
126
161
|
contentTree = ContentTree.new()
|
127
162
|
contentTree.time = Time.now.utc
|
@@ -135,9 +170,20 @@ module Synqa
|
|
135
170
|
end
|
136
171
|
end
|
137
172
|
|
173
|
+
# Representation of a remote system accessible via SSH
|
138
174
|
class SshContentHost<DirContentHost
|
139
175
|
|
140
|
-
|
176
|
+
# The SSH client, e.g. ["ssh"] or ["plink","-pw","mysecretpassword"] (i.e. command + args as an array)
|
177
|
+
attr_reader :shell
|
178
|
+
|
179
|
+
# The SCP client, e.g. ["scp"] or ["pscp","-pw","mysecretpassword"] (i.e. command + args as an array)
|
180
|
+
attr_reader :scpProgram
|
181
|
+
|
182
|
+
# The remote host, e.g. "username@host.example.com"
|
183
|
+
attr_reader :host
|
184
|
+
|
185
|
+
# The SCP command as a string
|
186
|
+
attr_reader :scpCommandString
|
141
187
|
|
142
188
|
def initialize(host, hashCommand, shell, scpProgram)
|
143
189
|
super(hashCommand)
|
@@ -147,11 +193,14 @@ module Synqa
|
|
147
193
|
@scpCommandString = @scpProgram.join(" ")
|
148
194
|
end
|
149
195
|
|
196
|
+
# Return readable description of base directory on remote system
|
150
197
|
def locationDescriptor(baseDir)
|
151
198
|
baseDir = normalisedDir(baseDir)
|
152
199
|
return "#{host}:#{baseDir} (connect = #{shell}/#{scpProgram}, hashCommand = #{hashCommand})"
|
153
200
|
end
|
154
201
|
|
202
|
+
# execute an SSH command on the remote system, yielding lines of output
|
203
|
+
# (or don't actually execute, if dryRun is true)
|
155
204
|
def executeRemoteCommand(commandString, dryRun = false)
|
156
205
|
puts "SSH #{host} (#{shell.join(" ")}): executing #{commandString}"
|
157
206
|
if not dryRun
|
@@ -164,12 +213,15 @@ module Synqa
|
|
164
213
|
end
|
165
214
|
end
|
166
215
|
|
216
|
+
# execute an SSH command on the remote system, displaying output to stdout,
|
217
|
+
# (or don't actually execute, if dryRun is true)
|
167
218
|
def ssh(commandString, dryRun = false)
|
168
219
|
executeRemoteCommand(commandString, dryRun) do |line|
|
169
220
|
puts line
|
170
221
|
end
|
171
222
|
end
|
172
223
|
|
224
|
+
# Return a list of all subdirectories of the base directory (as paths relative to the base directory)
|
173
225
|
def listDirectories(baseDir)
|
174
226
|
baseDir = normalisedDir(baseDir)
|
175
227
|
puts "Listing directories ..."
|
@@ -186,6 +238,8 @@ module Synqa
|
|
186
238
|
return directories
|
187
239
|
end
|
188
240
|
|
241
|
+
# Yield lines of output from the command to display hash values and file names
|
242
|
+
# of all files within the base directory
|
189
243
|
def listFileHashLines(baseDir)
|
190
244
|
baseDir = normalisedDir(baseDir)
|
191
245
|
remoteFileHashLinesCommand = findFilesCommand(baseDir) + ["|", "xargs", "-r"] + @hashCommand.command
|
@@ -195,6 +249,7 @@ module Synqa
|
|
195
249
|
end
|
196
250
|
end
|
197
251
|
|
252
|
+
# List all files within the base directory to stdout
|
198
253
|
def listFiles(baseDir)
|
199
254
|
baseDir = normalisedDir(baseDir)
|
200
255
|
executeRemoteCommand(findFilesCommand(baseDir).join(" ")) do |line|
|
@@ -202,13 +257,30 @@ module Synqa
|
|
202
257
|
end
|
203
258
|
end
|
204
259
|
|
260
|
+
# Get the remote path of the directory or file on the host, in the format required by SCP
|
205
261
|
def getScpPath(path)
|
206
262
|
return host + ":" + path
|
207
263
|
end
|
208
264
|
end
|
209
265
|
|
266
|
+
# An object representing the content of a file within a ContentTree.
|
267
|
+
# The file may be marked for copying (if it's in a source ContentTree)
|
268
|
+
# or for deletion (if it's in a destination ContentTree)
|
210
269
|
class FileContent
|
211
|
-
|
270
|
+
# The name of the file
|
271
|
+
attr_reader :name
|
272
|
+
|
273
|
+
# The hash value of the file's contents
|
274
|
+
attr_reader :hash
|
275
|
+
|
276
|
+
# The components of the relative path where the file is found
|
277
|
+
attr_reader :parentPathElements
|
278
|
+
|
279
|
+
# The destination to which the file should be copied
|
280
|
+
attr_reader :copyDestination
|
281
|
+
|
282
|
+
# Should this file be deleted
|
283
|
+
attr_reader :toBeDeleted
|
212
284
|
|
213
285
|
def initialize(name, hash, parentPathElements)
|
214
286
|
@name = name
|
@@ -218,10 +290,12 @@ module Synqa
|
|
218
290
|
@toBeDeleted = false
|
219
291
|
end
|
220
292
|
|
293
|
+
# Mark this file to be copied to a destination directory (from a destination content tree)
|
221
294
|
def markToCopy(destinationDirectory)
|
222
295
|
@copyDestination = destinationDirectory
|
223
296
|
end
|
224
297
|
|
298
|
+
# Mark this file to be deleted
|
225
299
|
def markToDelete
|
226
300
|
@toBeDeleted = true
|
227
301
|
end
|
@@ -230,14 +304,43 @@ module Synqa
|
|
230
304
|
return "#{name} (#{hash})"
|
231
305
|
end
|
232
306
|
|
307
|
+
# The full (relative) name of this file in the content tree
|
233
308
|
def fullPath
|
234
309
|
return (parentPathElements + [name]).join("/")
|
235
310
|
end
|
236
311
|
end
|
237
312
|
|
313
|
+
# A "content tree" consisting of a description of the contents of files and
|
314
|
+
# sub-directories within a base directory. The file contents are described via
|
315
|
+
# cryptographic hash values.
|
316
|
+
# Each sub-directory within a content tree is also represented as a ContentTree.
|
238
317
|
class ContentTree
|
239
|
-
|
240
|
-
attr_reader :
|
318
|
+
# name of the sub-directory within the containing directory (or nil if this is the base directory)
|
319
|
+
attr_reader :name
|
320
|
+
|
321
|
+
# path elements from base directory leading to this one
|
322
|
+
attr_reader :pathElements
|
323
|
+
|
324
|
+
# files within this sub-directory (as FileContent's)
|
325
|
+
attr_reader :files
|
326
|
+
|
327
|
+
# immediate sub-directories of this directory
|
328
|
+
attr_reader :dirs
|
329
|
+
|
330
|
+
# the files within this sub-directory, indexed by file name
|
331
|
+
attr_reader :fileByName
|
332
|
+
|
333
|
+
# immediate sub-directories of this directory, indexed by name
|
334
|
+
attr_reader :dirByName
|
335
|
+
|
336
|
+
# where this directory should be copied to
|
337
|
+
attr_reader :copyDestination
|
338
|
+
|
339
|
+
# whether this directory should be deleted
|
340
|
+
attr_reader :toBeDeleted
|
341
|
+
|
342
|
+
# the UTC time (on the local system, even if this content tree represents a remote directory)
|
343
|
+
# that this content tree was constructed. Only set for the base directory.
|
241
344
|
attr_accessor :time
|
242
345
|
|
243
346
|
def initialize(name = nil, parentPathElements = nil)
|
@@ -252,22 +355,27 @@ module Synqa
|
|
252
355
|
@time = nil
|
253
356
|
end
|
254
357
|
|
358
|
+
# mark this directory to be copied to a destination directory
|
255
359
|
def markToCopy(destinationDirectory)
|
256
360
|
@copyDestination = destinationDirectory
|
257
361
|
end
|
258
362
|
|
363
|
+
# mark this directory (on a remote system) to be deleted
|
259
364
|
def markToDelete
|
260
365
|
@toBeDeleted = true
|
261
366
|
end
|
262
367
|
|
368
|
+
# the full path of the directory that this content tree represents (relative to the base directory)
|
263
369
|
def fullPath
|
264
370
|
return @pathElements.join("/")
|
265
371
|
end
|
266
372
|
|
373
|
+
# convert a path string to an array of path elements (or return it as is if it's already an array)
|
267
374
|
def getPathElements(path)
|
268
375
|
return path.is_a?(String) ? (path == "" ? [] : path.split("/")) : path
|
269
376
|
end
|
270
377
|
|
378
|
+
# get the content tree for a sub-directory (creating it if it doesn't yet exist)
|
271
379
|
def getContentTreeForSubDir(subDir)
|
272
380
|
dirContentTree = dirByName.fetch(subDir, nil)
|
273
381
|
if dirContentTree == nil
|
@@ -278,6 +386,7 @@ module Synqa
|
|
278
386
|
return dirContentTree
|
279
387
|
end
|
280
388
|
|
389
|
+
# add a sub-directory to this content tree
|
281
390
|
def addDir(dirPath)
|
282
391
|
pathElements = getPathElements(dirPath)
|
283
392
|
if pathElements.length > 0
|
@@ -287,6 +396,7 @@ module Synqa
|
|
287
396
|
end
|
288
397
|
end
|
289
398
|
|
399
|
+
# recursively sort the files and sub-directories of this content tree alphabetically
|
290
400
|
def sort!
|
291
401
|
dirs.sort_by! {|dir| dir.name}
|
292
402
|
files.sort_by! {|file| file.name}
|
@@ -295,6 +405,7 @@ module Synqa
|
|
295
405
|
end
|
296
406
|
end
|
297
407
|
|
408
|
+
# given a relative path, add a file and hash value to this content tree
|
298
409
|
def addFile(filePath, hash)
|
299
410
|
pathElements = getPathElements(filePath)
|
300
411
|
if pathElements.length == 0
|
@@ -312,8 +423,10 @@ module Synqa
|
|
312
423
|
end
|
313
424
|
end
|
314
425
|
|
426
|
+
# date-time format for reading and writing times, e.g. "2007-12-23 13:03:99.012 +0000"
|
315
427
|
@@dateTimeFormat = "%Y-%m-%d %H:%M:%S.%L %z"
|
316
428
|
|
429
|
+
# pretty-print this content tree
|
317
430
|
def showIndented(name = "", indent = " ", currentIndent = "")
|
318
431
|
if time != nil
|
319
432
|
puts "#{currentIndent}[TIME: #{time.strftime(@@dateTimeFormat)}]"
|
@@ -341,7 +454,8 @@ module Synqa
|
|
341
454
|
end
|
342
455
|
end
|
343
456
|
end
|
344
|
-
|
457
|
+
|
458
|
+
# write this content tree to an open file, indented
|
345
459
|
def writeLinesToFile(outFile, prefix = "")
|
346
460
|
if time != nil
|
347
461
|
outFile.puts("T #{time.strftime(@@dateTimeFormat)}\n")
|
@@ -355,6 +469,7 @@ module Synqa
|
|
355
469
|
end
|
356
470
|
end
|
357
471
|
|
472
|
+
# write this content tree to a file (in a format which readFromFile can read back in)
|
358
473
|
def writeToFile(fileName)
|
359
474
|
puts "Writing content tree to file #{fileName} ..."
|
360
475
|
File.open(fileName, "w") do |outFile|
|
@@ -362,10 +477,16 @@ module Synqa
|
|
362
477
|
end
|
363
478
|
end
|
364
479
|
|
480
|
+
# regular expression for directory entries in content tree file
|
365
481
|
@@dirLineRegex = /^D (.*)$/
|
482
|
+
|
483
|
+
# regular expression for file entries in content tree file
|
366
484
|
@@fileLineRegex = /^F ([^ ]*) (.*)$/
|
485
|
+
|
486
|
+
# regular expression for time entry in content tree file
|
367
487
|
@@timeRegex = /^T (.*)$/
|
368
488
|
|
489
|
+
# read a content tree from a file (in format written by writeToFile)
|
369
490
|
def self.readFromFile(fileName)
|
370
491
|
contentTree = ContentTree.new()
|
371
492
|
puts "Reading content tree from #{fileName} ..."
|
@@ -393,7 +514,9 @@ module Synqa
|
|
393
514
|
end
|
394
515
|
return contentTree
|
395
516
|
end
|
396
|
-
|
517
|
+
|
518
|
+
# read a content tree as a map of hashes, i.e. from relative file path to hash value for the file
|
519
|
+
# Actually returns an array of the time entry (if any) and the map of hashes
|
397
520
|
def self.readMapOfHashesFromFile(fileName)
|
398
521
|
mapOfHashes = {}
|
399
522
|
time = nil
|
@@ -413,19 +536,26 @@ module Synqa
|
|
413
536
|
return [time, mapOfHashes]
|
414
537
|
end
|
415
538
|
|
539
|
+
# Mark operations for this (source) content tree and the destination content tree
|
540
|
+
# in order to synch the destination content tree with this one
|
416
541
|
def markSyncOperationsForDestination(destination)
|
417
542
|
markCopyOperations(destination)
|
418
543
|
destination.markDeleteOptions(self)
|
419
544
|
end
|
420
545
|
|
546
|
+
# Get the named sub-directory content tree, if it exists
|
421
547
|
def getDir(dir)
|
422
548
|
return dirByName.fetch(dir, nil)
|
423
549
|
end
|
424
550
|
|
551
|
+
# Get the named file & hash value, if it exists
|
425
552
|
def getFile(file)
|
426
553
|
return fileByName.fetch(file, nil)
|
427
554
|
end
|
428
555
|
|
556
|
+
# Mark copy operations, given that the corresponding destination directory already exists.
|
557
|
+
# For files and directories that don't exist in the destination, mark them to be copied.
|
558
|
+
# For sub-directories that do exist, recursively mark the corresponding sub-directory copy operations.
|
429
559
|
def markCopyOperations(destinationDir)
|
430
560
|
for dir in dirs
|
431
561
|
destinationSubDir = destinationDir.getDir(dir.name)
|
@@ -443,6 +573,9 @@ module Synqa
|
|
443
573
|
end
|
444
574
|
end
|
445
575
|
|
576
|
+
# Mark delete operations, given that the corresponding source directory exists.
|
577
|
+
# For files and directories that don't exist in the source, mark them to be deleted.
|
578
|
+
# For sub-directories that do exist, recursively mark the corresponding sub-directory delete operations.
|
446
579
|
def markDeleteOptions(sourceDir)
|
447
580
|
for dir in dirs
|
448
581
|
sourceSubDir = sourceDir.getDir(dir.name)
|
@@ -461,13 +594,18 @@ module Synqa
|
|
461
594
|
end
|
462
595
|
end
|
463
596
|
|
597
|
+
# Base class for a content location which consists of a base directory
|
598
|
+
# on a local or remote system.
|
464
599
|
class ContentLocation
|
600
|
+
|
601
|
+
# The name of a file used to hold a cached content tree for this location (can optionally be specified)
|
465
602
|
attr_reader :cachedContentFile
|
466
603
|
|
467
604
|
def initialize(cachedContentFile)
|
468
605
|
@cachedContentFile = cachedContentFile
|
469
606
|
end
|
470
607
|
|
608
|
+
# Get the cached content file name, if specified, and if the file exists
|
471
609
|
def getExistingCachedContentTreeFile
|
472
610
|
if cachedContentFile == nil
|
473
611
|
puts "No cached content file specified for location"
|
@@ -480,6 +618,7 @@ module Synqa
|
|
480
618
|
end
|
481
619
|
end
|
482
620
|
|
621
|
+
# Delete any existing cached content file
|
483
622
|
def clearCachedContentFile
|
484
623
|
if cachedContentFile and File.exists?(cachedContentFile)
|
485
624
|
puts " deleting cached content file #{cachedContentFile} ..."
|
@@ -487,6 +626,7 @@ module Synqa
|
|
487
626
|
end
|
488
627
|
end
|
489
628
|
|
629
|
+
# Get the cached content tree (if any), read from the specified cached content file.
|
490
630
|
def getCachedContentTree
|
491
631
|
file = getExistingCachedContentTreeFile
|
492
632
|
if file
|
@@ -496,6 +636,8 @@ module Synqa
|
|
496
636
|
end
|
497
637
|
end
|
498
638
|
|
639
|
+
# Read a map of file hashes (mapping from relative file name to hash value) from the
|
640
|
+
# specified cached content file
|
499
641
|
def getCachedContentTreeMapOfHashes
|
500
642
|
file = getExistingCachedContentTreeFile
|
501
643
|
if file
|
@@ -508,8 +650,14 @@ module Synqa
|
|
508
650
|
|
509
651
|
end
|
510
652
|
|
653
|
+
# A directory of files on a local system. The corresponding content tree
|
654
|
+
# can be calculated directly using Ruby library functions.
|
511
655
|
class LocalContentLocation<ContentLocation
|
512
|
-
|
656
|
+
|
657
|
+
# the base directory
|
658
|
+
attr_reader :baseDir
|
659
|
+
# the ruby class that generates the hash, e.g. Digest::SHA256
|
660
|
+
attr_reader :hashClass
|
513
661
|
|
514
662
|
def initialize(baseDir, hashClass, cachedContentFile = nil, options = {})
|
515
663
|
super(cachedContentFile)
|
@@ -519,6 +667,7 @@ module Synqa
|
|
519
667
|
@excludeGlobs = options.fetch(:excludes, [])
|
520
668
|
end
|
521
669
|
|
670
|
+
# get the path of a file name relative to the base directory
|
522
671
|
def getRelativePath(fileName)
|
523
672
|
if fileName.start_with? @baseDir
|
524
673
|
return fileName[@baseDirLen..-1]
|
@@ -527,15 +676,18 @@ module Synqa
|
|
527
676
|
end
|
528
677
|
end
|
529
678
|
|
679
|
+
# get the path as required for an SCP command
|
530
680
|
def getScpPath(relativePath)
|
531
681
|
return getFullPath(relativePath)
|
532
682
|
end
|
533
683
|
|
684
|
+
# get the full path of a relative path (i.e. of a file/directory within the base directory)
|
534
685
|
def getFullPath(relativePath)
|
535
686
|
return @baseDir + relativePath
|
536
687
|
end
|
537
688
|
|
538
|
-
|
689
|
+
# is the relative path name excluded by one of the specified exclusion globs?
|
690
|
+
def fileIsExcluded?(relativeFile)
|
539
691
|
for excludeGlob in @excludeGlobs
|
540
692
|
if File.fnmatch(excludeGlob, relativeFile)
|
541
693
|
puts " file #{relativeFile} excluded by glob #{excludeGlob}"
|
@@ -545,6 +697,12 @@ module Synqa
|
|
545
697
|
return false
|
546
698
|
end
|
547
699
|
|
700
|
+
# get the content tree for this base directory by iterating over all
|
701
|
+
# sub-directories and files within the base directory (and excluding the excluded files)
|
702
|
+
# and calculating file hashes using the specified Ruby hash class
|
703
|
+
# If there is an existing cached content file, use that to get the hash values
|
704
|
+
# of files whose modification time is earlier than the time value for the cached content tree.
|
705
|
+
# Also, if a cached content file is specified, write the final content tree back out to the cached content file.
|
548
706
|
def getContentTree
|
549
707
|
cachedTimeAndMapOfHashes = getCachedContentTreeMapOfHashes
|
550
708
|
cachedTime = cachedTimeAndMapOfHashes[0]
|
@@ -559,7 +717,7 @@ module Synqa
|
|
559
717
|
if File.directory? fileOrDir
|
560
718
|
contentTree.addDir(relativePath)
|
561
719
|
else
|
562
|
-
if not fileIsExcluded(relativePath)
|
720
|
+
if not fileIsExcluded?(relativePath)
|
563
721
|
cachedDigest = cachedMapOfHashes[relativePath]
|
564
722
|
if cachedTime and cachedDigest and File.stat(fileOrDir).mtime < cachedTime
|
565
723
|
digest = cachedDigest
|
@@ -579,8 +737,13 @@ module Synqa
|
|
579
737
|
end
|
580
738
|
end
|
581
739
|
|
740
|
+
# A directory of files on a remote system
|
582
741
|
class RemoteContentLocation<ContentLocation
|
583
|
-
|
742
|
+
# the remote username@host value
|
743
|
+
attr_reader :host
|
744
|
+
|
745
|
+
# the base directory on the remote system
|
746
|
+
attr_reader :baseDir
|
584
747
|
|
585
748
|
def initialize(host, baseDir, cachedContentFile = nil)
|
586
749
|
super(cachedContentFile)
|
@@ -588,30 +751,37 @@ module Synqa
|
|
588
751
|
@baseDir = normalisedDir(baseDir)
|
589
752
|
end
|
590
753
|
|
754
|
+
# list files within the base directory on the remote host
|
591
755
|
def listFiles()
|
592
756
|
host.listFiles(baseDir)
|
593
757
|
end
|
594
758
|
|
759
|
+
# the command string required to execute SCP (e.g. "scp" or "pscp", possibly with extra args)
|
595
760
|
def scpCommandString
|
596
761
|
return host.scpCommandString
|
597
762
|
end
|
598
763
|
|
764
|
+
# get the full path of a relative path
|
599
765
|
def getFullPath(relativePath)
|
600
766
|
return baseDir + relativePath
|
601
767
|
end
|
602
768
|
|
769
|
+
# get the full path of a file as required in an SCP command (i.e. with username@host prepended)
|
603
770
|
def getScpPath(relativePath)
|
604
771
|
return host.getScpPath(getFullPath(relativePath))
|
605
772
|
end
|
606
773
|
|
774
|
+
# execute an SSH command on the remote host (or just pretend, if dryRun is true)
|
607
775
|
def ssh(commandString, dryRun = false)
|
608
776
|
host.ssh(commandString, dryRun)
|
609
777
|
end
|
610
778
|
|
779
|
+
# list all sub-directories of the base directory on the remote host
|
611
780
|
def listDirectories
|
612
781
|
return host.listDirectories(baseDir)
|
613
782
|
end
|
614
783
|
|
784
|
+
# list all the file hashes of the files within the base directory
|
615
785
|
def listFileHashes
|
616
786
|
return host.listFileHashes(baseDir)
|
617
787
|
end
|
@@ -619,7 +789,11 @@ module Synqa
|
|
619
789
|
def to_s
|
620
790
|
return host.locationDescriptor(baseDir)
|
621
791
|
end
|
622
|
-
|
792
|
+
|
793
|
+
# Get the content tree, from the cached content file if it exists,
|
794
|
+
# otherwise get if from listing directories and files and hash values thereof
|
795
|
+
# on the remote host. And also, if the cached content file name is specified,
|
796
|
+
# write the content tree out to that file.
|
623
797
|
def getContentTree
|
624
798
|
if cachedContentFile and File.exists?(cachedContentFile)
|
625
799
|
return ContentTree.readFromFile(cachedContentFile)
|
@@ -635,19 +809,27 @@ module Synqa
|
|
635
809
|
|
636
810
|
end
|
637
811
|
|
812
|
+
# The operation of synchronising files on the remote directory with files on the local directory.
|
638
813
|
class SyncOperation
|
639
|
-
|
814
|
+
# The source location (presumed to be local)
|
815
|
+
attr_reader :sourceLocation
|
816
|
+
|
817
|
+
# The destination location (presumed to be remote)
|
818
|
+
attr_reader :destinationLocation
|
640
819
|
|
641
820
|
def initialize(sourceLocation, destinationLocation)
|
642
821
|
@sourceLocation = sourceLocation
|
643
822
|
@destinationLocation = destinationLocation
|
644
823
|
end
|
645
824
|
|
825
|
+
# Get the local and remote content trees
|
646
826
|
def getContentTrees
|
647
827
|
@sourceContent = @sourceLocation.getContentTree()
|
648
828
|
@destinationContent = @destinationLocation.getContentTree()
|
649
829
|
end
|
650
830
|
|
831
|
+
# On the local and remote content trees, mark the copy and delete operations required
|
832
|
+
# to sync the remote location to the local location.
|
651
833
|
def markSyncOperations
|
652
834
|
@sourceContent.markSyncOperationsForDestination(@destinationContent)
|
653
835
|
puts " ================================================ "
|
@@ -660,11 +842,15 @@ module Synqa
|
|
660
842
|
@destinationContent.showIndented()
|
661
843
|
end
|
662
844
|
|
845
|
+
# Delete the local and remote cached content files (which will force a full recalculation
|
846
|
+
# of both content trees next time)
|
663
847
|
def clearCachedContentFiles
|
664
848
|
@sourceLocation.clearCachedContentFile()
|
665
849
|
@destinationLocation.clearCachedContentFile()
|
666
850
|
end
|
667
851
|
|
852
|
+
# Do the sync. Options: :full = true means clear the cached content files first, :dryRun
|
853
|
+
# means don't do the actual copies and deletes, but just show what they would be.
|
668
854
|
def doSync(options = {})
|
669
855
|
if options[:full]
|
670
856
|
clearCachedContentFiles()
|
@@ -682,15 +868,19 @@ module Synqa
|
|
682
868
|
FileUtils::Verbose.cp(@sourceLocation.cachedContentFile, @destinationLocation.cachedContentFile)
|
683
869
|
end
|
684
870
|
end
|
685
|
-
|
871
|
+
|
872
|
+
# Do all the copy operations, copying local directories or files which are missing from the remote location
|
686
873
|
def doAllCopyOperations(dryRun)
|
687
874
|
doCopyOperations(@sourceContent, @destinationContent, dryRun)
|
688
875
|
end
|
689
876
|
|
877
|
+
# Do all delete operations, deleting remote directories or files which do not exist at the local location
|
690
878
|
def doAllDeleteOperations(dryRun)
|
691
879
|
doDeleteOperations(@destinationContent, dryRun)
|
692
880
|
end
|
693
881
|
|
882
|
+
# Execute a (local) command, or, if dryRun, just pretend to execute it.
|
883
|
+
# Raise an exception if the process exit status is not 0.
|
694
884
|
def executeCommand(command, dryRun)
|
695
885
|
puts "EXECUTE: #{command}"
|
696
886
|
if not dryRun
|
@@ -699,6 +889,8 @@ module Synqa
|
|
699
889
|
end
|
700
890
|
end
|
701
891
|
|
892
|
+
# Recursively perform all marked copy operations from the source content tree to the
|
893
|
+
# destination content tree, or if dryRun, just pretend to perform them.
|
702
894
|
def doCopyOperations(sourceContent, destinationContent, dryRun)
|
703
895
|
for dir in sourceContent.dirs do
|
704
896
|
if dir.copyDestination != nil
|
@@ -718,6 +910,8 @@ module Synqa
|
|
718
910
|
end
|
719
911
|
end
|
720
912
|
|
913
|
+
# Recursively perform all marked delete operations on the destination content tree,
|
914
|
+
# or if dryRun, just pretend to perform them.
|
721
915
|
def doDeleteOperations(destinationContent, dryRun)
|
722
916
|
for dir in destinationContent.dirs do
|
723
917
|
if dir.toBeDeleted
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: synqa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,12 +9,12 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-02
|
12
|
+
date: 2011-03-02 00:00:00.000000000 +13:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: shoulda
|
17
|
-
requirement: &
|
17
|
+
requirement: &24960612 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
20
|
- - ! '>='
|
@@ -22,10 +22,10 @@ dependencies:
|
|
22
22
|
version: '0'
|
23
23
|
type: :development
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
25
|
+
version_requirements: *24960612
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
27
|
name: bundler
|
28
|
-
requirement: &
|
28
|
+
requirement: &24959928 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
31
31
|
- - ~>
|
@@ -33,10 +33,10 @@ dependencies:
|
|
33
33
|
version: 1.0.0
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
|
-
version_requirements: *
|
36
|
+
version_requirements: *24959928
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
38
|
name: jeweler
|
39
|
-
requirement: &
|
39
|
+
requirement: &24942648 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ~>
|
@@ -44,10 +44,10 @@ dependencies:
|
|
44
44
|
version: 1.5.2
|
45
45
|
type: :development
|
46
46
|
prerelease: false
|
47
|
-
version_requirements: *
|
47
|
+
version_requirements: *24942648
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
name: rcov
|
50
|
-
requirement: &
|
50
|
+
requirement: &24941832 !ruby/object:Gem::Requirement
|
51
51
|
none: false
|
52
52
|
requirements:
|
53
53
|
- - ! '>='
|
@@ -55,7 +55,7 @@ dependencies:
|
|
55
55
|
version: '0'
|
56
56
|
type: :development
|
57
57
|
prerelease: false
|
58
|
-
version_requirements: *
|
58
|
+
version_requirements: *24941832
|
59
59
|
description: Sync files from a local directory to a remote directory via SSH/SCP
|
60
60
|
email: http://www.1729.com/email.html
|
61
61
|
executables: []
|
@@ -93,7 +93,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
93
93
|
version: '0'
|
94
94
|
segments:
|
95
95
|
- 0
|
96
|
-
hash:
|
96
|
+
hash: 346342291
|
97
97
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
98
98
|
none: false
|
99
99
|
requirements:
|