synqa 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1005 +1,1052 @@
1
- require 'time'
2
- require 'net/ssh'
3
- require 'net/scp'
4
-
5
- module Synqa
6
-
7
- # ensure that a directory exists
8
- def ensureDirectoryExists(directoryName)
9
- if File.exist? directoryName
10
- if not File.directory? directoryName
11
- raise "#{directoryName} is a non-directory file"
12
- end
13
- else
14
- FileUtils.makedirs(directoryName)
15
- end
16
- end
17
-
18
- # Return the enumerated lines of the command's output
19
- def getCommandOutput(command)
20
- puts "#{command.inspect} ..."
21
- return IO.popen(command)
22
- end
23
-
24
- # Check if the last executed process exited with status 0, if not, raise an exception
25
- def checkProcessStatus(description)
26
- processStatus = $?
27
- if not processStatus.exited?
28
- raise "#{description}: process did not exit normally"
29
- end
30
- exitStatus = processStatus.exitstatus
31
- if exitStatus != 0
32
- raise "#{description}: exit status = #{exitStatus}"
33
- end
34
- end
35
-
36
- # An object representing a file path relative to a base directory, and a hash string
37
- class RelativePathWithHash
38
- # The relative file path (e.g. c:/dir/subdir/file.txt relative to c:/dir would be subdir/file.txt)
39
- attr_reader :relativePath
40
-
41
- # The hash code, e.g. a1c5b67fdb3cf0df8f1d29ae90561f9ad099bada44aeb6b2574ad9e15f2a84ed
42
- attr_reader :hash
43
-
44
- def initialize(relativePath, hash)
45
- @relativePath = relativePath
46
- @hash = hash
47
- end
48
-
49
- def inspect
50
- return "RelativePathWithHash[#{relativePath}, #{hash}]"
51
- end
52
- end
53
-
54
- # A command to be executed on the remote system which calculates a hash value for
55
- # a file (of a given length), in the format: *hexadecimal-hash* *a-fixed-number-of-characters* *file-name*
56
- class HashCommand
57
- # The command - a string or array of strings e.g. "sha256sum" or ["sha256", "-r"]
58
- attr_reader :command
59
-
60
- # The length of the calculated hash value e.g. 64 for sha256
61
- attr_reader :length
62
-
63
- # The number of characters between the hash value and the file name (usually 1 or 2)
64
- attr_reader :spacerLen
65
-
66
- def initialize(command, length, spacerLen)
67
- @command = command
68
- @length = length
69
- @spacerLen = spacerLen
70
- end
71
-
72
- # Parse a hash line relative to a base directory, returning a RelativePathWithHash
73
- def parseFileHashLine(baseDir, fileHashLine)
74
- hash = fileHashLine[0...length]
75
- fullPath = fileHashLine[(length + spacerLen)..-1]
76
- if fullPath.start_with?(baseDir)
77
- return RelativePathWithHash.new(fullPath[baseDir.length..-1], hash)
78
- else
79
- raise "File #{fullPath} from hash line is not in base dir #{baseDir}"
80
- end
81
- end
82
-
83
- def to_s
84
- return command.join(" ")
85
- end
86
- end
87
-
88
- # Hash command for sha256sum, which generates a 64 hexadecimal digit hash, and outputs two characters between
89
- # the hash and the file name.
90
- class Sha256SumCommand<HashCommand
91
- def initialize
92
- super(["sha256sum"], 64, 2)
93
- end
94
- end
95
-
96
- # Hash command for sha256, which generates a 64 hexadecimal digit hash, and outputs one character between
97
- # the hash and the file name, and which requires a "-r" argument to put the hash value first.
98
- class Sha256Command<HashCommand
99
- def initialize
100
- super(["sha256", "-r"], 64, 1)
101
- end
102
- end
103
-
104
- # Put "/" at the end of a directory name if it is not already there.
105
- def normalisedDir(baseDir)
106
- return baseDir.end_with?("/") ? baseDir : baseDir + "/"
107
- end
108
-
109
-
110
- # Base class for an object representing a remote system where the contents of a directory
111
- # on the system are enumerated by one command to list all sub-directories and another command
112
- # to list all files in the directory and their hash values.
113
- class DirContentHost
114
-
115
- # The HashCommand object used to calculate and parse hash values of files
116
- attr_reader :hashCommand
117
-
118
- # Prefix required for *find* command (usually nothing, since it should be on the system path)
119
- attr_reader :pathPrefix
120
-
121
- def initialize(hashCommand, pathPrefix = "")
122
- @hashCommand = hashCommand
123
- @pathPrefix = pathPrefix
124
- end
125
-
126
- # Generate the *find* command which will list all the sub-directories of the base directory
127
- def findDirectoriesCommand(baseDir)
128
- return ["#{@pathPrefix}find", baseDir, "-type", "d", "-print"]
129
- end
130
-
131
- # Return the list of sub-directories relative to the base directory
132
- def listDirectories(baseDir)
133
- baseDir = normalisedDir(baseDir)
134
- command = findDirectoriesCommand(baseDir)
135
- output = getCommandOutput(command)
136
- directories = []
137
- baseDirLen = baseDir.length
138
- puts "Listing directories ..."
139
- while (line = output.gets)
140
- line = line.chomp
141
- puts " #{line}"
142
- if line.start_with?(baseDir)
143
- directories << line[baseDirLen..-1]
144
- else
145
- raise "Directory #{line} is not a sub-directory of base directory #{baseDir}"
146
- end
147
- end
148
- output.close()
149
- checkProcessStatus(command)
150
- return directories
151
- end
152
-
153
- # Generate the *find* command which will list all the files within the base directory
154
- def findFilesCommand(baseDir)
155
- return ["#{@pathPrefix}find", baseDir, "-type", "f", "-print"]
156
- end
157
-
158
- # List file hashes by executing the command to hash each file on the output of the
159
- # *find* command which lists all files, and parse the output.
160
- def listFileHashes(baseDir)
161
- baseDir = normalisedDir(baseDir)
162
- fileHashes = []
163
- listFileHashLines(baseDir) do |fileHashLine|
164
- fileHash = self.hashCommand.parseFileHashLine(baseDir, fileHashLine)
165
- if fileHash != nil
166
- fileHashes << fileHash
167
- end
168
- end
169
- return fileHashes
170
- end
171
-
172
- # Construct the ContentTree for the given base directory
173
- def getContentTree(baseDir)
174
- contentTree = ContentTree.new()
175
- contentTree.time = Time.now.utc
176
- for dir in listDirectories(baseDir)
177
- contentTree.addDir(dir)
178
- end
179
- for fileHash in listFileHashes(baseDir)
180
- contentTree.addFile(fileHash.relativePath, fileHash.hash)
181
- end
182
- return contentTree
183
- end
184
- end
185
-
186
- # Execute a (local) command, or, if dryRun, just pretend to execute it.
187
- # Raise an exception if the process exit status is not 0.
188
- def executeCommand(command, dryRun)
189
- puts "EXECUTE: #{command}"
190
- if not dryRun
191
- system(command)
192
- checkProcessStatus(command)
193
- end
194
- end
195
-
196
- # Base SSH/SCP implementation
197
- class BaseSshScp
198
-
199
- # delete remote directory (if dryRun is false) using "rm -r"
200
- def deleteDirectory(userAtHost, dirPath, dryRun)
201
- ssh(userAtHost, "rm -r #{dirPath}", dryRun)
202
- end
203
-
204
- # delete remote file (if dryRun is false) using "rm"
205
- def deleteFile(userAtHost, filePath, dryRun)
206
- ssh(userAtHost, "rm #{filePath}", dryRun)
207
- end
208
- end
209
-
210
- # SSH/SCP using Ruby Net::SSH & Net::SCP
211
- class InternalSshScp<BaseSshScp
212
-
213
- # execute command on remote host (if dryRun is false), yielding lines of output
214
- def ssh(userAtHost, commandString, dryRun)
215
- user, host = userAtHost.split("@")
216
- description = "SSH #{user}@#{host}: executing #{commandString}"
217
- puts description
218
- if not dryRun
219
- Net::SSH.start(host, user) do |ssh|
220
- outputText = ssh.exec!(commandString)
221
- if outputText != nil then
222
- for line in outputText.split("\n") do
223
- yield line
224
- end
225
- end
226
- end
227
- end
228
- end
229
-
230
- # copy a local directory to a remote directory (if dryRun is false)
231
- def copyLocalToRemoteDirectory(userAtHost, sourcePath, destinationPath, dryRun)
232
- user, host = userAtHost.split("@")
233
- description = "SCP: copy directory #{sourcePath} to #{user}@#{host}:#{destinationPath}"
234
- puts description
235
- if not dryRun
236
- Net::SCP.upload!(host, user, sourcePath, destinationPath, :recursive => true)
237
- end
238
- end
239
-
240
- # copy a local file to a remote directory (if dryRun is false)
241
- def copyLocalFileToRemoteDirectory(userAtHost, sourcePath, destinationPath, dryRun)
242
- user, host = userAtHost.split("@")
243
- description = "SCP: copy file #{sourcePath} to #{user}@#{host}:#{destinationPath}"
244
- puts description
245
- if not dryRun
246
- Net::SCP.upload!(host, user, sourcePath, destinationPath)
247
- end
248
- end
249
-
250
- end
251
-
252
- # SSH/SCP using external commands, such as "plink" and "pscp"
253
- class ExternalSshScp<BaseSshScp
254
- # The SSH client, e.g. ["ssh"] or ["plink","-pw","mysecretpassword"] (i.e. command + args as an array)
255
- attr_reader :shell
256
-
257
- # The SCP client, e.g. ["scp"] or ["pscp","-pw","mysecretpassword"] (i.e. command + args as an array)
258
- attr_reader :scpProgram
259
-
260
- # The SCP command as a string
261
- attr_reader :scpCommandString
262
-
263
- def initialize(shell, scpProgram)
264
- @shell = shell.is_a?(String) ? [shell] : shell
265
- @scpProgram = scpProgram.is_a?(String) ? [scpProgram] : scpProgram
266
- @scpCommandString = @scpProgram.join(" ")
267
- end
268
-
269
- # execute command on remote host (if dryRun is false), yielding lines of output
270
- def ssh(userAtHost, commandString, dryRun)
271
- puts "SSH #{userAtHost} (#{shell.join(" ")}): executing #{commandString}"
272
- if not dryRun
273
- output = getCommandOutput(shell + [userAtHost, commandString])
274
- while (line = output.gets)
275
- yield line.chomp
276
- end
277
- output.close()
278
- checkProcessStatus("SSH #{userAtHost} #{commandString}")
279
- end
280
- end
281
-
282
- # copy a local directory to a remote directory (if dryRun is false)
283
- def copyLocalToRemoteDirectory(userAtHost, sourcePath, destinationPath, dryRun)
284
- executeCommand("#{@scpCommandString} -r #{sourcePath} #{userAtHost}:#{destinationPath}", dryRun)
285
- end
286
-
287
- # copy a local file to a remote directory (if dryRun is false)
288
- def copyLocalFileToRemoteDirectory(userAtHost, sourcePath, destinationPath, dryRun)
289
- executeCommand("#{@scpCommandString} #{sourcePath} #{userAtHost}:#{destinationPath}", dryRun)
290
- end
291
-
292
- end
293
-
294
- # Representation of a remote system accessible via SSH
295
- class SshContentHost<DirContentHost
296
-
297
- # The remote userAtHost, e.g. "username@host.example.com"
298
- attr_reader :userAtHost, :sshAndScp
299
-
300
- def initialize(userAtHost, hashCommand, sshAndScp = nil)
301
- super(hashCommand)
302
- @sshAndScp = sshAndScp != nil ? sshAndScp : InternalSshScp.new()
303
- @userAtHost = userAtHost
304
- end
305
-
306
- # Return readable description of base directory on remote system
307
- def locationDescriptor(baseDir)
308
- baseDir = normalisedDir(baseDir)
309
- return "#{userAtHost}:#{baseDir} (connect = #{shell}/#{scpProgram}, hashCommand = #{hashCommand})"
310
- end
311
-
312
- # execute an SSH command on the remote system, yielding lines of output
313
- # (or don't actually execute, if dryRun is false)
314
- def ssh(commandString, dryRun = false)
315
- sshAndScp.ssh(userAtHost, commandString, dryRun) do |line|
316
- yield line
317
- end
318
- end
319
-
320
- # delete a remote directory, if dryRun is false
321
- def deleteDirectory(dirPath, dryRun)
322
- sshAndScp.deleteDirectory(userAtHost, dirPath, dryRun)
323
- end
324
-
325
- # delete a remote file, if dryRun is false
326
- def deleteFile(filePath, dryRun)
327
- sshAndScp.deleteFile(userAtHost, filePath, dryRun)
328
- end
329
-
330
- # copy a local directory to a remote directory, if dryRun is false
331
- def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
332
- sshAndScp.copyLocalToRemoteDirectory(userAtHost, sourcePath, destinationPath, dryRun)
333
- end
334
-
335
- # copy a local file to a remote directory, if dryRun is false
336
- def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
337
- sshAndScp.copyLocalFileToRemoteDirectory(userAtHost, sourcePath, destinationPath, dryRun)
338
- end
339
-
340
- # Return a list of all subdirectories of the base directory (as paths relative to the base directory)
341
- def listDirectories(baseDir)
342
- baseDir = normalisedDir(baseDir)
343
- puts "Listing directories ..."
344
- directories = []
345
- baseDirLen = baseDir.length
346
- ssh(findDirectoriesCommand(baseDir).join(" ")) do |line|
347
- puts " #{line}"
348
- if line.start_with?(baseDir)
349
- directories << line[baseDirLen..-1]
350
- else
351
- raise "Directory #{line} is not a sub-directory of base directory #{baseDir}"
352
- end
353
- end
354
- return directories
355
- end
356
-
357
- # Yield lines of output from the command to display hash values and file names
358
- # of all files within the base directory
359
- def listFileHashLines(baseDir)
360
- baseDir = normalisedDir(baseDir)
361
- remoteFileHashLinesCommand = findFilesCommand(baseDir) + ["|", "xargs", "-r"] + @hashCommand.command
362
- ssh(remoteFileHashLinesCommand.join(" ")) do |line|
363
- puts " #{line}"
364
- yield line
365
- end
366
- end
367
-
368
- # List all files within the base directory to stdout
369
- def listFiles(baseDir)
370
- baseDir = normalisedDir(baseDir)
371
- ssh(findFilesCommand(baseDir).join(" ")) do |line|
372
- puts " #{line}"
373
- end
374
- end
375
-
376
- end
377
-
378
- # An object representing the content of a file within a ContentTree.
379
- # The file may be marked for copying (if it's in a source ContentTree)
380
- # or for deletion (if it's in a destination ContentTree)
381
- class FileContent
382
- # The name of the file
383
- attr_reader :name
384
-
385
- # The hash value of the file's contents
386
- attr_reader :hash
387
-
388
- # The components of the relative path where the file is found
389
- attr_reader :parentPathElements
390
-
391
- # The destination to which the file should be copied
392
- attr_reader :copyDestination
393
-
394
- # Should this file be deleted
395
- attr_reader :toBeDeleted
396
-
397
- def initialize(name, hash, parentPathElements)
398
- @name = name
399
- @hash = hash
400
- @parentPathElements = parentPathElements
401
- @copyDestination = nil
402
- @toBeDeleted = false
403
- end
404
-
405
- # Mark this file to be copied to a destination directory (from a destination content tree)
406
- def markToCopy(destinationDirectory)
407
- @copyDestination = destinationDirectory
408
- end
409
-
410
- # Mark this file to be deleted
411
- def markToDelete
412
- @toBeDeleted = true
413
- end
414
-
415
- def to_s
416
- return "#{name} (#{hash})"
417
- end
418
-
419
- # The relative name of this file in the content tree (relative to the base dir)
420
- def relativePath
421
- return (parentPathElements + [name]).join("/")
422
- end
423
- end
424
-
425
- # A "content tree" consisting of a description of the contents of files and
426
- # sub-directories within a base directory. The file contents are described via
427
- # cryptographic hash values.
428
- # Each sub-directory within a content tree is also represented as a ContentTree.
429
- class ContentTree
430
- # name of the sub-directory within the containing directory (or nil if this is the base directory)
431
- attr_reader :name
432
-
433
- # path elements from base directory leading to this one
434
- attr_reader :pathElements
435
-
436
- # files within this sub-directory (as FileContent's)
437
- attr_reader :files
438
-
439
- # immediate sub-directories of this directory
440
- attr_reader :dirs
441
-
442
- # the files within this sub-directory, indexed by file name
443
- attr_reader :fileByName
444
-
445
- # immediate sub-directories of this directory, indexed by name
446
- attr_reader :dirByName
447
-
448
- # where this directory should be copied to
449
- attr_reader :copyDestination
450
-
451
- # whether this directory should be deleted
452
- attr_reader :toBeDeleted
453
-
454
- # the UTC time (on the local system, even if this content tree represents a remote directory)
455
- # that this content tree was constructed. Only set for the base directory.
456
- attr_accessor :time
457
-
458
- def initialize(name = nil, parentPathElements = nil)
459
- @name = name
460
- @pathElements = name == nil ? [] : parentPathElements + [name]
461
- @files = []
462
- @dirs = []
463
- @fileByName = {}
464
- @dirByName = {}
465
- @copyDestination = nil
466
- @toBeDeleted = false
467
- @time = nil
468
- end
469
-
470
- # mark this directory to be copied to a destination directory
471
- def markToCopy(destinationDirectory)
472
- @copyDestination = destinationDirectory
473
- end
474
-
475
- # mark this directory (on a remote system) to be deleted
476
- def markToDelete
477
- @toBeDeleted = true
478
- end
479
-
480
- # the path of the directory that this content tree represents, relative to the base directory
481
- def relativePath
482
- return @pathElements.join("/")
483
- end
484
-
485
- # convert a path string to an array of path elements (or return it as is if it's already an array)
486
- def getPathElements(path)
487
- return path.is_a?(String) ? (path == "" ? [] : path.split("/")) : path
488
- end
489
-
490
- # get the content tree for a sub-directory (creating it if it doesn't yet exist)
491
- def getContentTreeForSubDir(subDir)
492
- dirContentTree = dirByName.fetch(subDir, nil)
493
- if dirContentTree == nil
494
- dirContentTree = ContentTree.new(subDir, @pathElements)
495
- dirs << dirContentTree
496
- dirByName[subDir] = dirContentTree
497
- end
498
- return dirContentTree
499
- end
500
-
501
- # add a sub-directory to this content tree
502
- def addDir(dirPath)
503
- pathElements = getPathElements(dirPath)
504
- if pathElements.length > 0
505
- pathStart = pathElements[0]
506
- restOfPath = pathElements[1..-1]
507
- getContentTreeForSubDir(pathStart).addDir(restOfPath)
508
- end
509
- end
510
-
511
- # recursively sort the files and sub-directories of this content tree alphabetically
512
- def sort!
513
- dirs.sort_by! {|dir| dir.name}
514
- files.sort_by! {|file| file.name}
515
- for dir in dirs
516
- dir.sort!
517
- end
518
- end
519
-
520
- # given a relative path, add a file and hash value to this content tree
521
- def addFile(filePath, hash)
522
- pathElements = getPathElements(filePath)
523
- if pathElements.length == 0
524
- raise "Invalid file path: #{filePath.inspect}"
525
- end
526
- if pathElements.length == 1
527
- fileName = pathElements[0]
528
- fileContent = FileContent.new(fileName, hash, @pathElements)
529
- files << fileContent
530
- fileByName[fileName] = fileContent
531
- else
532
- pathStart = pathElements[0]
533
- restOfPath = pathElements[1..-1]
534
- getContentTreeForSubDir(pathStart).addFile(restOfPath, hash)
535
- end
536
- end
537
-
538
- # date-time format for reading and writing times, e.g. "2007-12-23 13:03:99.012 +0000"
539
- @@dateTimeFormat = "%Y-%m-%d %H:%M:%S.%L %z"
540
-
541
- # pretty-print this content tree
542
- def showIndented(name = "", indent = " ", currentIndent = "")
543
- if time != nil
544
- puts "#{currentIndent}[TIME: #{time.strftime(@@dateTimeFormat)}]"
545
- end
546
- if name != ""
547
- puts "#{currentIndent}#{name}"
548
- end
549
- if copyDestination != nil
550
- puts "#{currentIndent} [COPY to #{copyDestination.relativePath}]"
551
- end
552
- if toBeDeleted
553
- puts "#{currentIndent} [DELETE]"
554
- end
555
- nextIndent = currentIndent + indent
556
- for dir in dirs
557
- dir.showIndented("#{dir.name}/", indent = indent, currentIndent = nextIndent)
558
- end
559
- for file in files
560
- puts "#{nextIndent}#{file.name} - #{file.hash}"
561
- if file.copyDestination != nil
562
- puts "#{nextIndent} [COPY to #{file.copyDestination.relativePath}]"
563
- end
564
- if file.toBeDeleted
565
- puts "#{nextIndent} [DELETE]"
566
- end
567
- end
568
- end
569
-
570
- # write this content tree to an open file, indented
571
- def writeLinesToFile(outFile, prefix = "")
572
- if time != nil
573
- outFile.puts("T #{time.strftime(@@dateTimeFormat)}\n")
574
- end
575
- for dir in dirs
576
- outFile.puts("D #{prefix}#{dir.name}\n")
577
- dir.writeLinesToFile(outFile, "#{prefix}#{dir.name}/")
578
- end
579
- for file in files
580
- outFile.puts("F #{file.hash} #{prefix}#{file.name}\n")
581
- end
582
- end
583
-
584
- # write this content tree to a file (in a format which readFromFile can read back in)
585
- def writeToFile(fileName)
586
- puts "Writing content tree to file #{fileName} ..."
587
- File.open(fileName, "w") do |outFile|
588
- writeLinesToFile(outFile)
589
- end
590
- end
591
-
592
- # regular expression for directory entries in content tree file
593
- @@dirLineRegex = /^D (.*)$/
594
-
595
- # regular expression for file entries in content tree file
596
- @@fileLineRegex = /^F ([^ ]*) (.*)$/
597
-
598
- # regular expression for time entry in content tree file
599
- @@timeRegex = /^T (.*)$/
600
-
601
- # read a content tree from a file (in format written by writeToFile)
602
- def self.readFromFile(fileName)
603
- contentTree = ContentTree.new()
604
- puts "Reading content tree from #{fileName} ..."
605
- IO.foreach(fileName) do |line|
606
- dirLineMatch = @@dirLineRegex.match(line)
607
- if dirLineMatch
608
- dirName = dirLineMatch[1]
609
- contentTree.addDir(dirName)
610
- else
611
- fileLineMatch = @@fileLineRegex.match(line)
612
- if fileLineMatch
613
- hash = fileLineMatch[1]
614
- fileName = fileLineMatch[2]
615
- contentTree.addFile(fileName, hash)
616
- else
617
- timeLineMatch = @@timeRegex.match(line)
618
- if timeLineMatch
619
- timeString = timeLineMatch[1]
620
- contentTree.time = Time.strptime(timeString, @@dateTimeFormat)
621
- else
622
- raise "Invalid line in content tree file: #{line.inspect}"
623
- end
624
- end
625
- end
626
- end
627
- return contentTree
628
- end
629
-
630
- # read a content tree as a map of hashes, i.e. from relative file path to hash value for the file
631
- # Actually returns an array of the time entry (if any) and the map of hashes
632
- def self.readMapOfHashesFromFile(fileName)
633
- mapOfHashes = {}
634
- time = nil
635
- File.open(fileName).each_line do |line|
636
- fileLineMatch = @@fileLineRegex.match(line)
637
- if fileLineMatch
638
- hash = fileLineMatch[1]
639
- fileName = fileLineMatch[2]
640
- mapOfHashes[fileName] = hash
641
- end
642
- timeLineMatch = @@timeRegex.match(line)
643
- if timeLineMatch
644
- timeString = timeLineMatch[1]
645
- time = Time.strptime(timeString, @@dateTimeFormat)
646
- end
647
- end
648
- return [time, mapOfHashes]
649
- end
650
-
651
- # Mark operations for this (source) content tree and the destination content tree
652
- # in order to synch the destination content tree with this one
653
- def markSyncOperationsForDestination(destination)
654
- markCopyOperations(destination)
655
- destination.markDeleteOptions(self)
656
- end
657
-
658
- # Get the named sub-directory content tree, if it exists
659
- def getDir(dir)
660
- return dirByName.fetch(dir, nil)
661
- end
662
-
663
- # Get the named file & hash value, if it exists
664
- def getFile(file)
665
- return fileByName.fetch(file, nil)
666
- end
667
-
668
- # Mark copy operations, given that the corresponding destination directory already exists.
669
- # For files and directories that don't exist in the destination, mark them to be copied.
670
- # For sub-directories that do exist, recursively mark the corresponding sub-directory copy operations.
671
- def markCopyOperations(destinationDir)
672
- for dir in dirs
673
- destinationSubDir = destinationDir.getDir(dir.name)
674
- if destinationSubDir != nil
675
- dir.markCopyOperations(destinationSubDir)
676
- else
677
- dir.markToCopy(destinationDir)
678
- end
679
- end
680
- for file in files
681
- destinationFile = destinationDir.getFile(file.name)
682
- if destinationFile == nil or destinationFile.hash != file.hash
683
- file.markToCopy(destinationDir)
684
- end
685
- end
686
- end
687
-
688
- # Mark delete operations, given that the corresponding source directory exists.
689
- # For files and directories that don't exist in the source, mark them to be deleted.
690
- # For sub-directories that do exist, recursively mark the corresponding sub-directory delete operations.
691
- def markDeleteOptions(sourceDir)
692
- for dir in dirs
693
- sourceSubDir = sourceDir.getDir(dir.name)
694
- if sourceSubDir == nil
695
- dir.markToDelete()
696
- else
697
- dir.markDeleteOptions(sourceSubDir)
698
- end
699
- end
700
- for file in files
701
- sourceFile = sourceDir.getFile(file.name)
702
- if sourceFile == nil
703
- file.markToDelete()
704
- end
705
- end
706
- end
707
- end
708
-
709
- # Base class for a content location which consists of a base directory
710
- # on a local or remote system.
711
- class ContentLocation
712
-
713
- # The name of a file used to hold a cached content tree for this location (can optionally be specified)
714
- attr_reader :cachedContentFile
715
-
716
- def initialize(cachedContentFile)
717
- @cachedContentFile = cachedContentFile
718
- end
719
-
720
- # Get the cached content file name, if specified, and if the file exists
721
- def getExistingCachedContentTreeFile
722
- if cachedContentFile == nil
723
- puts "No cached content file specified for location"
724
- return nil
725
- elsif File.exists?(cachedContentFile)
726
- return cachedContentFile
727
- else
728
- puts "Cached content file #{cachedContentFile} does not yet exist."
729
- return nil
730
- end
731
- end
732
-
733
- # Delete any existing cached content file
734
- def clearCachedContentFile
735
- if cachedContentFile and File.exists?(cachedContentFile)
736
- puts " deleting cached content file #{cachedContentFile} ..."
737
- File.delete(cachedContentFile)
738
- end
739
- end
740
-
741
- # Get the cached content tree (if any), read from the specified cached content file.
742
- def getCachedContentTree
743
- file = getExistingCachedContentTreeFile
744
- if file
745
- return ContentTree.readFromFile(file)
746
- else
747
- return nil
748
- end
749
- end
750
-
751
- # Read a map of file hashes (mapping from relative file name to hash value) from the
752
- # specified cached content file
753
- def getCachedContentTreeMapOfHashes
754
- file = getExistingCachedContentTreeFile
755
- if file
756
- puts "Reading cached file hashes from #{file} ..."
757
- return ContentTree.readMapOfHashesFromFile(file)
758
- else
759
- return [nil, {}]
760
- end
761
- end
762
-
763
- end
764
-
765
- # A directory of files on a local system. The corresponding content tree
766
- # can be calculated directly using Ruby library functions.
767
- class LocalContentLocation<ContentLocation
768
-
769
- # the base directory, for example of type Based::BaseDirectory. Methods invoked are: allFiles, subDirs and fullPath.
770
- # For file and dir objects returned by allFiles & subDirs, methods invoked are: relativePath and fullPath
771
- attr_reader :baseDirectory
772
- # the ruby class that generates the hash, e.g. Digest::SHA256
773
- attr_reader :hashClass
774
-
775
- def initialize(baseDirectory, hashClass, cachedContentFile = nil)
776
- super(cachedContentFile)
777
- @baseDirectory = baseDirectory
778
- @hashClass = hashClass
779
- end
780
-
781
- # get the full path of a relative path (i.e. of a file/directory within the base directory)
782
- def getFullPath(relativePath)
783
- return @baseDirectory.fullPath + relativePath
784
- end
785
-
786
- # get the content tree for this base directory by iterating over all
787
- # sub-directories and files within the base directory (and excluding the excluded files)
788
- # and calculating file hashes using the specified Ruby hash class
789
- # If there is an existing cached content file, use that to get the hash values
790
- # of files whose modification time is earlier than the time value for the cached content tree.
791
- # Also, if a cached content file is specified, write the final content tree back out to the cached content file.
792
- def getContentTree
793
- cachedTimeAndMapOfHashes = getCachedContentTreeMapOfHashes
794
- cachedTime = cachedTimeAndMapOfHashes[0]
795
- cachedMapOfHashes = cachedTimeAndMapOfHashes[1]
796
- contentTree = ContentTree.new()
797
- contentTree.time = Time.now.utc
798
- for subDir in @baseDirectory.subDirs
799
- contentTree.addDir(subDir.relativePath)
800
- end
801
- for file in @baseDirectory.allFiles
802
- cachedDigest = cachedMapOfHashes[file.relativePath]
803
- if cachedTime and cachedDigest and File.stat(file.fullPath).mtime < cachedTime
804
- digest = cachedDigest
805
- else
806
- digest = hashClass.file(file.fullPath).hexdigest
807
- end
808
- contentTree.addFile(file.relativePath, digest)
809
- end
810
- contentTree.sort!
811
- if cachedContentFile != nil
812
- contentTree.writeToFile(cachedContentFile)
813
- end
814
- return contentTree
815
- end
816
- end
817
-
818
- # A directory of files on a remote system
819
- class RemoteContentLocation<ContentLocation
820
- # the remote username@host value
821
- attr_reader :userAtHost
822
-
823
- # the base directory on the remote system
824
- attr_reader :baseDir
825
-
826
- def initialize(userAtHost, baseDir, cachedContentFile = nil)
827
- super(cachedContentFile)
828
- @userAtHost = userAtHost
829
- @baseDir = normalisedDir(baseDir)
830
- end
831
-
832
- # list files within the base directory on the remote userAtHost
833
- def listFiles()
834
- userAtHost.listFiles(baseDir)
835
- end
836
-
837
- # object required to execute SCP (e.g. "scp" or "pscp", possibly with extra args)
838
- def sshAndScp
839
- return userAtHost.sshAndScp
840
- end
841
-
842
- # get the full path of a relative path
843
- def getFullPath(relativePath)
844
- return baseDir + relativePath
845
- end
846
-
847
- # execute an SSH command on the remote host (or just pretend, if dryRun is true)
848
- def ssh(commandString, dryRun = false)
849
- userAtHost.sshAndScp.ssh(commandString, dryRun)
850
- end
851
-
852
- # list all sub-directories of the base directory on the remote host
853
- def listDirectories
854
- return userAtHost.listDirectories(baseDir)
855
- end
856
-
857
- # list all the file hashes of the files within the base directory
858
- def listFileHashes
859
- return userAtHost.listFileHashes(baseDir)
860
- end
861
-
862
- def to_s
863
- return userAtHost.locationDescriptor(baseDir)
864
- end
865
-
866
- # Get the content tree, from the cached content file if it exists,
867
- # otherwise get if from listing directories and files and hash values thereof
868
- # on the remote host. And also, if the cached content file name is specified,
869
- # write the content tree out to that file.
870
- def getContentTree
871
- if cachedContentFile and File.exists?(cachedContentFile)
872
- return ContentTree.readFromFile(cachedContentFile)
873
- else
874
- contentTree = userAtHost.getContentTree(baseDir)
875
- contentTree.sort!
876
- if cachedContentFile != nil
877
- contentTree.writeToFile(cachedContentFile)
878
- end
879
- return contentTree
880
- end
881
- end
882
-
883
- end
884
-
885
- # The operation of synchronising files on the remote directory with files on the local directory.
886
- class SyncOperation
887
- # The source location (presumed to be local)
888
- attr_reader :sourceLocation
889
-
890
- # The destination location (presumed to be remote)
891
- attr_reader :destinationLocation
892
-
893
- def initialize(sourceLocation, destinationLocation)
894
- @sourceLocation = sourceLocation
895
- @destinationLocation = destinationLocation
896
- end
897
-
898
- # Get the local and remote content trees
899
- def getContentTrees
900
- @sourceContent = @sourceLocation.getContentTree()
901
- @destinationContent = @destinationLocation.getContentTree()
902
- end
903
-
904
- # On the local and remote content trees, mark the copy and delete operations required
905
- # to sync the remote location to the local location.
906
- def markSyncOperations
907
- @sourceContent.markSyncOperationsForDestination(@destinationContent)
908
- puts " ================================================ "
909
- puts "After marking for sync --"
910
- puts ""
911
- puts "Local:"
912
- @sourceContent.showIndented()
913
- puts ""
914
- puts "Remote:"
915
- @destinationContent.showIndented()
916
- end
917
-
918
- # Delete the local and remote cached content files (which will force a full recalculation
919
- # of both content trees next time)
920
- def clearCachedContentFiles
921
- @sourceLocation.clearCachedContentFile()
922
- @destinationLocation.clearCachedContentFile()
923
- end
924
-
925
- # Do the sync. Options: :full = true means clear the cached content files first, :dryRun
926
- # means don't do the actual copies and deletes, but just show what they would be.
927
- def doSync(options = {})
928
- if options[:full]
929
- clearCachedContentFiles()
930
- end
931
- getContentTrees()
932
- markSyncOperations()
933
- dryRun = options[:dryRun]
934
- if not dryRun
935
- @destinationLocation.clearCachedContentFile()
936
- end
937
- doAllCopyOperations(dryRun)
938
- doAllDeleteOperations(dryRun)
939
- if (not dryRun and @destinationLocation.cachedContentFile and @sourceLocation.cachedContentFile and
940
- File.exists?(@sourceLocation.cachedContentFile))
941
- FileUtils::Verbose.cp(@sourceLocation.cachedContentFile, @destinationLocation.cachedContentFile)
942
- end
943
- end
944
-
945
- # Do all the copy operations, copying local directories or files which are missing from the remote location
946
- def doAllCopyOperations(dryRun)
947
- doCopyOperations(@sourceContent, @destinationContent, dryRun)
948
- end
949
-
950
- # Do all delete operations, deleting remote directories or files which do not exist at the local location
951
- def doAllDeleteOperations(dryRun)
952
- doDeleteOperations(@destinationContent, dryRun)
953
- end
954
-
955
- # Execute a (local) command, or, if dryRun, just pretend to execute it.
956
- # Raise an exception if the process exit status is not 0.
957
- def executeCommand(command, dryRun)
958
- puts "EXECUTE: #{command}"
959
- if not dryRun
960
- system(command)
961
- checkProcessStatus(command)
962
- end
963
- end
964
-
965
- # Recursively perform all marked copy operations from the source content tree to the
966
- # destination content tree, or if dryRun, just pretend to perform them.
967
- def doCopyOperations(sourceContent, destinationContent, dryRun)
968
- for dir in sourceContent.dirs
969
- if dir.copyDestination != nil
970
- sourcePath = sourceLocation.getFullPath(dir.relativePath)
971
- destinationPath = destinationLocation.getFullPath(dir.copyDestination.relativePath)
972
- destinationLocation.userAtHost.copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
973
- else
974
- doCopyOperations(dir, destinationContent.getDir(dir.name), dryRun)
975
- end
976
- end
977
- for file in sourceContent.files
978
- if file.copyDestination != nil
979
- sourcePath = sourceLocation.getFullPath(file.relativePath)
980
- destinationPath = destinationLocation.getFullPath(file.copyDestination.relativePath)
981
- destinationLocation.userAtHost.copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
982
- end
983
- end
984
- end
985
-
986
- # Recursively perform all marked delete operations on the destination content tree,
987
- # or if dryRun, just pretend to perform them.
988
- def doDeleteOperations(destinationContent, dryRun)
989
- for dir in destinationContent.dirs
990
- if dir.toBeDeleted
991
- dirPath = destinationLocation.getFullPath(dir.relativePath)
992
- destinationLocation.userAtHost.deleteDirectory(dirPath, dryRun)
993
- else
994
- doDeleteOperations(dir, dryRun)
995
- end
996
- end
997
- for file in destinationContent.files
998
- if file.toBeDeleted
999
- filePath = destinationLocation.getFullPath(file.relativePath)
1000
- destinationLocation.userAtHost.deleteFile(filePath, dryRun)
1001
- end
1002
- end
1003
- end
1004
- end
1005
- end
1
+ require 'time'
2
+ require 'net/ssh'
3
+ require 'net/scp'
4
+ require 'fileutils'
5
+
6
+ module Synqa
7
+
8
+ # ensure that a directory exists
9
+ def ensureDirectoryExists(directoryName)
10
+ if File.exist? directoryName
11
+ if not File.directory? directoryName
12
+ raise "#{directoryName} is a non-directory file"
13
+ end
14
+ else
15
+ FileUtils.makedirs(directoryName)
16
+ end
17
+ end
18
+
19
+ # Return the enumerated lines of the command's output
20
+ def getCommandOutput(command)
21
+ puts "#{command.inspect} ..."
22
+ return IO.popen(command)
23
+ end
24
+
25
+ # Check if the last executed process exited with status 0, if not, raise an exception
26
+ def checkProcessStatus(description)
27
+ processStatus = $?
28
+ if not processStatus.exited?
29
+ raise "#{description}: process did not exit normally"
30
+ end
31
+ exitStatus = processStatus.exitstatus
32
+ if exitStatus != 0
33
+ raise "#{description}: exit status = #{exitStatus}"
34
+ end
35
+ end
36
+
37
+ # An object representing a file path relative to a base directory, and a hash string
38
+ class RelativePathWithHash
39
+ # The relative file path (e.g. c:/dir/subdir/file.txt relative to c:/dir would be subdir/file.txt)
40
+ attr_reader :relativePath
41
+
42
+ # The hash code, e.g. a1c5b67fdb3cf0df8f1d29ae90561f9ad099bada44aeb6b2574ad9e15f2a84ed
43
+ attr_reader :hash
44
+
45
+ def initialize(relativePath, hash)
46
+ @relativePath = relativePath
47
+ @hash = hash
48
+ end
49
+
50
+ def inspect
51
+ return "RelativePathWithHash[#{relativePath}, #{hash}]"
52
+ end
53
+ end
54
+
55
+ # A command to be executed on the remote system which calculates a hash value for
56
+ # a file (of a given length), in the format: *hexadecimal-hash* *a-fixed-number-of-characters* *file-name*
57
+ class HashCommand
58
+ # The command - a string or array of strings e.g. "sha256sum" or ["sha256", "-r"]
59
+ attr_reader :command
60
+
61
+ # The length of the calculated hash value e.g. 64 for sha256
62
+ attr_reader :length
63
+
64
+ # The number of characters between the hash value and the file name (usually 1 or 2)
65
+ attr_reader :spacerLen
66
+
67
+ def initialize(command, length, spacerLen)
68
+ @command = command
69
+ @length = length
70
+ @spacerLen = spacerLen
71
+ end
72
+
73
+ # Parse a hash line relative to a base directory, returning a RelativePathWithHash
74
+ def parseFileHashLine(baseDir, fileHashLine)
75
+ hash = fileHashLine[0...length]
76
+ fullPath = fileHashLine[(length + spacerLen)..-1]
77
+ if fullPath.start_with?(baseDir)
78
+ return RelativePathWithHash.new(fullPath[baseDir.length..-1], hash)
79
+ else
80
+ raise "File #{fullPath} from hash line is not in base dir #{baseDir}"
81
+ end
82
+ end
83
+
84
+ def to_s
85
+ return command.join(" ")
86
+ end
87
+ end
88
+
89
+ # Hash command for sha256sum, which generates a 64 hexadecimal digit hash, and outputs two characters between
90
+ # the hash and the file name.
91
+ class Sha256SumCommand<HashCommand
92
+ def initialize
93
+ super(["sha256sum"], 64, 2)
94
+ end
95
+ end
96
+
97
+ # Hash command for sha256, which generates a 64 hexadecimal digit hash, and outputs one character between
98
+ # the hash and the file name, and which requires a "-r" argument to put the hash value first.
99
+ class Sha256Command<HashCommand
100
+ def initialize
101
+ super(["sha256", "-r"], 64, 1)
102
+ end
103
+ end
104
+
105
+ # Put "/" at the end of a directory name if it is not already there.
106
+ def normalisedDir(baseDir)
107
+ return baseDir.end_with?("/") ? baseDir : baseDir + "/"
108
+ end
109
+
110
+
111
+ # Base class for an object representing a remote system where the contents of a directory
112
+ # on the system are enumerated by one command to list all sub-directories and another command
113
+ # to list all files in the directory and their hash values.
114
+ class DirContentHost
115
+
116
+ # The HashCommand object used to calculate and parse hash values of files
117
+ attr_reader :hashCommand
118
+
119
+ # Prefix required for *find* command (usually nothing, since it should be on the system path)
120
+ attr_reader :pathPrefix
121
+
122
+ def initialize(hashCommand, pathPrefix = "")
123
+ @hashCommand = hashCommand
124
+ @pathPrefix = pathPrefix
125
+ end
126
+
127
+ # Generate the *find* command which will list all the sub-directories of the base directory
128
+ def findDirectoriesCommand(baseDir)
129
+ return ["#{@pathPrefix}find", baseDir, "-type", "d", "-print"]
130
+ end
131
+
132
+ # Return the list of sub-directories relative to the base directory
133
+ def listDirectories(baseDir)
134
+ baseDir = normalisedDir(baseDir)
135
+ command = findDirectoriesCommand(baseDir)
136
+ output = getCommandOutput(command)
137
+ directories = []
138
+ baseDirLen = baseDir.length
139
+ puts "Listing directories ..."
140
+ while (line = output.gets)
141
+ line = line.chomp
142
+ puts " #{line}"
143
+ if line.start_with?(baseDir)
144
+ directories << line[baseDirLen..-1]
145
+ else
146
+ raise "Directory #{line} is not a sub-directory of base directory #{baseDir}"
147
+ end
148
+ end
149
+ output.close()
150
+ checkProcessStatus(command)
151
+ return directories
152
+ end
153
+
154
+ # Generate the *find* command which will list all the files within the base directory
155
+ def findFilesCommand(baseDir)
156
+ return ["#{@pathPrefix}find", baseDir, "-type", "f", "-print"]
157
+ end
158
+
159
+ # List file hashes by executing the command to hash each file on the output of the
160
+ # *find* command which lists all files, and parse the output.
161
+ def listFileHashes(baseDir)
162
+ baseDir = normalisedDir(baseDir)
163
+ fileHashes = []
164
+ listFileHashLines(baseDir) do |fileHashLine|
165
+ fileHash = self.hashCommand.parseFileHashLine(baseDir, fileHashLine)
166
+ if fileHash != nil
167
+ fileHashes << fileHash
168
+ end
169
+ end
170
+ return fileHashes
171
+ end
172
+
173
+ # Construct the ContentTree for the given base directory
174
+ def getContentTree(baseDir)
175
+ contentTree = ContentTree.new()
176
+ contentTree.time = Time.now.utc
177
+ for dir in listDirectories(baseDir)
178
+ contentTree.addDir(dir)
179
+ end
180
+ for fileHash in listFileHashes(baseDir)
181
+ contentTree.addFile(fileHash.relativePath, fileHash.hash)
182
+ end
183
+ return contentTree
184
+ end
185
+ end
186
+
187
+ # Execute a (local) command, or, if dryRun, just pretend to execute it.
188
+ # Raise an exception if the process exit status is not 0.
189
+ def executeCommand(command, dryRun)
190
+ puts "EXECUTE: #{command}"
191
+ if not dryRun
192
+ system(command)
193
+ checkProcessStatus(command)
194
+ end
195
+ end
196
+
197
+ # Base SSH/SCP implementation
198
+ class BaseSshScp
199
+ attr_reader :userAtHost, :user, :host
200
+
201
+ def setUserAtHost(userAtHost)
202
+ @userAtHost = userAtHost
203
+ @user, @host = @userAtHost.split("@")
204
+ end
205
+
206
+ def close
207
+ # by default do nothing - close any cached connections
208
+ end
209
+
210
+ # delete remote directory (if dryRun is false) using "rm -r"
211
+ def deleteDirectory(dirPath, dryRun)
212
+ ssh("rm -r #{dirPath}", dryRun)
213
+ end
214
+
215
+ # delete remote file (if dryRun is false) using "rm"
216
+ def deleteFile(filePath, dryRun)
217
+ ssh("rm #{filePath}", dryRun)
218
+ end
219
+ end
220
+
221
+ # SSH/SCP using Ruby Net::SSH & Net::SCP
222
+ class InternalSshScp<BaseSshScp
223
+
224
+ def initialize
225
+ @connection = nil
226
+ end
227
+
228
+ def connection
229
+ if @connection == nil
230
+ puts "Opening SSH connection to #{user}@#{host} ..."
231
+ @connection = Net::SSH.start(host, user)
232
+ end
233
+ return @connection
234
+ end
235
+
236
+ def scpConnection
237
+ return connection.scp
238
+ end
239
+
240
+ def close()
241
+ if @connection != nil
242
+ puts "Closing SSH connection to #{user}@#{host} ..."
243
+ @connection.close()
244
+ @connection = nil
245
+ end
246
+ end
247
+
248
+ # execute command on remote host (if dryRun is false), yielding lines of output
249
+ def ssh(commandString, dryRun)
250
+ description = "SSH #{user}@#{host}: executing #{commandString}"
251
+ puts description
252
+ if not dryRun
253
+ outputText = connection.exec!(commandString)
254
+ if outputText != nil then
255
+ for line in outputText.split("\n") do
256
+ yield line
257
+ end
258
+ end
259
+ end
260
+ end
261
+
262
+ # copy a local directory to a remote directory (if dryRun is false)
263
+ def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
264
+ description = "SCP: copy directory #{sourcePath} to #{user}@#{host}:#{destinationPath}"
265
+ puts description
266
+ if not dryRun
267
+ scpConnection.upload!(sourcePath, destinationPath, :recursive => true)
268
+ end
269
+ end
270
+
271
+ # copy a local file to a remote directory (if dryRun is false)
272
+ def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
273
+ description = "SCP: copy file #{sourcePath} to #{user}@#{host}:#{destinationPath}"
274
+ puts description
275
+ if not dryRun
276
+ scpConnection.upload!(sourcePath, destinationPath)
277
+ end
278
+ end
279
+
280
+ end
281
+
282
+ # SSH/SCP using external commands, such as "plink" and "pscp"
283
+ class ExternalSshScp<BaseSshScp
284
+ # The SSH client, e.g. ["ssh"] or ["plink","-pw","mysecretpassword"] (i.e. command + args as an array)
285
+ attr_reader :shell
286
+
287
+ # The SCP client, e.g. ["scp"] or ["pscp","-pw","mysecretpassword"] (i.e. command + args as an array)
288
+ attr_reader :scpProgram
289
+
290
+ # The SCP command as a string
291
+ attr_reader :scpCommandString
292
+
293
+ def initialize(shell, scpProgram)
294
+ @shell = shell.is_a?(String) ? [shell] : shell
295
+ @scpProgram = scpProgram.is_a?(String) ? [scpProgram] : scpProgram
296
+ @scpCommandString = @scpProgram.join(" ")
297
+ end
298
+
299
+ # execute command on remote host (if dryRun is false), yielding lines of output
300
+ def ssh(commandString, dryRun)
301
+ puts "SSH #{userAtHost} (#{shell.join(" ")}): executing #{commandString}"
302
+ if not dryRun
303
+ output = getCommandOutput(shell + [userAtHost, commandString])
304
+ while (line = output.gets)
305
+ yield line.chomp
306
+ end
307
+ output.close()
308
+ checkProcessStatus("SSH #{userAtHost} #{commandString}")
309
+ end
310
+ end
311
+
312
+ # copy a local directory to a remote directory (if dryRun is false)
313
+ def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
314
+ executeCommand("#{@scpCommandString} -r #{sourcePath} #{userAtHost}:#{destinationPath}", dryRun)
315
+ end
316
+
317
+ # copy a local file to a remote directory (if dryRun is false)
318
+ def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
319
+ executeCommand("#{@scpCommandString} #{sourcePath} #{userAtHost}:#{destinationPath}", dryRun)
320
+ end
321
+
322
+ end
323
+
324
+ # Representation of a remote system accessible via SSH
325
+ class SshContentHost<DirContentHost
326
+
327
+ # The remote SSH/SCP login, e.g. SSH via "username@host.example.com"
328
+ attr_reader :sshAndScp
329
+
330
+ def initialize(userAtHost, hashCommand, sshAndScp = nil)
331
+ super(hashCommand)
332
+ @sshAndScp = sshAndScp != nil ? sshAndScp : InternalSshScp.new()
333
+ @sshAndScp.setUserAtHost(userAtHost)
334
+ end
335
+
336
+ def userAtHost
337
+ return @sshAndScp.userAtHost
338
+ end
339
+
340
+ def closeConnections()
341
+ @sshAndScp.close()
342
+ end
343
+
344
+ # Return readable description of base directory on remote system
345
+ def locationDescriptor(baseDir)
346
+ baseDir = normalisedDir(baseDir)
347
+ return "#{userAtHost}:#{baseDir} (connect = #{shell}/#{scpProgram}, hashCommand = #{hashCommand})"
348
+ end
349
+
350
+ # execute an SSH command on the remote system, yielding lines of output
351
+ # (or don't actually execute, if dryRun is false)
352
+ def ssh(commandString, dryRun = false)
353
+ sshAndScp.ssh(commandString, dryRun) do |line|
354
+ yield line
355
+ end
356
+ end
357
+
358
+ # delete a remote directory, if dryRun is false
359
+ def deleteDirectory(dirPath, dryRun)
360
+ sshAndScp.deleteDirectory(dirPath, dryRun)
361
+ end
362
+
363
+ # delete a remote file, if dryRun is false
364
+ def deleteFile(filePath, dryRun)
365
+ sshAndScp.deleteFile(filePath, dryRun)
366
+ end
367
+
368
+ # copy a local directory to a remote directory, if dryRun is false
369
+ def copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
370
+ sshAndScp.copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
371
+ end
372
+
373
+ # copy a local file to a remote directory, if dryRun is false
374
+ def copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
375
+ sshAndScp.copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
376
+ end
377
+
378
+ # Return a list of all subdirectories of the base directory (as paths relative to the base directory)
379
+ def listDirectories(baseDir)
380
+ baseDir = normalisedDir(baseDir)
381
+ puts "Listing directories ..."
382
+ directories = []
383
+ baseDirLen = baseDir.length
384
+ ssh(findDirectoriesCommand(baseDir).join(" ")) do |line|
385
+ puts " #{line}"
386
+ if line.start_with?(baseDir)
387
+ directories << line[baseDirLen..-1]
388
+ else
389
+ raise "Directory #{line} is not a sub-directory of base directory #{baseDir}"
390
+ end
391
+ end
392
+ return directories
393
+ end
394
+
395
+ # Yield lines of output from the command to display hash values and file names
396
+ # of all files within the base directory
397
+ def listFileHashLines(baseDir)
398
+ baseDir = normalisedDir(baseDir)
399
+ remoteFileHashLinesCommand = findFilesCommand(baseDir) + ["|", "xargs", "-r"] + @hashCommand.command
400
+ ssh(remoteFileHashLinesCommand.join(" ")) do |line|
401
+ puts " #{line}"
402
+ yield line
403
+ end
404
+ end
405
+
406
+ # List all files within the base directory to stdout
407
+ def listFiles(baseDir)
408
+ baseDir = normalisedDir(baseDir)
409
+ ssh(findFilesCommand(baseDir).join(" ")) do |line|
410
+ puts " #{line}"
411
+ end
412
+ end
413
+
414
+ end
415
+
416
+ # An object representing the content of a file within a ContentTree.
417
+ # The file may be marked for copying (if it's in a source ContentTree)
418
+ # or for deletion (if it's in a destination ContentTree)
419
+ class FileContent
420
+ # The name of the file
421
+ attr_reader :name
422
+
423
+ # The hash value of the file's contents
424
+ attr_reader :hash
425
+
426
+ # The components of the relative path where the file is found
427
+ attr_reader :parentPathElements
428
+
429
+ # The destination to which the file should be copied
430
+ attr_reader :copyDestination
431
+
432
+ # Should this file be deleted
433
+ attr_reader :toBeDeleted
434
+
435
+ def initialize(name, hash, parentPathElements)
436
+ @name = name
437
+ @hash = hash
438
+ @parentPathElements = parentPathElements
439
+ @copyDestination = nil
440
+ @toBeDeleted = false
441
+ end
442
+
443
+ # Mark this file to be copied to a destination directory (from a destination content tree)
444
+ def markToCopy(destinationDirectory)
445
+ @copyDestination = destinationDirectory
446
+ end
447
+
448
+ # Mark this file to be deleted
449
+ def markToDelete
450
+ @toBeDeleted = true
451
+ end
452
+
453
+ def to_s
454
+ return "#{name} (#{hash})"
455
+ end
456
+
457
+ # The relative name of this file in the content tree (relative to the base dir)
458
+ def relativePath
459
+ return (parentPathElements + [name]).join("/")
460
+ end
461
+ end
462
+
463
+ # A "content tree" consisting of a description of the contents of files and
464
+ # sub-directories within a base directory. The file contents are described via
465
+ # cryptographic hash values.
466
+ # Each sub-directory within a content tree is also represented as a ContentTree.
467
+ class ContentTree
468
+ # name of the sub-directory within the containing directory (or nil if this is the base directory)
469
+ attr_reader :name
470
+
471
+ # path elements from base directory leading to this one
472
+ attr_reader :pathElements
473
+
474
+ # files within this sub-directory (as FileContent's)
475
+ attr_reader :files
476
+
477
+ # immediate sub-directories of this directory
478
+ attr_reader :dirs
479
+
480
+ # the files within this sub-directory, indexed by file name
481
+ attr_reader :fileByName
482
+
483
+ # immediate sub-directories of this directory, indexed by name
484
+ attr_reader :dirByName
485
+
486
+ # where this directory should be copied to
487
+ attr_reader :copyDestination
488
+
489
+ # whether this directory should be deleted
490
+ attr_reader :toBeDeleted
491
+
492
+ # the UTC time (on the local system, even if this content tree represents a remote directory)
493
+ # that this content tree was constructed. Only set for the base directory.
494
+ attr_accessor :time
495
+
496
+ def initialize(name = nil, parentPathElements = nil)
497
+ @name = name
498
+ @pathElements = name == nil ? [] : parentPathElements + [name]
499
+ @files = []
500
+ @dirs = []
501
+ @fileByName = {}
502
+ @dirByName = {}
503
+ @copyDestination = nil
504
+ @toBeDeleted = false
505
+ @time = nil
506
+ end
507
+
508
+ # mark this directory to be copied to a destination directory
509
+ def markToCopy(destinationDirectory)
510
+ @copyDestination = destinationDirectory
511
+ end
512
+
513
+ # mark this directory (on a remote system) to be deleted
514
+ def markToDelete
515
+ @toBeDeleted = true
516
+ end
517
+
518
+ # the path of the directory that this content tree represents, relative to the base directory
519
+ def relativePath
520
+ return @pathElements.join("/")
521
+ end
522
+
523
+ # convert a path string to an array of path elements (or return it as is if it's already an array)
524
+ def getPathElements(path)
525
+ return path.is_a?(String) ? (path == "" ? [] : path.split("/")) : path
526
+ end
527
+
528
+ # get the content tree for a sub-directory (creating it if it doesn't yet exist)
529
+ def getContentTreeForSubDir(subDir)
530
+ dirContentTree = dirByName.fetch(subDir, nil)
531
+ if dirContentTree == nil
532
+ dirContentTree = ContentTree.new(subDir, @pathElements)
533
+ dirs << dirContentTree
534
+ dirByName[subDir] = dirContentTree
535
+ end
536
+ return dirContentTree
537
+ end
538
+
539
+ # add a sub-directory to this content tree
540
+ def addDir(dirPath)
541
+ pathElements = getPathElements(dirPath)
542
+ if pathElements.length > 0
543
+ pathStart = pathElements[0]
544
+ restOfPath = pathElements[1..-1]
545
+ getContentTreeForSubDir(pathStart).addDir(restOfPath)
546
+ end
547
+ end
548
+
549
+ # recursively sort the files and sub-directories of this content tree alphabetically
550
+ def sort!
551
+ dirs.sort_by! {|dir| dir.name}
552
+ files.sort_by! {|file| file.name}
553
+ for dir in dirs
554
+ dir.sort!
555
+ end
556
+ end
557
+
558
+ # given a relative path, add a file and hash value to this content tree
559
+ def addFile(filePath, hash)
560
+ pathElements = getPathElements(filePath)
561
+ if pathElements.length == 0
562
+ raise "Invalid file path: #{filePath.inspect}"
563
+ end
564
+ if pathElements.length == 1
565
+ fileName = pathElements[0]
566
+ fileContent = FileContent.new(fileName, hash, @pathElements)
567
+ files << fileContent
568
+ fileByName[fileName] = fileContent
569
+ else
570
+ pathStart = pathElements[0]
571
+ restOfPath = pathElements[1..-1]
572
+ getContentTreeForSubDir(pathStart).addFile(restOfPath, hash)
573
+ end
574
+ end
575
+
576
+ # date-time format for reading and writing times, e.g. "2007-12-23 13:03:99.012 +0000"
577
+ @@dateTimeFormat = "%Y-%m-%d %H:%M:%S.%L %z"
578
+
579
+ # pretty-print this content tree
580
+ def showIndented(name = "", indent = " ", currentIndent = "")
581
+ if time != nil
582
+ puts "#{currentIndent}[TIME: #{time.strftime(@@dateTimeFormat)}]"
583
+ end
584
+ if name != ""
585
+ puts "#{currentIndent}#{name}"
586
+ end
587
+ if copyDestination != nil
588
+ puts "#{currentIndent} [COPY to #{copyDestination.relativePath}]"
589
+ end
590
+ if toBeDeleted
591
+ puts "#{currentIndent} [DELETE]"
592
+ end
593
+ nextIndent = currentIndent + indent
594
+ for dir in dirs
595
+ dir.showIndented("#{dir.name}/", indent = indent, currentIndent = nextIndent)
596
+ end
597
+ for file in files
598
+ puts "#{nextIndent}#{file.name} - #{file.hash}"
599
+ if file.copyDestination != nil
600
+ puts "#{nextIndent} [COPY to #{file.copyDestination.relativePath}]"
601
+ end
602
+ if file.toBeDeleted
603
+ puts "#{nextIndent} [DELETE]"
604
+ end
605
+ end
606
+ end
607
+
608
+ # write this content tree to an open file, indented
609
+ def writeLinesToFile(outFile, prefix = "")
610
+ if time != nil
611
+ outFile.puts("T #{time.strftime(@@dateTimeFormat)}\n")
612
+ end
613
+ for dir in dirs
614
+ outFile.puts("D #{prefix}#{dir.name}\n")
615
+ dir.writeLinesToFile(outFile, "#{prefix}#{dir.name}/")
616
+ end
617
+ for file in files
618
+ outFile.puts("F #{file.hash} #{prefix}#{file.name}\n")
619
+ end
620
+ end
621
+
622
+ # write this content tree to a file (in a format which readFromFile can read back in)
623
+ def writeToFile(fileName)
624
+ puts "Writing content tree to file #{fileName} ..."
625
+ File.open(fileName, "w") do |outFile|
626
+ writeLinesToFile(outFile)
627
+ end
628
+ end
629
+
630
+ # regular expression for directory entries in content tree file
631
+ @@dirLineRegex = /^D (.*)$/
632
+
633
+ # regular expression for file entries in content tree file
634
+ @@fileLineRegex = /^F ([^ ]*) (.*)$/
635
+
636
+ # regular expression for time entry in content tree file
637
+ @@timeRegex = /^T (.*)$/
638
+
639
+ # read a content tree from a file (in format written by writeToFile)
640
+ def self.readFromFile(fileName)
641
+ contentTree = ContentTree.new()
642
+ puts "Reading content tree from #{fileName} ..."
643
+ IO.foreach(fileName) do |line|
644
+ dirLineMatch = @@dirLineRegex.match(line)
645
+ if dirLineMatch
646
+ dirName = dirLineMatch[1]
647
+ contentTree.addDir(dirName)
648
+ else
649
+ fileLineMatch = @@fileLineRegex.match(line)
650
+ if fileLineMatch
651
+ hash = fileLineMatch[1]
652
+ fileName = fileLineMatch[2]
653
+ contentTree.addFile(fileName, hash)
654
+ else
655
+ timeLineMatch = @@timeRegex.match(line)
656
+ if timeLineMatch
657
+ timeString = timeLineMatch[1]
658
+ contentTree.time = Time.strptime(timeString, @@dateTimeFormat)
659
+ else
660
+ raise "Invalid line in content tree file: #{line.inspect}"
661
+ end
662
+ end
663
+ end
664
+ end
665
+ return contentTree
666
+ end
667
+
668
+ # read a content tree as a map of hashes, i.e. from relative file path to hash value for the file
669
+ # Actually returns an array of the time entry (if any) and the map of hashes
670
+ def self.readMapOfHashesFromFile(fileName)
671
+ mapOfHashes = {}
672
+ time = nil
673
+ File.open(fileName).each_line do |line|
674
+ fileLineMatch = @@fileLineRegex.match(line)
675
+ if fileLineMatch
676
+ hash = fileLineMatch[1]
677
+ fileName = fileLineMatch[2]
678
+ mapOfHashes[fileName] = hash
679
+ end
680
+ timeLineMatch = @@timeRegex.match(line)
681
+ if timeLineMatch
682
+ timeString = timeLineMatch[1]
683
+ time = Time.strptime(timeString, @@dateTimeFormat)
684
+ end
685
+ end
686
+ return [time, mapOfHashes]
687
+ end
688
+
689
+ # Mark operations for this (source) content tree and the destination content tree
690
+ # in order to synch the destination content tree with this one
691
+ def markSyncOperationsForDestination(destination)
692
+ markCopyOperations(destination)
693
+ destination.markDeleteOptions(self)
694
+ end
695
+
696
+ # Get the named sub-directory content tree, if it exists
697
+ def getDir(dir)
698
+ return dirByName.fetch(dir, nil)
699
+ end
700
+
701
+ # Get the named file & hash value, if it exists
702
+ def getFile(file)
703
+ return fileByName.fetch(file, nil)
704
+ end
705
+
706
+ # Mark copy operations, given that the corresponding destination directory already exists.
707
+ # For files and directories that don't exist in the destination, mark them to be copied.
708
+ # For sub-directories that do exist, recursively mark the corresponding sub-directory copy operations.
709
+ def markCopyOperations(destinationDir)
710
+ for dir in dirs
711
+ destinationSubDir = destinationDir.getDir(dir.name)
712
+ if destinationSubDir != nil
713
+ dir.markCopyOperations(destinationSubDir)
714
+ else
715
+ dir.markToCopy(destinationDir)
716
+ end
717
+ end
718
+ for file in files
719
+ destinationFile = destinationDir.getFile(file.name)
720
+ if destinationFile == nil or destinationFile.hash != file.hash
721
+ file.markToCopy(destinationDir)
722
+ end
723
+ end
724
+ end
725
+
726
+ # Mark delete operations, given that the corresponding source directory exists.
727
+ # For files and directories that don't exist in the source, mark them to be deleted.
728
+ # For sub-directories that do exist, recursively mark the corresponding sub-directory delete operations.
729
+ def markDeleteOptions(sourceDir)
730
+ for dir in dirs
731
+ sourceSubDir = sourceDir.getDir(dir.name)
732
+ if sourceSubDir == nil
733
+ dir.markToDelete()
734
+ else
735
+ dir.markDeleteOptions(sourceSubDir)
736
+ end
737
+ end
738
+ for file in files
739
+ sourceFile = sourceDir.getFile(file.name)
740
+ if sourceFile == nil
741
+ file.markToDelete()
742
+ end
743
+ end
744
+ end
745
+ end
746
+
747
+ # Base class for a content location which consists of a base directory
748
+ # on a local or remote system.
749
+ class ContentLocation
750
+
751
+ # The name of a file used to hold a cached content tree for this location (can optionally be specified)
752
+ attr_reader :cachedContentFile
753
+
754
+ def initialize(cachedContentFile)
755
+ @cachedContentFile = cachedContentFile
756
+ end
757
+
758
+ # Get the cached content file name, if specified, and if the file exists
759
+ def getExistingCachedContentTreeFile
760
+ if cachedContentFile == nil
761
+ puts "No cached content file specified for location"
762
+ return nil
763
+ elsif File.exists?(cachedContentFile)
764
+ return cachedContentFile
765
+ else
766
+ puts "Cached content file #{cachedContentFile} does not yet exist."
767
+ return nil
768
+ end
769
+ end
770
+
771
+ # Delete any existing cached content file
772
+ def clearCachedContentFile
773
+ if cachedContentFile and File.exists?(cachedContentFile)
774
+ puts " deleting cached content file #{cachedContentFile} ..."
775
+ File.delete(cachedContentFile)
776
+ end
777
+ end
778
+
779
+ # Get the cached content tree (if any), read from the specified cached content file.
780
+ def getCachedContentTree
781
+ file = getExistingCachedContentTreeFile
782
+ if file
783
+ return ContentTree.readFromFile(file)
784
+ else
785
+ return nil
786
+ end
787
+ end
788
+
789
+ # Read a map of file hashes (mapping from relative file name to hash value) from the
790
+ # specified cached content file
791
+ def getCachedContentTreeMapOfHashes
792
+ file = getExistingCachedContentTreeFile
793
+ if file
794
+ puts "Reading cached file hashes from #{file} ..."
795
+ return ContentTree.readMapOfHashesFromFile(file)
796
+ else
797
+ return [nil, {}]
798
+ end
799
+ end
800
+
801
+ end
802
+
803
+ # A directory of files on a local system. The corresponding content tree
804
+ # can be calculated directly using Ruby library functions.
805
+ class LocalContentLocation<ContentLocation
806
+
807
+ # the base directory, for example of type Based::BaseDirectory. Methods invoked are: allFiles, subDirs and fullPath.
808
+ # For file and dir objects returned by allFiles & subDirs, methods invoked are: relativePath and fullPath
809
+ attr_reader :baseDirectory
810
+ # the ruby class that generates the hash, e.g. Digest::SHA256
811
+ attr_reader :hashClass
812
+
813
+ def initialize(baseDirectory, hashClass, cachedContentFile = nil)
814
+ super(cachedContentFile)
815
+ @baseDirectory = baseDirectory
816
+ @hashClass = hashClass
817
+ end
818
+
819
+ # get the full path of a relative path (i.e. of a file/directory within the base directory)
820
+ def getFullPath(relativePath)
821
+ return @baseDirectory.fullPath + relativePath
822
+ end
823
+
824
+ # get the content tree for this base directory by iterating over all
825
+ # sub-directories and files within the base directory (and excluding the excluded files)
826
+ # and calculating file hashes using the specified Ruby hash class
827
+ # If there is an existing cached content file, use that to get the hash values
828
+ # of files whose modification time is earlier than the time value for the cached content tree.
829
+ # Also, if a cached content file is specified, write the final content tree back out to the cached content file.
830
+ def getContentTree
831
+ cachedTimeAndMapOfHashes = getCachedContentTreeMapOfHashes
832
+ cachedTime = cachedTimeAndMapOfHashes[0]
833
+ cachedMapOfHashes = cachedTimeAndMapOfHashes[1]
834
+ contentTree = ContentTree.new()
835
+ contentTree.time = Time.now.utc
836
+ for subDir in @baseDirectory.subDirs
837
+ contentTree.addDir(subDir.relativePath)
838
+ end
839
+ for file in @baseDirectory.allFiles
840
+ cachedDigest = cachedMapOfHashes[file.relativePath]
841
+ if cachedTime and cachedDigest and File.stat(file.fullPath).mtime < cachedTime
842
+ digest = cachedDigest
843
+ else
844
+ digest = hashClass.file(file.fullPath).hexdigest
845
+ end
846
+ contentTree.addFile(file.relativePath, digest)
847
+ end
848
+ contentTree.sort!
849
+ if cachedContentFile != nil
850
+ contentTree.writeToFile(cachedContentFile)
851
+ end
852
+ return contentTree
853
+ end
854
+ end
855
+
856
+ # A directory of files on a remote system
857
+ class RemoteContentLocation<ContentLocation
858
+ # the remote SshContentHost
859
+ attr_reader :contentHost
860
+
861
+ # the base directory on the remote system
862
+ attr_reader :baseDir
863
+
864
+ def initialize(contentHost, baseDir, cachedContentFile = nil)
865
+ super(cachedContentFile)
866
+ @contentHost = contentHost
867
+ @baseDir = normalisedDir(baseDir)
868
+ end
869
+
870
+ def closeConnections
871
+ @contentHost.closeConnections()
872
+ end
873
+
874
+ # list files within the base directory on the remote contentHost
875
+ def listFiles()
876
+ contentHost.listFiles(baseDir)
877
+ end
878
+
879
+ # object required to execute SCP (e.g. "scp" or "pscp", possibly with extra args)
880
+ def sshAndScp
881
+ return contentHost.sshAndScp
882
+ end
883
+
884
+ # get the full path of a relative path
885
+ def getFullPath(relativePath)
886
+ return baseDir + relativePath
887
+ end
888
+
889
+ # execute an SSH command on the remote host (or just pretend, if dryRun is true)
890
+ def ssh(commandString, dryRun = false)
891
+ contentHost.sshAndScp.ssh(commandString, dryRun)
892
+ end
893
+
894
+ # list all sub-directories of the base directory on the remote host
895
+ def listDirectories
896
+ return contentHost.listDirectories(baseDir)
897
+ end
898
+
899
+ # list all the file hashes of the files within the base directory
900
+ def listFileHashes
901
+ return contentHost.listFileHashes(baseDir)
902
+ end
903
+
904
+ def to_s
905
+ return contentHost.locationDescriptor(baseDir)
906
+ end
907
+
908
+ # Get the content tree, from the cached content file if it exists,
909
+ # otherwise get if from listing directories and files and hash values thereof
910
+ # on the remote host. And also, if the cached content file name is specified,
911
+ # write the content tree out to that file.
912
+ def getContentTree
913
+ if cachedContentFile and File.exists?(cachedContentFile)
914
+ return ContentTree.readFromFile(cachedContentFile)
915
+ else
916
+ contentTree = contentHost.getContentTree(baseDir)
917
+ contentTree.sort!
918
+ if cachedContentFile != nil
919
+ contentTree.writeToFile(cachedContentFile)
920
+ end
921
+ return contentTree
922
+ end
923
+ end
924
+
925
+ end
926
+
927
+ # The operation of synchronising files on the remote directory with files on the local directory.
928
+ class SyncOperation
929
+ # The source location (presumed to be local)
930
+ attr_reader :sourceLocation
931
+
932
+ # The destination location (presumed to be remote)
933
+ attr_reader :destinationLocation
934
+
935
+ def initialize(sourceLocation, destinationLocation)
936
+ @sourceLocation = sourceLocation
937
+ @destinationLocation = destinationLocation
938
+ end
939
+
940
+ # Get the local and remote content trees
941
+ def getContentTrees
942
+ @sourceContent = @sourceLocation.getContentTree()
943
+ @destinationContent = @destinationLocation.getContentTree()
944
+ end
945
+
946
+ # On the local and remote content trees, mark the copy and delete operations required
947
+ # to sync the remote location to the local location.
948
+ def markSyncOperations
949
+ @sourceContent.markSyncOperationsForDestination(@destinationContent)
950
+ puts " ================================================ "
951
+ puts "After marking for sync --"
952
+ puts ""
953
+ puts "Local:"
954
+ @sourceContent.showIndented()
955
+ puts ""
956
+ puts "Remote:"
957
+ @destinationContent.showIndented()
958
+ end
959
+
960
+ # Delete the local and remote cached content files (which will force a full recalculation
961
+ # of both content trees next time)
962
+ def clearCachedContentFiles
963
+ @sourceLocation.clearCachedContentFile()
964
+ @destinationLocation.clearCachedContentFile()
965
+ end
966
+
967
+ # Do the sync. Options: :full = true means clear the cached content files first, :dryRun
968
+ # means don't do the actual copies and deletes, but just show what they would be.
969
+ def doSync(options = {})
970
+ if options[:full]
971
+ clearCachedContentFiles()
972
+ end
973
+ getContentTrees()
974
+ markSyncOperations()
975
+ dryRun = options[:dryRun]
976
+ if not dryRun
977
+ @destinationLocation.clearCachedContentFile()
978
+ end
979
+ doAllCopyOperations(dryRun)
980
+ doAllDeleteOperations(dryRun)
981
+ if (not dryRun and @destinationLocation.cachedContentFile and @sourceLocation.cachedContentFile and
982
+ File.exists?(@sourceLocation.cachedContentFile))
983
+ FileUtils::Verbose.cp(@sourceLocation.cachedContentFile, @destinationLocation.cachedContentFile)
984
+ end
985
+ closeConnections()
986
+ end
987
+
988
+ # Do all the copy operations, copying local directories or files which are missing from the remote location
989
+ def doAllCopyOperations(dryRun)
990
+ doCopyOperations(@sourceContent, @destinationContent, dryRun)
991
+ end
992
+
993
+ # Do all delete operations, deleting remote directories or files which do not exist at the local location
994
+ def doAllDeleteOperations(dryRun)
995
+ doDeleteOperations(@destinationContent, dryRun)
996
+ end
997
+
998
+ # Execute a (local) command, or, if dryRun, just pretend to execute it.
999
+ # Raise an exception if the process exit status is not 0.
1000
+ def executeCommand(command, dryRun)
1001
+ puts "EXECUTE: #{command}"
1002
+ if not dryRun
1003
+ system(command)
1004
+ checkProcessStatus(command)
1005
+ end
1006
+ end
1007
+
1008
+ # Recursively perform all marked copy operations from the source content tree to the
1009
+ # destination content tree, or if dryRun, just pretend to perform them.
1010
+ def doCopyOperations(sourceContent, destinationContent, dryRun)
1011
+ for dir in sourceContent.dirs
1012
+ if dir.copyDestination != nil
1013
+ sourcePath = sourceLocation.getFullPath(dir.relativePath)
1014
+ destinationPath = destinationLocation.getFullPath(dir.copyDestination.relativePath)
1015
+ destinationLocation.contentHost.copyLocalToRemoteDirectory(sourcePath, destinationPath, dryRun)
1016
+ else
1017
+ doCopyOperations(dir, destinationContent.getDir(dir.name), dryRun)
1018
+ end
1019
+ end
1020
+ for file in sourceContent.files
1021
+ if file.copyDestination != nil
1022
+ sourcePath = sourceLocation.getFullPath(file.relativePath)
1023
+ destinationPath = destinationLocation.getFullPath(file.copyDestination.relativePath)
1024
+ destinationLocation.contentHost.copyLocalFileToRemoteDirectory(sourcePath, destinationPath, dryRun)
1025
+ end
1026
+ end
1027
+ end
1028
+
1029
+ # Recursively perform all marked delete operations on the destination content tree,
1030
+ # or if dryRun, just pretend to perform them.
1031
+ def doDeleteOperations(destinationContent, dryRun)
1032
+ for dir in destinationContent.dirs
1033
+ if dir.toBeDeleted
1034
+ dirPath = destinationLocation.getFullPath(dir.relativePath)
1035
+ destinationLocation.contentHost.deleteDirectory(dirPath, dryRun)
1036
+ else
1037
+ doDeleteOperations(dir, dryRun)
1038
+ end
1039
+ end
1040
+ for file in destinationContent.files
1041
+ if file.toBeDeleted
1042
+ filePath = destinationLocation.getFullPath(file.relativePath)
1043
+ destinationLocation.contentHost.deleteFile(filePath, dryRun)
1044
+ end
1045
+ end
1046
+ end
1047
+
1048
+ def closeConnections
1049
+ destinationLocation.closeConnections()
1050
+ end
1051
+ end
1052
+ end