rsyncmanager 1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/rsyncmanager.rb +1210 -0
- data/tests/test_configuration.rb +3 -0
- metadata +41 -0
data/bin/rsyncmanager.rb
ADDED
@@ -0,0 +1,1210 @@
|
|
1
|
+
#!/usr/local/bin/ruby
|
2
|
+
#
|
3
|
+
# Author:: David Powers
|
4
|
+
# Copyright:: September, 2004
|
5
|
+
# Version:: 1.1
|
6
|
+
#
|
7
|
+
#Overview:
|
8
|
+
#
|
9
|
+
#Rsyncmanager is a daemon designed to simplify the task of coordinating
|
10
|
+
#multiple file transfers to or from a machine. It was originally written
|
11
|
+
#to serve as the managing hub of a backup system that used rsync to feed
|
12
|
+
#in files from many other machines so that all backups could be run from
|
13
|
+
#a single place. That said, it can be used to coordinate any kind of regular
|
14
|
+
#file transfer, from a single hourly copy of an email store from a primary
|
15
|
+
#to a backup server, to multiple concurrent rsyncs to multiple machines on
|
16
|
+
#varying schedules.
|
17
|
+
#
|
18
|
+
#
|
19
|
+
#Features:
|
20
|
+
#
|
21
|
+
#* Coordinate multiple file transfers using rsync, scp or a combination
|
22
|
+
# of the two
|
23
|
+
#* Limit the number of concurrently running file transfers to prevent
|
24
|
+
# resource starvation
|
25
|
+
#* Prevent file transfers from starting based on arbitrary conditions using
|
26
|
+
# external scripts (time of day, machine load, memory usage, phase of the
|
27
|
+
# moon...)
|
28
|
+
#* Run commands before and after a file transfer to handle setup and cleanup
|
29
|
+
# conditions (e.g. dump a database to disk, transfer it to a backup, then
|
30
|
+
# delete the dump)
|
31
|
+
#* Intelligently limit concurrent file transfers to or from the same machine
|
32
|
+
# to prevent resource limitations from slowing or starving a transfer
|
33
|
+
#* Limit concurrent file transfers based on arbitrary grouping (e.g. only
|
34
|
+
# one transfer from internal to external servers over a slow line at any
|
35
|
+
# given point)
|
36
|
+
#* Coordinate transfers between multiple machines (grouping over multiple
|
37
|
+
# machines, avoiding having two machines push data to a third at the same
|
38
|
+
# time, etc.)
|
39
|
+
#* Provides a clear web-based administrative report of all currently
|
40
|
+
# managed transfers and their current status (running, behind schedule,
|
41
|
+
# etc)
|
42
|
+
#* Intelligently schedule file transfers based on average runtimes so that
|
43
|
+
# transfers happen within a defined schedule but do not run constantly if
|
44
|
+
# not needed
|
45
|
+
|
46
|
+
require 'getoptlong'
|
47
|
+
require 'rexml/document'
|
48
|
+
require 'logger'
|
49
|
+
require 'net/http'
|
50
|
+
require 'optparse'
|
51
|
+
require 'socket'
|
52
|
+
require 'timeout'
|
53
|
+
require 'thread'
|
54
|
+
require 'webrick'
|
55
|
+
|
56
|
+
|
57
|
+
|
58
|
+
# parent class for configuration exceptions
|
59
|
+
class ConfigurationError < Exception
|
60
|
+
end
|
61
|
+
|
62
|
+
# raised when field content does not match allowed values
|
63
|
+
class InvalidContentError < ConfigurationError
|
64
|
+
end
|
65
|
+
|
66
|
+
# raised when a required element is not found
|
67
|
+
class MissingElementError < ConfigurationError
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
|
72
|
+
# holds configuration information about a single transfer and can start and manage
|
73
|
+
# a thread to run the transfer
|
74
|
+
class Transfer
|
75
|
+
attr_reader :options, :group, :from, :to, :commandpath, :description, :laststarted, :lastfinished, :frequency, :ignorepartialerrors, :cutoffprogram, :runbefore, :runafter
|
76
|
+
|
77
|
+
def initialize()
|
78
|
+
@laststarted = nil
|
79
|
+
@lastfinished = nil
|
80
|
+
@runtimes = Array.new()
|
81
|
+
|
82
|
+
# set @thread to a new dead thread to avoid special handling of thread state query functions
|
83
|
+
@thread = Thread.new {}
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
# turns the internal data into a command
|
88
|
+
def command()
|
89
|
+
return("#{@commandpath} #{@options} '#{@from.gsub(/\'/, '\\\&')}' '#{@to.gsub(/\'/, '\\\&')}'")
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
# returns true if this transfer is eligable to be run
|
94
|
+
def eligable?()
|
95
|
+
# never eligable if it is running right now
|
96
|
+
if(running?())
|
97
|
+
return(false)
|
98
|
+
end
|
99
|
+
|
100
|
+
# if we have runtimes compare the average runtime with the time remaining
|
101
|
+
# until our ideal runtime. We need to be reasonably close to our ideal time
|
102
|
+
# to be eligable
|
103
|
+
if(@runtimes.length > 0)
|
104
|
+
@perfecttime = @lastfinished + @frequency - averageruntime()
|
105
|
+
if(Time.now() < @perfecttime - (averageruntime() / 2))
|
106
|
+
return(false)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
return(true)
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
# returns the average runtime
|
115
|
+
def averageruntime()
|
116
|
+
totaltime = 0
|
117
|
+
@runtimes.each() { |runtime|
|
118
|
+
totaltime = totaltime + runtime
|
119
|
+
}
|
120
|
+
|
121
|
+
if(@runtimes.length == 0)
|
122
|
+
return(@frequency)
|
123
|
+
else
|
124
|
+
return(totaltime/@runtimes.length)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
# returns the current calculated priority of this transfer. The priority approaches 100 as the
|
130
|
+
# transfer nears the perfect time for the next start.
|
131
|
+
def priority()
|
132
|
+
$logger.debug("checking priority of \"#{@description}\"")
|
133
|
+
# always return 0 if the transfer is currently running
|
134
|
+
if(running?())
|
135
|
+
$logger.debug("transfer is running - returning 0")
|
136
|
+
return(0)
|
137
|
+
end
|
138
|
+
|
139
|
+
# if it has never been run - set priority to 250
|
140
|
+
if(@lastfinished == nil)
|
141
|
+
$logger.debug("transfer has never been run - returning 1000")
|
142
|
+
return(250)
|
143
|
+
end
|
144
|
+
|
145
|
+
@step = @frequency.to_f() / 100.0
|
146
|
+
|
147
|
+
# as we approach the perfect time (@lastfinished + @frequency) the priority should approach 100
|
148
|
+
priority = (Time.now - @lastfinished) / @step
|
149
|
+
$logger.debug("transfer has a real priority 0f #{priority}")
|
150
|
+
return(priority)
|
151
|
+
end
|
152
|
+
|
153
|
+
|
154
|
+
# runs this transfer in a thread
|
155
|
+
def run()
|
156
|
+
@thread = Thread.new() {
|
157
|
+
begin
|
158
|
+
# mark the start time
|
159
|
+
@laststarted = Time.now()
|
160
|
+
|
161
|
+
$logger.info("running \"#{@description}\"")
|
162
|
+
|
163
|
+
# run the runbefore command if it exists
|
164
|
+
if(@runbefore != '')
|
165
|
+
$logger.debug("running command #{@runbefore}")
|
166
|
+
runbeforestatus = system(@runbefore)
|
167
|
+
if(runbeforestatus == false)
|
168
|
+
$logger.error("error during run of #{@runbefore} before #{@description} - error code #{$?}")
|
169
|
+
raise(Exception)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
|
174
|
+
# actually run the command and report on success/failure
|
175
|
+
$logger.debug("running command #{command()}")
|
176
|
+
ransuccessfully = system(command())
|
177
|
+
|
178
|
+
# update variable and internal state
|
179
|
+
if(ransuccessfully == false and (@ignorepartialerrors == false or $?.to_i() != 5888))
|
180
|
+
$logger.error("error during run of #{command()} - error code #{$?}")
|
181
|
+
else
|
182
|
+
# run the runbefore command if it exists
|
183
|
+
if(@runafter != '')
|
184
|
+
$logger.debug("running command #{@runbefore}")
|
185
|
+
runafterstatus = system(@runafter)
|
186
|
+
if(runafterstatus == false)
|
187
|
+
$logger.error("error during run of #{@runafter} after #{@description} - error code #{$?}")
|
188
|
+
raise(Exception)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
$logger.info("\"#{@description}\" completed")
|
193
|
+
|
194
|
+
# mark the time of completion
|
195
|
+
@lastfinished = Time.now()
|
196
|
+
|
197
|
+
# set last runtime here to handle clock shifts
|
198
|
+
if(@lastfinished > @laststarted)
|
199
|
+
@runtimes << @lastfinished - @laststarted
|
200
|
+
|
201
|
+
# lop off the oldest time if we have more than 10
|
202
|
+
if(@runtimes.length > 10)
|
203
|
+
@runtimes = @runtimes[1..-1]
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
rescue Exception
|
208
|
+
$logger.error("error running \"#{description}\" - #{$!}")
|
209
|
+
end
|
210
|
+
}
|
211
|
+
end
|
212
|
+
|
213
|
+
|
214
|
+
def running?()
|
215
|
+
return(@thread.alive?())
|
216
|
+
end
|
217
|
+
|
218
|
+
|
219
|
+
def ignorepartialerrors=(newignorepartialerrors)
|
220
|
+
@ignorepartialerrors = newignorepartialerrors
|
221
|
+
$logger.debug("ignorepartialerrors set to \"#{@ignorepartialerrors}\"")
|
222
|
+
end
|
223
|
+
|
224
|
+
|
225
|
+
def commandpath=(newcommandpath)
|
226
|
+
@commandpath = newcommandpath
|
227
|
+
$logger.debug("commandpath set to \"#{@commandpath}\"")
|
228
|
+
end
|
229
|
+
|
230
|
+
|
231
|
+
def description=(newdescription)
|
232
|
+
@description = newdescription
|
233
|
+
$logger.debug("description set to \"#{@description}\"")
|
234
|
+
end
|
235
|
+
|
236
|
+
|
237
|
+
def options=(newoptions)
|
238
|
+
@options = newoptions
|
239
|
+
$logger.debug("options set to \"#{@options}\"")
|
240
|
+
end
|
241
|
+
|
242
|
+
|
243
|
+
def from=(newfrom)
|
244
|
+
@from = newfrom
|
245
|
+
$logger.debug("from set to \"#{@from}\"")
|
246
|
+
end
|
247
|
+
|
248
|
+
|
249
|
+
def to=(newto)
|
250
|
+
@to = newto
|
251
|
+
$logger.debug("to set to \"#{@to}\"")
|
252
|
+
end
|
253
|
+
|
254
|
+
|
255
|
+
def cutoffprogram=(newcutoffprogram)
|
256
|
+
@cutoffprogram = newcutoffprogram
|
257
|
+
$logger.debug("cutoffprogram set to \"#{@cutoffprogram}\"")
|
258
|
+
end
|
259
|
+
|
260
|
+
|
261
|
+
def runbefore=(newrunbefore)
|
262
|
+
@runbefore = newrunbefore
|
263
|
+
$logger.debug("runbefore set to \"#{@runbefore}\"")
|
264
|
+
end
|
265
|
+
|
266
|
+
|
267
|
+
def runafter=(newrunafter)
|
268
|
+
@runafter = newrunafter
|
269
|
+
$logger.debug("runafter set to \"#{@runafter}\"")
|
270
|
+
end
|
271
|
+
|
272
|
+
|
273
|
+
def group=(newgroup)
|
274
|
+
@group = newgroup
|
275
|
+
$logger.debug("group set to \"#{@group}\"")
|
276
|
+
end
|
277
|
+
|
278
|
+
|
279
|
+
# sets the frequency with a human readable string
|
280
|
+
def frequency=(newfrequency)
|
281
|
+
# if it is purely numeric assume minutes, otherwise get a suffix and compute from that
|
282
|
+
if(newfrequency =~ /^[0-9]+$/)
|
283
|
+
@frequency = newfrequency.to_i() * 60
|
284
|
+
elsif(newfrequency =~ /m/)
|
285
|
+
@frequency = newfrequency[/^[0-9]+/].to_i() * 60
|
286
|
+
elsif(newfrequency =~ /h/)
|
287
|
+
@frequency = newfrequency[/^[0-9]+/].to_i() * 60 * 60
|
288
|
+
elsif(newfrequency =~ /d/)
|
289
|
+
@frequency = newfrequency[/^[0-9]+/].to_i() * 24 * 60 * 60
|
290
|
+
else
|
291
|
+
raise(ArgumentError, "frequency could not be set to \"#{newfrequency}\"")
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
|
297
|
+
|
298
|
+
# Reads and holds the configuration for rsyncmanager
|
299
|
+
class Configuration
|
300
|
+
attr_reader :general, :transfers
|
301
|
+
|
302
|
+
# takes the filename of a configuration to read in
|
303
|
+
def initialize(filename)
|
304
|
+
# open the configuration file and read it in as an XML document
|
305
|
+
xmldoc = REXML::Document.new(File.open(filename, File::RDONLY))
|
306
|
+
|
307
|
+
parse_general(xmldoc.root.elements['general'])
|
308
|
+
parse_transfers(xmldoc.root.elements['transfers'])
|
309
|
+
end
|
310
|
+
|
311
|
+
|
312
|
+
# parses the general section of the configuration file
|
313
|
+
def parse_general(generalxml)
|
314
|
+
# setup a hash to hold the options
|
315
|
+
@general = Hash.new()
|
316
|
+
|
317
|
+
# get the maximum number of transfers to run at once
|
318
|
+
@general['maxtransfers'] = get_setting(generalxml.elements['maxtransfers'], 3, :to_i) { |setting|
|
319
|
+
if(setting <= 0)
|
320
|
+
raise(InvalidContentError, "maxtransfers must be set greater than 0")
|
321
|
+
end
|
322
|
+
}
|
323
|
+
|
324
|
+
# get the path of the program that controls when to cutoff running any more transfers
|
325
|
+
@general['cutoffprogram'] = get_setting(generalxml.elements['cutoffprogram'], "")
|
326
|
+
|
327
|
+
# set the command paths (automagically if necessary)
|
328
|
+
@general['rsyncpath'] = get_setting(generalxml.elements['rsyncpath'], get_command_path('rsync')) { |setting|
|
329
|
+
if(setting == "")
|
330
|
+
raise(ConfigurationError, "rsyncpath is not defined and the rsync binary could not be found on your system")
|
331
|
+
end
|
332
|
+
}
|
333
|
+
|
334
|
+
@general['scppath'] = get_setting(generalxml.elements['scppath'], get_command_path('scp')) { |setting|
|
335
|
+
if(setting == "")
|
336
|
+
raise(ConfigurationError, "scppath is not defined and the scp binary could not be found on your system")
|
337
|
+
end
|
338
|
+
}
|
339
|
+
|
340
|
+
# set the coordination parameters
|
341
|
+
@general['coordinate'] = Array.new()
|
342
|
+
|
343
|
+
coordinationxml = generalxml.elements['coordinate']
|
344
|
+
if(coordinationxml)
|
345
|
+
coordinationxml.each_element('url') { |url|
|
346
|
+
@general['coordinate'] << url.text()
|
347
|
+
}
|
348
|
+
end
|
349
|
+
|
350
|
+
# set the http server defaults
|
351
|
+
httpserverxml = generalxml.elements['httpserver']
|
352
|
+
if(httpserverxml)
|
353
|
+
@general['httpserver'] = Hash.new()
|
354
|
+
|
355
|
+
@general['httpserver']['port'] = get_setting(httpserverxml.elements['port'], 7962, :to_i)
|
356
|
+
@general['httpserver']['listen'] = get_setting(httpserverxml.elements['listen'], '0.0.0.0')
|
357
|
+
end
|
358
|
+
|
359
|
+
|
360
|
+
# get the defaults for transfers
|
361
|
+
parse_transferdefaults(generalxml.elements['transferdefaults'])
|
362
|
+
end
|
363
|
+
|
364
|
+
|
365
|
+
# parses the transferdefaults subsection of the general section of the configuration file
|
366
|
+
def parse_transferdefaults(transferdefaultsxml)
|
367
|
+
@transferdefaults = Hash.new()
|
368
|
+
|
369
|
+
@transferdefaults['type'] = get_setting(transferdefaultsxml.elements['type'], 'rsync') { |setting|
|
370
|
+
if(setting != 'rsync' and setting != 'scp')
|
371
|
+
raise(InvalidContentError, "transferdefaults/type must be either 'rsync' or 'scp' but is currently set to \"#{setting}\"")
|
372
|
+
end
|
373
|
+
}
|
374
|
+
|
375
|
+
if(transferdefaultsxml.elements['group'])
|
376
|
+
@transferdefaults['group'] = get_setting(transferdefaultsxml.elements['group'], "")
|
377
|
+
else
|
378
|
+
@transferdefaults['group'] = nil
|
379
|
+
end
|
380
|
+
|
381
|
+
@transferdefaults['options'] = get_setting(transferdefaultsxml.elements['options'], '-a -r -q')
|
382
|
+
|
383
|
+
@transferdefaults['ignorepartialerrors'] = get_setting(transferdefaultsxml.elements['ignorepartialerrors'], 'false') { |setting|
|
384
|
+
if(setting.downcase() == 'true')
|
385
|
+
setting = true
|
386
|
+
elsif(setting.downcase() == 'false')
|
387
|
+
setting = false
|
388
|
+
else
|
389
|
+
raise(InvalidContentError, "transferdefaults/ignorepartialerrors must be either 'true' or 'false' but is currently set to \"#{setting}\"")
|
390
|
+
end
|
391
|
+
}
|
392
|
+
|
393
|
+
@transferdefaults['frequency'] = get_setting(transferdefaultsxml.elements['frequency'], '1h') { |setting|
|
394
|
+
if(setting !~ /^[1-9][0-9]*(m|h|d|w)?$/)
|
395
|
+
raise(InvalidContentError, "transferdefaults/frequency (\"#{setting}\") cannot be parsed")
|
396
|
+
end
|
397
|
+
}
|
398
|
+
|
399
|
+
# get the path of the program that controls when to cutoff running any more transfers
|
400
|
+
@transferdefaults['cutoffprogram'] = get_setting(transferdefaultsxml.elements['cutoffprogram'], "")
|
401
|
+
|
402
|
+
# set default programs to run before and after a transfer - defaults to nil
|
403
|
+
@transferdefaults['runbefore'] = get_setting(transferdefaultsxml.elements['runbefore'], '')
|
404
|
+
@transferdefaults['runafter'] = get_setting(transferdefaultsxml.elements['runafter'], '')
|
405
|
+
end
|
406
|
+
|
407
|
+
|
408
|
+
# parses the transfers section of the configuration file
|
409
|
+
def parse_transfers(transfersxml)
|
410
|
+
@transfers = Array.new()
|
411
|
+
|
412
|
+
transfersxml.each_element('transfer') { |transferxml|
|
413
|
+
transfer = Transfer.new()
|
414
|
+
|
415
|
+
# set the description (optional)
|
416
|
+
transfer.description = get_setting(transferxml.elements['description'], 'No description given')
|
417
|
+
|
418
|
+
# set the type
|
419
|
+
type = get_setting(transferxml.elements['type'], @transferdefaults['type']) { |setting|
|
420
|
+
if(setting != 'rsync' and setting != 'scp')
|
421
|
+
raise(InvalidContentError, "type must be either 'rsync' or 'scp' but is currently set to \"#{setting}\"")
|
422
|
+
end
|
423
|
+
}
|
424
|
+
|
425
|
+
# set the command path
|
426
|
+
if(type == 'rsync')
|
427
|
+
transfer.commandpath = @general['rsyncpath']
|
428
|
+
elsif(type == 'scp')
|
429
|
+
transfer.commandpath = @general['scppath']
|
430
|
+
end
|
431
|
+
|
432
|
+
# set the group
|
433
|
+
if(transferxml.elements['group'])
|
434
|
+
transfer.group = get_setting(transferxml.elements['group'], '')
|
435
|
+
else
|
436
|
+
transfer.group = @transferdefaults['group']
|
437
|
+
end
|
438
|
+
|
439
|
+
# set the frequency
|
440
|
+
transfer.frequency = get_setting(transferxml.elements['frequency'], @transferdefaults['frequency']) { |setting|
|
441
|
+
if(setting !~ /^[1-9][0-9]*(m|h|d|w)?$/)
|
442
|
+
raise(InvalidContentError, "transferdefaults/frequency (\"#{setting}\") cannot be parsed")
|
443
|
+
end
|
444
|
+
}
|
445
|
+
|
446
|
+
# set the options
|
447
|
+
transfer.options = get_setting(transferxml.elements['options'], @transferdefaults['options'])
|
448
|
+
|
449
|
+
# set the from and to
|
450
|
+
transfer.from = get_setting(transferxml.elements['from'], "") { |setting|
|
451
|
+
if(setting == "")
|
452
|
+
raise(MissingElementError, "\"from\" must be set in each transfer section")
|
453
|
+
end
|
454
|
+
}
|
455
|
+
|
456
|
+
transfer.to = get_setting(transferxml.elements['to'], "") { |setting|
|
457
|
+
if(setting == nil)
|
458
|
+
raise(MissingElementError, "\"to\" must be set in each transfer section")
|
459
|
+
end
|
460
|
+
}
|
461
|
+
|
462
|
+
# set whether or not we are ignoring partial transfer errors
|
463
|
+
transfer.ignorepartialerrors = get_setting(transferxml.elements['ignorepartialerrors'], @transferdefaults['ignorepartialerrors']) { |setting|
|
464
|
+
if(setting.downcase() == 'true')
|
465
|
+
setting = true
|
466
|
+
elsif(setting.downcase() == 'false')
|
467
|
+
setting = false
|
468
|
+
elsif(setting != true and setting != false)
|
469
|
+
raise(InvalidContentError, "transferdefaults/ignorepartialerrors must be either 'true' or 'false' but is currently set to \"#{setting}\"")
|
470
|
+
end
|
471
|
+
}
|
472
|
+
|
473
|
+
# set a possible per-transfer cuttoffprogram
|
474
|
+
transfer.cutoffprogram = get_setting(transferxml.elements['cutoffprogram'], @transferdefaults['cutoffprogram'])
|
475
|
+
|
476
|
+
# set programs to run before and after a transfer - this can handle setup and cleanup that rsync and scp cannot
|
477
|
+
transfer.runbefore = get_setting(transferxml.elements['runbefore'], @transferdefaults['runbefore'])
|
478
|
+
transfer.runafter = get_setting(transferxml.elements['runafter'], @transferdefaults['runafter'])
|
479
|
+
|
480
|
+
@transfers << transfer
|
481
|
+
$logger.debug("Added transfer to list")
|
482
|
+
}
|
483
|
+
end
|
484
|
+
|
485
|
+
|
486
|
+
# finds the full path of a given command - returns false if the path cannot be found
|
487
|
+
def get_command_path(command)
|
488
|
+
paths = Array.[]('/sbin', '/bin', '/usr/sbin', '/usr/bin', '/usr/local/sbin', '/usr/local/bin')
|
489
|
+
|
490
|
+
paths.each() { |path|
|
491
|
+
if(test(?x, "#{path}/#{command}"))
|
492
|
+
return("#{path}/#{command}")
|
493
|
+
end
|
494
|
+
}
|
495
|
+
|
496
|
+
return(false)
|
497
|
+
end
|
498
|
+
|
499
|
+
|
500
|
+
# returns either the setting from the xml as text or default. conversionmethod is applied to the
|
501
|
+
# setting to ensure the final return value is of a known type. If a block is provided setting will
|
502
|
+
# be passed to the block (after conversion) for constraint checking or modification.
|
503
|
+
def get_setting(xml, default, conversionmethod = :to_s)
|
504
|
+
setting = nil
|
505
|
+
|
506
|
+
if(xml == nil)
|
507
|
+
setting = default
|
508
|
+
else
|
509
|
+
setting = xml.text()
|
510
|
+
end
|
511
|
+
|
512
|
+
# convert based on the conversion method
|
513
|
+
setting = setting.method(conversionmethod).call()
|
514
|
+
|
515
|
+
if(block_given?())
|
516
|
+
yield(setting)
|
517
|
+
end
|
518
|
+
|
519
|
+
return(setting)
|
520
|
+
end
|
521
|
+
end
|
522
|
+
|
523
|
+
|
524
|
+
|
525
|
+
# Serves a simple xml description of the transfers via WEBrick - mostly
|
526
|
+
# for collaboration purposes between instances of rsyncmanager running on
|
527
|
+
# separate machines
|
528
|
+
class XMLInformationServlet < WEBrick::HTTPServlet::AbstractServlet
|
529
|
+
# handles GET requests using the helper methods of the class
|
530
|
+
def do_GET(req, res)
|
531
|
+
begin
|
532
|
+
@configuration = @options[0]
|
533
|
+
if(!@configuration.kind_of?(Configuration))
|
534
|
+
raise(ArgumentError, "configuration must be a Configuration object")
|
535
|
+
end
|
536
|
+
|
537
|
+
timeout(30) {
|
538
|
+
xmldoc = REXML::Document.new("<transfers></transfers>")
|
539
|
+
xmldoc << REXML::XMLDecl.new()
|
540
|
+
|
541
|
+
# now the information
|
542
|
+
@configuration.transfers.each() { |transfer|
|
543
|
+
xmldoc.root.add_element(xml_transfer_description(transfer))
|
544
|
+
}
|
545
|
+
|
546
|
+
res.body = ""
|
547
|
+
xmldoc.write(res.body, 0)
|
548
|
+
}
|
549
|
+
rescue Exception
|
550
|
+
$logger.error("Error serving xml information page - #{$!}")
|
551
|
+
end
|
552
|
+
end
|
553
|
+
|
554
|
+
|
555
|
+
protected
|
556
|
+
|
557
|
+
|
558
|
+
# helper function to build a transfer information table
|
559
|
+
def xml_transfer_description(transfer)
|
560
|
+
now = Time.now()
|
561
|
+
transferelement = REXML::Element.new("transfer")
|
562
|
+
|
563
|
+
if(transfer.group != nil)
|
564
|
+
group = REXML::Element.new("group")
|
565
|
+
group.text = transfer.group.to_s
|
566
|
+
transferelement.add_element(group)
|
567
|
+
end
|
568
|
+
|
569
|
+
running = REXML::Element.new("running")
|
570
|
+
if(transfer.running?)
|
571
|
+
running.text = "true"
|
572
|
+
else
|
573
|
+
running.text = "false"
|
574
|
+
end
|
575
|
+
transferelement.add_element(running)
|
576
|
+
|
577
|
+
description = REXML::Element.new("description")
|
578
|
+
description.text = transfer.description
|
579
|
+
transferelement.add_element(description)
|
580
|
+
|
581
|
+
runbefore = REXML::Element.new("runbefore")
|
582
|
+
runbefore.text = transfer.runbefore
|
583
|
+
transferelement.add_element(runbefore)
|
584
|
+
|
585
|
+
runafter = REXML::Element.new("runafter")
|
586
|
+
runafter.text = transfer.runafter
|
587
|
+
transferelement.add_element(runafter)
|
588
|
+
|
589
|
+
from = REXML::Element.new("from")
|
590
|
+
from.text = transfer.from
|
591
|
+
transferelement.add_element(from)
|
592
|
+
|
593
|
+
to = REXML::Element.new("to")
|
594
|
+
to.text = transfer.to
|
595
|
+
transferelement.add_element(to)
|
596
|
+
|
597
|
+
command = REXML::Element.new("command")
|
598
|
+
command.text = transfer.command
|
599
|
+
transferelement.add_element(command)
|
600
|
+
|
601
|
+
frequency = REXML::Element.new("frequency")
|
602
|
+
frequency.text = transfer.frequency.to_s()
|
603
|
+
transferelement.add_element(frequency)
|
604
|
+
|
605
|
+
lastfinished = REXML::Element.new("lastfinished")
|
606
|
+
if(transfer.lastfinished)
|
607
|
+
lastfinished.text = transfer.lastfinished.to_s()
|
608
|
+
end
|
609
|
+
|
610
|
+
averageruntime = REXML::Element.new("averageruntime")
|
611
|
+
averageruntime.text = transfer.averageruntime.to_s()
|
612
|
+
transferelement.add_element(averageruntime)
|
613
|
+
|
614
|
+
priority = REXML::Element.new("priority")
|
615
|
+
priority.text = transfer.priority.to_s()
|
616
|
+
transferelement.add_element(priority)
|
617
|
+
|
618
|
+
return(transferelement)
|
619
|
+
end
|
620
|
+
|
621
|
+
end
|
622
|
+
|
623
|
+
|
624
|
+
|
625
|
+
# Serves a simple informational page via WEBrick
|
626
|
+
class HTTPInformationServlet < WEBrick::HTTPServlet::AbstractServlet
|
627
|
+
|
628
|
+
# helper function to produce a formatted table row
|
629
|
+
def http_table_row(label, info)
|
630
|
+
row = ""
|
631
|
+
row << "\t<tr>\n"
|
632
|
+
row << "\t\t<td class=\"label\">\n\t\t"
|
633
|
+
row << "#{label}\n"
|
634
|
+
row << "\t\t</td>\n"
|
635
|
+
row << "\t\t<td class=\"info\">\n"
|
636
|
+
row << "#{info}\n"
|
637
|
+
row << "\t\t</td>\n"
|
638
|
+
row << "\t</tr>\n"
|
639
|
+
|
640
|
+
return(row)
|
641
|
+
end
|
642
|
+
|
643
|
+
|
644
|
+
# helper function to build a key
|
645
|
+
def http_key()
|
646
|
+
key = <<KEY
|
647
|
+
<div class="keybox">
|
648
|
+
<div class="keytitle">
|
649
|
+
Transfer Status
|
650
|
+
</div>
|
651
|
+
<table>
|
652
|
+
<tr>
|
653
|
+
<td class="keylabel">
|
654
|
+
On-Time:
|
655
|
+
</td>
|
656
|
+
<td>
|
657
|
+
<div class="demobox" style="background-color: #e1ffe1">
|
658
|
+
</div>
|
659
|
+
</td>
|
660
|
+
</tr>
|
661
|
+
<tr>
|
662
|
+
<td class="keylabel">
|
663
|
+
Slightly Behind:
|
664
|
+
</td>
|
665
|
+
<td>
|
666
|
+
<div class="demobox" style="background-color: #ffffe1">
|
667
|
+
</div>
|
668
|
+
</td>
|
669
|
+
</tr>
|
670
|
+
<tr>
|
671
|
+
<td class="keylabel">
|
672
|
+
Overdue:
|
673
|
+
</td>
|
674
|
+
<td>
|
675
|
+
<div class="demobox" style="background-color: #ffe1e1">
|
676
|
+
</div>
|
677
|
+
</td>
|
678
|
+
</tr>
|
679
|
+
<tr>
|
680
|
+
<td class="keylabel">
|
681
|
+
Never Completed:
|
682
|
+
</td>
|
683
|
+
<td>
|
684
|
+
<div class="demobox" style="background-color: rgb(95%, 95%, 95%)">
|
685
|
+
</div>
|
686
|
+
</td>
|
687
|
+
</tr>
|
688
|
+
<tr>
|
689
|
+
<td class="keylabel">
|
690
|
+
Currently Running:
|
691
|
+
</td>
|
692
|
+
<td>
|
693
|
+
<div class="demobox" style="border: 3px solid #0042ff">
|
694
|
+
</div>
|
695
|
+
</td>
|
696
|
+
</tr>
|
697
|
+
</table>
|
698
|
+
</div>
|
699
|
+
KEY
|
700
|
+
return(key)
|
701
|
+
end
|
702
|
+
|
703
|
+
|
704
|
+
# helper function to build a stylesheet
|
705
|
+
def http_stylesheet()
|
706
|
+
stylesheet = <<STYLESHEET
|
707
|
+
<style type=\"text/css\">
|
708
|
+
.transfersummary {
|
709
|
+
margin-left: auto;
|
710
|
+
margin-right: auto;
|
711
|
+
margin-bottom: 10px;
|
712
|
+
width: 90%;
|
713
|
+
border: 1px solid black;
|
714
|
+
background-color: rgb(95%, 95%, 95%);
|
715
|
+
}
|
716
|
+
.label {
|
717
|
+
width: 50px;
|
718
|
+
white-space: nowrap;
|
719
|
+
font-size: 12px;
|
720
|
+
font-weight: bold;
|
721
|
+
}
|
722
|
+
.info {
|
723
|
+
font-size: 10px;
|
724
|
+
font-weight: normal;
|
725
|
+
}
|
726
|
+
|
727
|
+
.keybox {
|
728
|
+
border: 1px solid black;
|
729
|
+
background-color: #ffffff;
|
730
|
+
z-index: 5;
|
731
|
+
position: absolute;
|
732
|
+
top: 3px;
|
733
|
+
right: 3px;
|
734
|
+
}
|
735
|
+
|
736
|
+
.keylabel {
|
737
|
+
font-size: 8px;
|
738
|
+
}
|
739
|
+
|
740
|
+
.keytitle {
|
741
|
+
font-size: 9px;
|
742
|
+
font-weight: bold;
|
743
|
+
color: white;
|
744
|
+
text-align: center;
|
745
|
+
background-color: rgb(20%,20%,20%);
|
746
|
+
padding: 2px;
|
747
|
+
}
|
748
|
+
|
749
|
+
.demobox {
|
750
|
+
width: 20px;
|
751
|
+
height: 20px;
|
752
|
+
border: 1px solid black;
|
753
|
+
margin-left: auto;
|
754
|
+
margin-right: auto;
|
755
|
+
}
|
756
|
+
|
757
|
+
</style>\n
|
758
|
+
STYLESHEET
|
759
|
+
return(stylesheet)
|
760
|
+
end
|
761
|
+
|
762
|
+
|
763
|
+
# helper function to build a transfer information table
|
764
|
+
def http_transfer_table(transfer)
|
765
|
+
now = Time.now()
|
766
|
+
|
767
|
+
table = "<table class=\"transfersummary\" style=\""
|
768
|
+
|
769
|
+
# change the color of the table based on how close to on-schedule we are
|
770
|
+
if(transfer.lastfinished != nil)
|
771
|
+
green = transfer.lastfinished + transfer.frequency + transfer.frequency/10
|
772
|
+
yellow = transfer.lastfinished + transfer.frequency*2
|
773
|
+
if(now < green)
|
774
|
+
table << "background-color: #e1ffe1; "
|
775
|
+
elsif(now < yellow)
|
776
|
+
table << "background-color: #ffffe1; "
|
777
|
+
else
|
778
|
+
table << "background-color: #ffe1e1; "
|
779
|
+
end
|
780
|
+
end
|
781
|
+
|
782
|
+
# if it is running, mark it
|
783
|
+
if(transfer.running?())
|
784
|
+
table << "border: 3px solid #0042ff;"
|
785
|
+
end
|
786
|
+
table << "\">\n"
|
787
|
+
|
788
|
+
if(transfer.group != nil)
|
789
|
+
table << http_table_row("Group:", transfer.group)
|
790
|
+
end
|
791
|
+
|
792
|
+
table << http_table_row("Description:", transfer.description)
|
793
|
+
if(transfer.runbefore != '')
|
794
|
+
table << http_table_row("Run Before:", transfer.runbefore)
|
795
|
+
end
|
796
|
+
table << http_table_row("Command:", transfer.command)
|
797
|
+
if(transfer.runafter != '')
|
798
|
+
table << http_table_row("Run After:", transfer.runafter)
|
799
|
+
end
|
800
|
+
table << http_table_row("Ideal Frequency:", "#{(transfer.frequency/60).round()} minutes")
|
801
|
+
table << http_table_row("Running:", transfer.running?())
|
802
|
+
if(transfer.lastfinished)
|
803
|
+
table << http_table_row("Last Finished:", "#{transfer.lastfinished} (#{((now - transfer.lastfinished)/60).round()} minutes ago)")
|
804
|
+
else
|
805
|
+
table << http_table_row("Last Finished:", "")
|
806
|
+
end
|
807
|
+
table << http_table_row("Average Run Time:", "#{(transfer.averageruntime/60).round()} minutes")
|
808
|
+
table << http_table_row("Current Priority:", transfer.priority.to_f.round())
|
809
|
+
table << "</table>\n"
|
810
|
+
|
811
|
+
return(table)
|
812
|
+
end
|
813
|
+
|
814
|
+
|
815
|
+
# handles GET requests using the helper methods of the class
|
816
|
+
def do_GET(req, res)
|
817
|
+
begin
|
818
|
+
@configuration = @options[0]
|
819
|
+
if(!@configuration.kind_of?(Configuration))
|
820
|
+
raise(ArgumentError, "configuration must be a Configuration object")
|
821
|
+
end
|
822
|
+
|
823
|
+
timeout(30) {
|
824
|
+
res.body = ""
|
825
|
+
res.body << "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
|
826
|
+
res.body << "<html>\n"
|
827
|
+
res.body << "<head>\n"
|
828
|
+
res.body << "\t<meta http-equiv=\"content-type\" content=\"text/html; charset=ISO-8859-1\"/>\n"
|
829
|
+
res.body << "\t<title>Summary</title>\n"
|
830
|
+
|
831
|
+
# style sheet
|
832
|
+
res.body << http_stylesheet()
|
833
|
+
|
834
|
+
# reload the page automatically every 5 minutes
|
835
|
+
res.body << "\t<script type=\"text/JavaScript\">\n"
|
836
|
+
res.body << "\t\tfunction reloadpage()\n"
|
837
|
+
res.body << "\t\t{\n"
|
838
|
+
res.body << "\t\t\tdocument.location.href = document.location.href;\n"
|
839
|
+
res.body << "\t\t}\n\n"
|
840
|
+
res.body << "\t\tsetInterval('reloadpage()', 600000);\n"
|
841
|
+
res.body << "\t</script>\n"
|
842
|
+
|
843
|
+
res.body << "</head>\n"
|
844
|
+
res.body << "<body>\n"
|
845
|
+
|
846
|
+
# build a nice little key
|
847
|
+
res.body << http_key()
|
848
|
+
|
849
|
+
# now the information
|
850
|
+
@configuration.transfers.each() { |transfer|
|
851
|
+
# build the table
|
852
|
+
table = http_transfer_table(transfer)
|
853
|
+
res.body << table
|
854
|
+
}
|
855
|
+
|
856
|
+
res.body << "</body>\n"
|
857
|
+
res.body << "</html>\n"
|
858
|
+
}
|
859
|
+
rescue Exception
|
860
|
+
$logger.error("Error serving http information page - #{$!}")
|
861
|
+
end
|
862
|
+
end
|
863
|
+
end
|
864
|
+
|
865
|
+
|
866
|
+
# Collects the non-class methods for rsyncmanager
|
867
|
+
module RsyncManager
|
868
|
+
|
869
|
+
# Sets the $options variable. Using defaults where none
|
870
|
+
# are provided and entries from the command line where possible
|
871
|
+
def RsyncManager.set_options()
|
872
|
+
$options = Hash.new()
|
873
|
+
$options['config'] = 'rsyncmanager.xml'
|
874
|
+
$options['log'] = '/var/log/rsyncmanager.log'
|
875
|
+
$options['verbosity'] = Logger::INFO
|
876
|
+
$options['pidfile'] = '/var/run/rsyncmanager.pid'
|
877
|
+
$options['daemon'] = true
|
878
|
+
|
879
|
+
|
880
|
+
opts = OptionParser.new() { |opts|
|
881
|
+
opts.on_tail("-v", "--verbosity VERBOSITY", Integer, "(0-4) 0 being fatal errors only, default 3") { |verbosity|
|
882
|
+
if(verbosity == 0)
|
883
|
+
$options['verbosity'] = Logger::FATAL
|
884
|
+
elsif(verbosity == 1)
|
885
|
+
$options['verbosity'] = Logger::ERROR
|
886
|
+
elsif(verbosity == 2)
|
887
|
+
$options['verbosity'] = Logger::WARN
|
888
|
+
elsif(verbosity == 3)
|
889
|
+
$options['verbosity'] = Logger::INFO
|
890
|
+
elsif(verbosity == 4)
|
891
|
+
$options['verbosity'] = Logger::DEBUG
|
892
|
+
else
|
893
|
+
$stderr.print("verbosity must be set between 0 and 4\n")
|
894
|
+
exit(1)
|
895
|
+
end
|
896
|
+
}
|
897
|
+
|
898
|
+
opts.on_tail("-h", "--help", "Print this message") {
|
899
|
+
print(opts)
|
900
|
+
exit()
|
901
|
+
}
|
902
|
+
|
903
|
+
opts.on("-c", "--config CONFIGFILE", "Location of the configuration file") { |filename|
|
904
|
+
$options['config'] = filename
|
905
|
+
}
|
906
|
+
|
907
|
+
opts.on("-l", "--log LOGFILE", "Location of the log file") { |filename|
|
908
|
+
$options['log'] = filename
|
909
|
+
}
|
910
|
+
|
911
|
+
opts.on("-p", "--pidfile PIDFILE", "Location of the pid file") { |filename|
|
912
|
+
$options['pidfile'] = filename
|
913
|
+
}
|
914
|
+
|
915
|
+
opts.on("-d", "--nodaemon", "Do not fork into the background") {
|
916
|
+
$options['daemon'] = false
|
917
|
+
}
|
918
|
+
}
|
919
|
+
|
920
|
+
opts.parse(ARGV)
|
921
|
+
end
|
922
|
+
|
923
|
+
|
924
|
+
# forks the program into the background and detaches it from the terminal
|
925
|
+
def RsyncManager.daemonize()
|
926
|
+
# fork into the background
|
927
|
+
# Parent exits, child continues.
|
928
|
+
if(fork())
|
929
|
+
exit()
|
930
|
+
end
|
931
|
+
|
932
|
+
# Become session leader.
|
933
|
+
Process.setsid()
|
934
|
+
|
935
|
+
# Zap session leader.
|
936
|
+
if(fork())
|
937
|
+
exit()
|
938
|
+
end
|
939
|
+
|
940
|
+
# Release old working directory.
|
941
|
+
Dir.chdir "/"
|
942
|
+
|
943
|
+
# Ensure sensible umask. Adjust as needed.
|
944
|
+
File.umask 0000
|
945
|
+
|
946
|
+
# reopen the file descriptors to nothing
|
947
|
+
STDIN.reopen '/dev/null'
|
948
|
+
STDOUT.reopen '/dev/null', 'a'
|
949
|
+
STDERR.reopen STDOUT
|
950
|
+
end
|
951
|
+
|
952
|
+
|
953
|
+
# writes the PID to a file
|
954
|
+
def RsyncManager.writepid(pidfile)
|
955
|
+
File.open(pidfile, File::WRONLY|File::CREAT|File::TRUNC, 0644) { |fp|
|
956
|
+
fp.write("#{Process.pid}\n")
|
957
|
+
}
|
958
|
+
end
|
959
|
+
|
960
|
+
|
961
|
+
# looks in each string in *strings and returns the first ip-address or hostname that
|
962
|
+
# it can find (assumes the strings will be of the form used in rsync or scp to designate
|
963
|
+
# a remote host)
|
964
|
+
def RsyncManager.extract_ip(*strings)
|
965
|
+
strings.each() { |str|
|
966
|
+
ipaddress = str[/^([^\/:@]*@)?([^:]+):/, 2]
|
967
|
+
if(ipaddress != nil)
|
968
|
+
return(ipaddress)
|
969
|
+
end
|
970
|
+
}
|
971
|
+
|
972
|
+
return(nil)
|
973
|
+
end
|
974
|
+
|
975
|
+
|
976
|
+
# pulls down xml from the given url and adds the information to runningipaddresses and runninggroups
|
977
|
+
# so that it can be used in making decisions about which transfers to run on this machine
|
978
|
+
def RsyncManager.get_coordination_info(url, runningipaddresses, runninggroups)
|
979
|
+
begin
|
980
|
+
# we can't afford to wait around forever for an answer - we'll take what we can get
|
981
|
+
# and make a best guess if we don't get anything
|
982
|
+
timeout(10) {
|
983
|
+
# we only handle HTTP for now
|
984
|
+
response = Net::HTTP.get(URI.parse(url))
|
985
|
+
xmldoc = REXML::Document.new(response)
|
986
|
+
xmldoc.root.each_element('transfer') { |transfer|
|
987
|
+
if(transfer.elements['running'].text == 'true')
|
988
|
+
group = transfer.elements['group']
|
989
|
+
from = transfer.elements['from'].text
|
990
|
+
to = transfer.elements['to'].text
|
991
|
+
if(group != nil)
|
992
|
+
runninggroups[group] = true
|
993
|
+
end
|
994
|
+
runningipaddresses[extract_ip(from, to)] = 1
|
995
|
+
end
|
996
|
+
}
|
997
|
+
}
|
998
|
+
rescue Exception
|
999
|
+
$logger.warn("Unable to get coordination information from #{url} - #{$!}")
|
1000
|
+
end
|
1001
|
+
end
|
1002
|
+
|
1003
|
+
|
1004
|
+
# The main logic loop of the program
|
1005
|
+
def RsyncManager.main()
|
1006
|
+
# blow up the whole thing for unhandled exceptions
|
1007
|
+
Thread.abort_on_exception = true
|
1008
|
+
|
1009
|
+
# set $options
|
1010
|
+
set_options()
|
1011
|
+
|
1012
|
+
# create a logger
|
1013
|
+
$logger = Logger.new($options['log'], 10, 1024000)
|
1014
|
+
$logger.level = $options['verbosity']
|
1015
|
+
$logger.info("rsyncmanager started")
|
1016
|
+
|
1017
|
+
# read in the configuration
|
1018
|
+
configuration = Configuration.new($options['config'])
|
1019
|
+
|
1020
|
+
# go into the background
|
1021
|
+
if($options['daemon'] == true)
|
1022
|
+
daemonize()
|
1023
|
+
end
|
1024
|
+
|
1025
|
+
# write the pid to a file
|
1026
|
+
writepid($options['pidfile'])
|
1027
|
+
|
1028
|
+
# trap term so we can exit gracefully
|
1029
|
+
trap("TERM") {
|
1030
|
+
$logger.info("recieved TERM signal")
|
1031
|
+
begin
|
1032
|
+
# kill the child processes
|
1033
|
+
File.unlink($options['pidfile'])
|
1034
|
+
exit(0)
|
1035
|
+
rescue Exception
|
1036
|
+
# do nothing except note that we failed
|
1037
|
+
$stderr.print "Unable to delete pid file \"#{$options['pidfile']}\" - #{$!}\n"
|
1038
|
+
exit(1)
|
1039
|
+
end
|
1040
|
+
}
|
1041
|
+
|
1042
|
+
# start an informational webserver if it has been requested
|
1043
|
+
webserver = nil
|
1044
|
+
if(configuration.general['httpserver'] != nil)
|
1045
|
+
$logger.info("Starting HTTP server on #{configuration.general['httpserver']['listen']}:#{configuration.general['httpserver']['port']}")
|
1046
|
+
|
1047
|
+
config = Hash.new()
|
1048
|
+
config[:Port] = configuration.general['httpserver']['port']
|
1049
|
+
config[:BindAddress] = configuration.general['httpserver']['listen']
|
1050
|
+
webserver = WEBrick::HTTPServer.new(config)
|
1051
|
+
webserver.mount('/info', HTTPInformationServlet, configuration)
|
1052
|
+
webserver.mount('/xml', XMLInformationServlet, configuration)
|
1053
|
+
|
1054
|
+
# start the server in its own thread so it doesn't stop us from doing real work
|
1055
|
+
Thread.new() {
|
1056
|
+
begin
|
1057
|
+
webserver.start()
|
1058
|
+
rescue Exception
|
1059
|
+
$logger.error("Stopping web server - #{$!}")
|
1060
|
+
end
|
1061
|
+
}
|
1062
|
+
end
|
1063
|
+
|
1064
|
+
# run in a loop, scheduling and running transfers. Set foundtransfer == true at the beginning to "trick"
|
1065
|
+
# it into skipping the initial 1 minute sleep
|
1066
|
+
foundtransfer = true
|
1067
|
+
while(true)
|
1068
|
+
# sleep for a minute if we didn't find any transfers available to run
|
1069
|
+
if(!foundtransfer)
|
1070
|
+
$logger.debug("sleeping")
|
1071
|
+
sleep(60)
|
1072
|
+
$logger.debug("waking up and considering which transfers need to be run")
|
1073
|
+
end
|
1074
|
+
|
1075
|
+
$logger.debug("choosing a new transfer to run")
|
1076
|
+
|
1077
|
+
# set foundtransfer to false so we will sleep if there are no more available transfers.
|
1078
|
+
foundtransfer = false
|
1079
|
+
|
1080
|
+
# skip all this if we have a global cutoff program that returns > 0
|
1081
|
+
if(configuration.general['cutoffprogram'] != '' and !system(configuration.general['cutoffprogram']))
|
1082
|
+
next
|
1083
|
+
end
|
1084
|
+
|
1085
|
+
# collect information about all the currently running transfers
|
1086
|
+
begin
|
1087
|
+
numberrunning = 0
|
1088
|
+
runningipaddresses = Hash.new()
|
1089
|
+
runninggroups = Hash.new()
|
1090
|
+
|
1091
|
+
# if we are coordinating with any other information sources, bring them in here
|
1092
|
+
configuration.general['coordinate'].each() { |url|
|
1093
|
+
get_coordination_info(url, runningipaddresses, runninggroups)
|
1094
|
+
}
|
1095
|
+
|
1096
|
+
# first loop through the transfers to get a count of the running transfers as well as to tag any
|
1097
|
+
# remote addresses that are currently handling transfers so we don't overload them
|
1098
|
+
configuration.transfers.each() { |transfer|
|
1099
|
+
if(transfer.running?())
|
1100
|
+
numberrunning = numberrunning + 1
|
1101
|
+
runningipaddresses[extract_ip(transfer.from, transfer.to)] = 1
|
1102
|
+
if(transfer.group != nil)
|
1103
|
+
runninggroups[transfer.group] = true
|
1104
|
+
end
|
1105
|
+
end
|
1106
|
+
}
|
1107
|
+
rescue Exception
|
1108
|
+
$logger.fatal("Error calculating running transfers - #{$!} - Exiting")
|
1109
|
+
$@.each() { |line|
|
1110
|
+
$logger.fatal(line)
|
1111
|
+
}
|
1112
|
+
exit(1)
|
1113
|
+
end
|
1114
|
+
|
1115
|
+
# now collect the eligable transfers for running
|
1116
|
+
begin
|
1117
|
+
# collect the possible transfers in a hash of arrays. The arrays are hashed by priority
|
1118
|
+
# so that we can easily randomly choose from amongst groups of like priority transfers
|
1119
|
+
eligabletransfers = Hash.new()
|
1120
|
+
configuration.transfers.each() { |transfer|
|
1121
|
+
if(transfer.eligable?())
|
1122
|
+
# get the ip address from either the from or the to. If we can get an ip address, and we already
|
1123
|
+
# have a transfer running for that ip, then this transfer is not eligable either
|
1124
|
+
ipaddress = extract_ip(transfer.from, transfer.to)
|
1125
|
+
if(ipaddress != nil and runningipaddresses.has_key?(ipaddress))
|
1126
|
+
next
|
1127
|
+
end
|
1128
|
+
|
1129
|
+
# get the group id from the transfer and compare it to a list of already involved
|
1130
|
+
# groups - never run more than one transfer in the same group
|
1131
|
+
if(transfer.group != nil and runninggroups.has_key?(transfer.group))
|
1132
|
+
next
|
1133
|
+
end
|
1134
|
+
|
1135
|
+
$logger.debug("priority of \"#{transfer.description}\" is #{transfer.priority}")
|
1136
|
+
|
1137
|
+
# add it to the eligabletransfers array if the priority is greater than 100 (meaning
|
1138
|
+
# it is at it's ideal running time). NOTE - it is important to get a static priority
|
1139
|
+
# instead of calling transfer.priority each time because the priority can change over the
|
1140
|
+
# course of the checks... leading to be juju when we make a slot for it and then try to add
|
1141
|
+
# to a different slot. ;)
|
1142
|
+
priority = transfer.priority
|
1143
|
+
if(priority >= 100)
|
1144
|
+
if(eligabletransfers[priority] == nil)
|
1145
|
+
eligabletransfers[priority] = Array.new()
|
1146
|
+
end
|
1147
|
+
eligabletransfers[priority] << transfer
|
1148
|
+
end
|
1149
|
+
end
|
1150
|
+
}
|
1151
|
+
rescue Exception
|
1152
|
+
$logger.fatal("Error during check of eligable transfers - #{$!} - Exiting")
|
1153
|
+
$@.each() { |line|
|
1154
|
+
$logger.fatal(line)
|
1155
|
+
}
|
1156
|
+
exit(1)
|
1157
|
+
end
|
1158
|
+
|
1159
|
+
|
1160
|
+
# choose a transfer to actually run
|
1161
|
+
begin
|
1162
|
+
# check the maximum number of transfers and whether we still have any to run
|
1163
|
+
if(numberrunning < configuration.general['maxtransfers'] and eligabletransfers.length > 0)
|
1164
|
+
priorities = eligabletransfers.keys()
|
1165
|
+
priorities.sort() { |x,y|
|
1166
|
+
y <=> x
|
1167
|
+
}
|
1168
|
+
|
1169
|
+
priorities.each() { |priority|
|
1170
|
+
if(foundtransfer == true)
|
1171
|
+
break
|
1172
|
+
end
|
1173
|
+
|
1174
|
+
idealtransfers = eligabletransfers[priority]
|
1175
|
+
|
1176
|
+
while(idealtransfers.length > 0)
|
1177
|
+
idealtransferindex = rand(idealtransfers.length)
|
1178
|
+
idealtransfer = idealtransfers[idealtransferindex]
|
1179
|
+
idealtransfers.delete_at(idealtransferindex)
|
1180
|
+
|
1181
|
+
# check the per-transfer cutoffprogram
|
1182
|
+
if(idealtransfer.cutoffprogram != '' and !system(idealtransfer.cutoffprogram))
|
1183
|
+
foundtransfer = false
|
1184
|
+
next
|
1185
|
+
end
|
1186
|
+
|
1187
|
+
foundtransfer = true
|
1188
|
+
idealtransfer.run()
|
1189
|
+
break;
|
1190
|
+
end
|
1191
|
+
}
|
1192
|
+
end
|
1193
|
+
|
1194
|
+
if(foundtransfer == false)
|
1195
|
+
$logger.debug("no transfer currently ready to run")
|
1196
|
+
end
|
1197
|
+
rescue Exception
|
1198
|
+
$logger.fatal("Error choosing ideal transfer - #{$!} - Exiting")
|
1199
|
+
$@.each() { |line|
|
1200
|
+
$logger.fatal(line)
|
1201
|
+
}
|
1202
|
+
exit(1)
|
1203
|
+
end
|
1204
|
+
end
|
1205
|
+
end
|
1206
|
+
end
|
1207
|
+
|
1208
|
+
|
1209
|
+
# run the program
|
1210
|
+
RsyncManager.main()
|
metadata
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.8.10
|
3
|
+
specification_version: 1
|
4
|
+
name: rsyncmanager
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: "1.1"
|
7
|
+
date: 2005-06-19
|
8
|
+
summary: RsyncManager is a daemon for controlling and monitoring rsync transfers
|
9
|
+
require_paths:
|
10
|
+
- lib
|
11
|
+
email: david@grayskies.net
|
12
|
+
homepage: http://rsyncmanager.rubyforge.org
|
13
|
+
rubyforge_project: rsyncmanager
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: false
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
-
|
22
|
+
- ">"
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.0.0
|
25
|
+
version:
|
26
|
+
platform: ruby
|
27
|
+
authors:
|
28
|
+
- David Powers
|
29
|
+
files:
|
30
|
+
- bin/rsyncmanager.rb
|
31
|
+
test_files:
|
32
|
+
- tests/test_configuration.rb
|
33
|
+
rdoc_options: []
|
34
|
+
extra_rdoc_files: []
|
35
|
+
executables:
|
36
|
+
- rsyncmanager.rb
|
37
|
+
extensions: []
|
38
|
+
requirements:
|
39
|
+
- rsync
|
40
|
+
- scp
|
41
|
+
dependencies: []
|