rsyncmanager 1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1210 @@
1
+ #!/usr/local/bin/ruby
2
+ #
3
+ # Author:: David Powers
4
+ # Copyright:: September, 2004
5
+ # Version:: 1.1
6
+ #
7
+ #Overview:
8
+ #
9
+ #Rsyncmanager is a daemon designed to simplify the task of coordinating
10
+ #multiple file transfers to or from a machine. It was originally written
11
+ #to serve as the managing hub of a backup system that used rsync to feed
12
+ #in files from many other machines so that all backups could be run from
13
+ #a single place. That said, it can be used to coordinate any kind of regular
14
+ #file transfer, from a single hourly copy of an email store from a primary
15
+ #to a backup server, to multiple concurrent rsyncs to multiple machines on
16
+ #varying schedules.
17
+ #
18
+ #
19
+ #Features:
20
+ #
21
+ #* Coordinate multiple file transfers using rsync, scp or a combination
22
+ # of the two
23
+ #* Limit the number of concurrently running file transfers to prevent
24
+ # resource starvation
25
+ #* Prevent file transfers from starting based on arbitrary conditions using
26
+ # external scripts (time of day, machine load, memory usage, phase of the
27
+ # moon...)
28
+ #* Run commands before and after a file transfer to handle setup and cleanup
29
+ # conditions (e.g. dump a database to disk, transfer it to a backup, then
30
+ # delete the dump)
31
+ #* Intelligently limit concurrent file transfers to or from the same machine
32
+ # to prevent resource limitations from slowing or starving a transfer
33
+ #* Limit concurrent file transfers based on arbitrary grouping (e.g. only
34
+ # one transfer from internal to external servers over a slow line at any
35
+ # given point)
36
+ #* Coordinate transfers between multiple machines (grouping over multiple
37
+ # machines, avoiding having two machines push data to a third at the same
38
+ # time, etc.)
39
+ #* Provides a clear web-based administrative report of all currently
40
+ # managed transfers and their current status (running, behind schedule,
41
+ # etc)
42
+ #* Intelligently schedule file transfers based on average runtimes so that
43
+ # transfers happen within a defined schedule but do not run constantly if
44
+ # not needed
45
+
46
+ require 'getoptlong'
47
+ require 'rexml/document'
48
+ require 'logger'
49
+ require 'net/http'
50
+ require 'optparse'
51
+ require 'socket'
52
+ require 'timeout'
53
+ require 'thread'
54
+ require 'webrick'
55
+
56
+
57
+
58
+ # parent class for configuration exceptions
59
+ class ConfigurationError < Exception
60
+ end
61
+
62
+ # raised when field content does not match allowed values
63
+ class InvalidContentError < ConfigurationError
64
+ end
65
+
66
+ # raised when a required element is not found
67
+ class MissingElementError < ConfigurationError
68
+ end
69
+
70
+
71
+
72
+ # holds configuration information about a single transfer and can start and manage
73
+ # a thread to run the transfer
74
+ class Transfer
75
+ attr_reader :options, :group, :from, :to, :commandpath, :description, :laststarted, :lastfinished, :frequency, :ignorepartialerrors, :cutoffprogram, :runbefore, :runafter
76
+
77
+ def initialize()
78
+ @laststarted = nil
79
+ @lastfinished = nil
80
+ @runtimes = Array.new()
81
+
82
+ # set @thread to a new dead thread to avoid special handling of thread state query functions
83
+ @thread = Thread.new {}
84
+ end
85
+
86
+
87
+ # turns the internal data into a command
88
+ def command()
89
+ return("#{@commandpath} #{@options} '#{@from.gsub(/\'/, '\\\&')}' '#{@to.gsub(/\'/, '\\\&')}'")
90
+ end
91
+
92
+
93
+ # returns true if this transfer is eligable to be run
94
+ def eligable?()
95
+ # never eligable if it is running right now
96
+ if(running?())
97
+ return(false)
98
+ end
99
+
100
+ # if we have runtimes compare the average runtime with the time remaining
101
+ # until our ideal runtime. We need to be reasonably close to our ideal time
102
+ # to be eligable
103
+ if(@runtimes.length > 0)
104
+ @perfecttime = @lastfinished + @frequency - averageruntime()
105
+ if(Time.now() < @perfecttime - (averageruntime() / 2))
106
+ return(false)
107
+ end
108
+ end
109
+
110
+ return(true)
111
+ end
112
+
113
+
114
+ # returns the average runtime
115
+ def averageruntime()
116
+ totaltime = 0
117
+ @runtimes.each() { |runtime|
118
+ totaltime = totaltime + runtime
119
+ }
120
+
121
+ if(@runtimes.length == 0)
122
+ return(@frequency)
123
+ else
124
+ return(totaltime/@runtimes.length)
125
+ end
126
+ end
127
+
128
+
129
+ # returns the current calculated priority of this transfer. The priority approaches 100 as the
130
+ # transfer nears the perfect time for the next start.
131
+ def priority()
132
+ $logger.debug("checking priority of \"#{@description}\"")
133
+ # always return 0 if the transfer is currently running
134
+ if(running?())
135
+ $logger.debug("transfer is running - returning 0")
136
+ return(0)
137
+ end
138
+
139
+ # if it has never been run - set priority to 250
140
+ if(@lastfinished == nil)
141
+ $logger.debug("transfer has never been run - returning 1000")
142
+ return(250)
143
+ end
144
+
145
+ @step = @frequency.to_f() / 100.0
146
+
147
+ # as we approach the perfect time (@lastfinished + @frequency) the priority should approach 100
148
+ priority = (Time.now - @lastfinished) / @step
149
+ $logger.debug("transfer has a real priority 0f #{priority}")
150
+ return(priority)
151
+ end
152
+
153
+
154
+ # runs this transfer in a thread
155
+ def run()
156
+ @thread = Thread.new() {
157
+ begin
158
+ # mark the start time
159
+ @laststarted = Time.now()
160
+
161
+ $logger.info("running \"#{@description}\"")
162
+
163
+ # run the runbefore command if it exists
164
+ if(@runbefore != '')
165
+ $logger.debug("running command #{@runbefore}")
166
+ runbeforestatus = system(@runbefore)
167
+ if(runbeforestatus == false)
168
+ $logger.error("error during run of #{@runbefore} before #{@description} - error code #{$?}")
169
+ raise(Exception)
170
+ end
171
+ end
172
+
173
+
174
+ # actually run the command and report on success/failure
175
+ $logger.debug("running command #{command()}")
176
+ ransuccessfully = system(command())
177
+
178
+ # update variable and internal state
179
+ if(ransuccessfully == false and (@ignorepartialerrors == false or $?.to_i() != 5888))
180
+ $logger.error("error during run of #{command()} - error code #{$?}")
181
+ else
182
+ # run the runbefore command if it exists
183
+ if(@runafter != '')
184
+ $logger.debug("running command #{@runbefore}")
185
+ runafterstatus = system(@runafter)
186
+ if(runafterstatus == false)
187
+ $logger.error("error during run of #{@runafter} after #{@description} - error code #{$?}")
188
+ raise(Exception)
189
+ end
190
+ end
191
+
192
+ $logger.info("\"#{@description}\" completed")
193
+
194
+ # mark the time of completion
195
+ @lastfinished = Time.now()
196
+
197
+ # set last runtime here to handle clock shifts
198
+ if(@lastfinished > @laststarted)
199
+ @runtimes << @lastfinished - @laststarted
200
+
201
+ # lop off the oldest time if we have more than 10
202
+ if(@runtimes.length > 10)
203
+ @runtimes = @runtimes[1..-1]
204
+ end
205
+ end
206
+ end
207
+ rescue Exception
208
+ $logger.error("error running \"#{description}\" - #{$!}")
209
+ end
210
+ }
211
+ end
212
+
213
+
214
+ def running?()
215
+ return(@thread.alive?())
216
+ end
217
+
218
+
219
+ def ignorepartialerrors=(newignorepartialerrors)
220
+ @ignorepartialerrors = newignorepartialerrors
221
+ $logger.debug("ignorepartialerrors set to \"#{@ignorepartialerrors}\"")
222
+ end
223
+
224
+
225
+ def commandpath=(newcommandpath)
226
+ @commandpath = newcommandpath
227
+ $logger.debug("commandpath set to \"#{@commandpath}\"")
228
+ end
229
+
230
+
231
+ def description=(newdescription)
232
+ @description = newdescription
233
+ $logger.debug("description set to \"#{@description}\"")
234
+ end
235
+
236
+
237
+ def options=(newoptions)
238
+ @options = newoptions
239
+ $logger.debug("options set to \"#{@options}\"")
240
+ end
241
+
242
+
243
+ def from=(newfrom)
244
+ @from = newfrom
245
+ $logger.debug("from set to \"#{@from}\"")
246
+ end
247
+
248
+
249
+ def to=(newto)
250
+ @to = newto
251
+ $logger.debug("to set to \"#{@to}\"")
252
+ end
253
+
254
+
255
+ def cutoffprogram=(newcutoffprogram)
256
+ @cutoffprogram = newcutoffprogram
257
+ $logger.debug("cutoffprogram set to \"#{@cutoffprogram}\"")
258
+ end
259
+
260
+
261
+ def runbefore=(newrunbefore)
262
+ @runbefore = newrunbefore
263
+ $logger.debug("runbefore set to \"#{@runbefore}\"")
264
+ end
265
+
266
+
267
+ def runafter=(newrunafter)
268
+ @runafter = newrunafter
269
+ $logger.debug("runafter set to \"#{@runafter}\"")
270
+ end
271
+
272
+
273
+ def group=(newgroup)
274
+ @group = newgroup
275
+ $logger.debug("group set to \"#{@group}\"")
276
+ end
277
+
278
+
279
+ # sets the frequency with a human readable string
280
+ def frequency=(newfrequency)
281
+ # if it is purely numeric assume minutes, otherwise get a suffix and compute from that
282
+ if(newfrequency =~ /^[0-9]+$/)
283
+ @frequency = newfrequency.to_i() * 60
284
+ elsif(newfrequency =~ /m/)
285
+ @frequency = newfrequency[/^[0-9]+/].to_i() * 60
286
+ elsif(newfrequency =~ /h/)
287
+ @frequency = newfrequency[/^[0-9]+/].to_i() * 60 * 60
288
+ elsif(newfrequency =~ /d/)
289
+ @frequency = newfrequency[/^[0-9]+/].to_i() * 24 * 60 * 60
290
+ else
291
+ raise(ArgumentError, "frequency could not be set to \"#{newfrequency}\"")
292
+ end
293
+ end
294
+ end
295
+
296
+
297
+
298
+ # Reads and holds the configuration for rsyncmanager
299
+ class Configuration
300
+ attr_reader :general, :transfers
301
+
302
+ # takes the filename of a configuration to read in
303
+ def initialize(filename)
304
+ # open the configuration file and read it in as an XML document
305
+ xmldoc = REXML::Document.new(File.open(filename, File::RDONLY))
306
+
307
+ parse_general(xmldoc.root.elements['general'])
308
+ parse_transfers(xmldoc.root.elements['transfers'])
309
+ end
310
+
311
+
312
+ # parses the general section of the configuration file
313
+ def parse_general(generalxml)
314
+ # setup a hash to hold the options
315
+ @general = Hash.new()
316
+
317
+ # get the maximum number of transfers to run at once
318
+ @general['maxtransfers'] = get_setting(generalxml.elements['maxtransfers'], 3, :to_i) { |setting|
319
+ if(setting <= 0)
320
+ raise(InvalidContentError, "maxtransfers must be set greater than 0")
321
+ end
322
+ }
323
+
324
+ # get the path of the program that controls when to cutoff running any more transfers
325
+ @general['cutoffprogram'] = get_setting(generalxml.elements['cutoffprogram'], "")
326
+
327
+ # set the command paths (automagically if necessary)
328
+ @general['rsyncpath'] = get_setting(generalxml.elements['rsyncpath'], get_command_path('rsync')) { |setting|
329
+ if(setting == "")
330
+ raise(ConfigurationError, "rsyncpath is not defined and the rsync binary could not be found on your system")
331
+ end
332
+ }
333
+
334
+ @general['scppath'] = get_setting(generalxml.elements['scppath'], get_command_path('scp')) { |setting|
335
+ if(setting == "")
336
+ raise(ConfigurationError, "scppath is not defined and the scp binary could not be found on your system")
337
+ end
338
+ }
339
+
340
+ # set the coordination parameters
341
+ @general['coordinate'] = Array.new()
342
+
343
+ coordinationxml = generalxml.elements['coordinate']
344
+ if(coordinationxml)
345
+ coordinationxml.each_element('url') { |url|
346
+ @general['coordinate'] << url.text()
347
+ }
348
+ end
349
+
350
+ # set the http server defaults
351
+ httpserverxml = generalxml.elements['httpserver']
352
+ if(httpserverxml)
353
+ @general['httpserver'] = Hash.new()
354
+
355
+ @general['httpserver']['port'] = get_setting(httpserverxml.elements['port'], 7962, :to_i)
356
+ @general['httpserver']['listen'] = get_setting(httpserverxml.elements['listen'], '0.0.0.0')
357
+ end
358
+
359
+
360
+ # get the defaults for transfers
361
+ parse_transferdefaults(generalxml.elements['transferdefaults'])
362
+ end
363
+
364
+
365
+ # parses the transferdefaults subsection of the general section of the configuration file
366
+ def parse_transferdefaults(transferdefaultsxml)
367
+ @transferdefaults = Hash.new()
368
+
369
+ @transferdefaults['type'] = get_setting(transferdefaultsxml.elements['type'], 'rsync') { |setting|
370
+ if(setting != 'rsync' and setting != 'scp')
371
+ raise(InvalidContentError, "transferdefaults/type must be either 'rsync' or 'scp' but is currently set to \"#{setting}\"")
372
+ end
373
+ }
374
+
375
+ if(transferdefaultsxml.elements['group'])
376
+ @transferdefaults['group'] = get_setting(transferdefaultsxml.elements['group'], "")
377
+ else
378
+ @transferdefaults['group'] = nil
379
+ end
380
+
381
+ @transferdefaults['options'] = get_setting(transferdefaultsxml.elements['options'], '-a -r -q')
382
+
383
+ @transferdefaults['ignorepartialerrors'] = get_setting(transferdefaultsxml.elements['ignorepartialerrors'], 'false') { |setting|
384
+ if(setting.downcase() == 'true')
385
+ setting = true
386
+ elsif(setting.downcase() == 'false')
387
+ setting = false
388
+ else
389
+ raise(InvalidContentError, "transferdefaults/ignorepartialerrors must be either 'true' or 'false' but is currently set to \"#{setting}\"")
390
+ end
391
+ }
392
+
393
+ @transferdefaults['frequency'] = get_setting(transferdefaultsxml.elements['frequency'], '1h') { |setting|
394
+ if(setting !~ /^[1-9][0-9]*(m|h|d|w)?$/)
395
+ raise(InvalidContentError, "transferdefaults/frequency (\"#{setting}\") cannot be parsed")
396
+ end
397
+ }
398
+
399
+ # get the path of the program that controls when to cutoff running any more transfers
400
+ @transferdefaults['cutoffprogram'] = get_setting(transferdefaultsxml.elements['cutoffprogram'], "")
401
+
402
+ # set default programs to run before and after a transfer - defaults to nil
403
+ @transferdefaults['runbefore'] = get_setting(transferdefaultsxml.elements['runbefore'], '')
404
+ @transferdefaults['runafter'] = get_setting(transferdefaultsxml.elements['runafter'], '')
405
+ end
406
+
407
+
408
+ # parses the transfers section of the configuration file
409
+ def parse_transfers(transfersxml)
410
+ @transfers = Array.new()
411
+
412
+ transfersxml.each_element('transfer') { |transferxml|
413
+ transfer = Transfer.new()
414
+
415
+ # set the description (optional)
416
+ transfer.description = get_setting(transferxml.elements['description'], 'No description given')
417
+
418
+ # set the type
419
+ type = get_setting(transferxml.elements['type'], @transferdefaults['type']) { |setting|
420
+ if(setting != 'rsync' and setting != 'scp')
421
+ raise(InvalidContentError, "type must be either 'rsync' or 'scp' but is currently set to \"#{setting}\"")
422
+ end
423
+ }
424
+
425
+ # set the command path
426
+ if(type == 'rsync')
427
+ transfer.commandpath = @general['rsyncpath']
428
+ elsif(type == 'scp')
429
+ transfer.commandpath = @general['scppath']
430
+ end
431
+
432
+ # set the group
433
+ if(transferxml.elements['group'])
434
+ transfer.group = get_setting(transferxml.elements['group'], '')
435
+ else
436
+ transfer.group = @transferdefaults['group']
437
+ end
438
+
439
+ # set the frequency
440
+ transfer.frequency = get_setting(transferxml.elements['frequency'], @transferdefaults['frequency']) { |setting|
441
+ if(setting !~ /^[1-9][0-9]*(m|h|d|w)?$/)
442
+ raise(InvalidContentError, "transferdefaults/frequency (\"#{setting}\") cannot be parsed")
443
+ end
444
+ }
445
+
446
+ # set the options
447
+ transfer.options = get_setting(transferxml.elements['options'], @transferdefaults['options'])
448
+
449
+ # set the from and to
450
+ transfer.from = get_setting(transferxml.elements['from'], "") { |setting|
451
+ if(setting == "")
452
+ raise(MissingElementError, "\"from\" must be set in each transfer section")
453
+ end
454
+ }
455
+
456
+ transfer.to = get_setting(transferxml.elements['to'], "") { |setting|
457
+ if(setting == nil)
458
+ raise(MissingElementError, "\"to\" must be set in each transfer section")
459
+ end
460
+ }
461
+
462
+ # set whether or not we are ignoring partial transfer errors
463
+ transfer.ignorepartialerrors = get_setting(transferxml.elements['ignorepartialerrors'], @transferdefaults['ignorepartialerrors']) { |setting|
464
+ if(setting.downcase() == 'true')
465
+ setting = true
466
+ elsif(setting.downcase() == 'false')
467
+ setting = false
468
+ elsif(setting != true and setting != false)
469
+ raise(InvalidContentError, "transferdefaults/ignorepartialerrors must be either 'true' or 'false' but is currently set to \"#{setting}\"")
470
+ end
471
+ }
472
+
473
+ # set a possible per-transfer cuttoffprogram
474
+ transfer.cutoffprogram = get_setting(transferxml.elements['cutoffprogram'], @transferdefaults['cutoffprogram'])
475
+
476
+ # set programs to run before and after a transfer - this can handle setup and cleanup that rsync and scp cannot
477
+ transfer.runbefore = get_setting(transferxml.elements['runbefore'], @transferdefaults['runbefore'])
478
+ transfer.runafter = get_setting(transferxml.elements['runafter'], @transferdefaults['runafter'])
479
+
480
+ @transfers << transfer
481
+ $logger.debug("Added transfer to list")
482
+ }
483
+ end
484
+
485
+
486
+ # finds the full path of a given command - returns false if the path cannot be found
487
+ def get_command_path(command)
488
+ paths = Array.[]('/sbin', '/bin', '/usr/sbin', '/usr/bin', '/usr/local/sbin', '/usr/local/bin')
489
+
490
+ paths.each() { |path|
491
+ if(test(?x, "#{path}/#{command}"))
492
+ return("#{path}/#{command}")
493
+ end
494
+ }
495
+
496
+ return(false)
497
+ end
498
+
499
+
500
+ # returns either the setting from the xml as text or default. conversionmethod is applied to the
501
+ # setting to ensure the final return value is of a known type. If a block is provided setting will
502
+ # be passed to the block (after conversion) for constraint checking or modification.
503
+ def get_setting(xml, default, conversionmethod = :to_s)
504
+ setting = nil
505
+
506
+ if(xml == nil)
507
+ setting = default
508
+ else
509
+ setting = xml.text()
510
+ end
511
+
512
+ # convert based on the conversion method
513
+ setting = setting.method(conversionmethod).call()
514
+
515
+ if(block_given?())
516
+ yield(setting)
517
+ end
518
+
519
+ return(setting)
520
+ end
521
+ end
522
+
523
+
524
+
525
+ # Serves a simple xml description of the transfers via WEBrick - mostly
526
+ # for collaboration purposes between instances of rsyncmanager running on
527
+ # separate machines
528
+ class XMLInformationServlet < WEBrick::HTTPServlet::AbstractServlet
529
+ # handles GET requests using the helper methods of the class
530
+ def do_GET(req, res)
531
+ begin
532
+ @configuration = @options[0]
533
+ if(!@configuration.kind_of?(Configuration))
534
+ raise(ArgumentError, "configuration must be a Configuration object")
535
+ end
536
+
537
+ timeout(30) {
538
+ xmldoc = REXML::Document.new("<transfers></transfers>")
539
+ xmldoc << REXML::XMLDecl.new()
540
+
541
+ # now the information
542
+ @configuration.transfers.each() { |transfer|
543
+ xmldoc.root.add_element(xml_transfer_description(transfer))
544
+ }
545
+
546
+ res.body = ""
547
+ xmldoc.write(res.body, 0)
548
+ }
549
+ rescue Exception
550
+ $logger.error("Error serving xml information page - #{$!}")
551
+ end
552
+ end
553
+
554
+
555
+ protected
556
+
557
+
558
+ # helper function to build a transfer information table
559
+ def xml_transfer_description(transfer)
560
+ now = Time.now()
561
+ transferelement = REXML::Element.new("transfer")
562
+
563
+ if(transfer.group != nil)
564
+ group = REXML::Element.new("group")
565
+ group.text = transfer.group.to_s
566
+ transferelement.add_element(group)
567
+ end
568
+
569
+ running = REXML::Element.new("running")
570
+ if(transfer.running?)
571
+ running.text = "true"
572
+ else
573
+ running.text = "false"
574
+ end
575
+ transferelement.add_element(running)
576
+
577
+ description = REXML::Element.new("description")
578
+ description.text = transfer.description
579
+ transferelement.add_element(description)
580
+
581
+ runbefore = REXML::Element.new("runbefore")
582
+ runbefore.text = transfer.runbefore
583
+ transferelement.add_element(runbefore)
584
+
585
+ runafter = REXML::Element.new("runafter")
586
+ runafter.text = transfer.runafter
587
+ transferelement.add_element(runafter)
588
+
589
+ from = REXML::Element.new("from")
590
+ from.text = transfer.from
591
+ transferelement.add_element(from)
592
+
593
+ to = REXML::Element.new("to")
594
+ to.text = transfer.to
595
+ transferelement.add_element(to)
596
+
597
+ command = REXML::Element.new("command")
598
+ command.text = transfer.command
599
+ transferelement.add_element(command)
600
+
601
+ frequency = REXML::Element.new("frequency")
602
+ frequency.text = transfer.frequency.to_s()
603
+ transferelement.add_element(frequency)
604
+
605
+ lastfinished = REXML::Element.new("lastfinished")
606
+ if(transfer.lastfinished)
607
+ lastfinished.text = transfer.lastfinished.to_s()
608
+ end
609
+
610
+ averageruntime = REXML::Element.new("averageruntime")
611
+ averageruntime.text = transfer.averageruntime.to_s()
612
+ transferelement.add_element(averageruntime)
613
+
614
+ priority = REXML::Element.new("priority")
615
+ priority.text = transfer.priority.to_s()
616
+ transferelement.add_element(priority)
617
+
618
+ return(transferelement)
619
+ end
620
+
621
+ end
622
+
623
+
624
+
625
+ # Serves a simple informational page via WEBrick
626
+ class HTTPInformationServlet < WEBrick::HTTPServlet::AbstractServlet
627
+
628
+ # helper function to produce a formatted table row
629
+ def http_table_row(label, info)
630
+ row = ""
631
+ row << "\t<tr>\n"
632
+ row << "\t\t<td class=\"label\">\n\t\t"
633
+ row << "#{label}\n"
634
+ row << "\t\t</td>\n"
635
+ row << "\t\t<td class=\"info\">\n"
636
+ row << "#{info}\n"
637
+ row << "\t\t</td>\n"
638
+ row << "\t</tr>\n"
639
+
640
+ return(row)
641
+ end
642
+
643
+
644
+ # helper function to build a key
645
+ def http_key()
646
+ key = <<KEY
647
+ <div class="keybox">
648
+ <div class="keytitle">
649
+ Transfer Status
650
+ </div>
651
+ <table>
652
+ <tr>
653
+ <td class="keylabel">
654
+ On-Time:
655
+ </td>
656
+ <td>
657
+ <div class="demobox" style="background-color: #e1ffe1">
658
+ </div>
659
+ </td>
660
+ </tr>
661
+ <tr>
662
+ <td class="keylabel">
663
+ Slightly Behind:
664
+ </td>
665
+ <td>
666
+ <div class="demobox" style="background-color: #ffffe1">
667
+ </div>
668
+ </td>
669
+ </tr>
670
+ <tr>
671
+ <td class="keylabel">
672
+ Overdue:
673
+ </td>
674
+ <td>
675
+ <div class="demobox" style="background-color: #ffe1e1">
676
+ </div>
677
+ </td>
678
+ </tr>
679
+ <tr>
680
+ <td class="keylabel">
681
+ Never Completed:
682
+ </td>
683
+ <td>
684
+ <div class="demobox" style="background-color: rgb(95%, 95%, 95%)">
685
+ </div>
686
+ </td>
687
+ </tr>
688
+ <tr>
689
+ <td class="keylabel">
690
+ Currently Running:
691
+ </td>
692
+ <td>
693
+ <div class="demobox" style="border: 3px solid #0042ff">
694
+ </div>
695
+ </td>
696
+ </tr>
697
+ </table>
698
+ </div>
699
+ KEY
700
+ return(key)
701
+ end
702
+
703
+
704
+ # helper function to build a stylesheet
705
+ def http_stylesheet()
706
+ stylesheet = <<STYLESHEET
707
+ <style type=\"text/css\">
708
+ .transfersummary {
709
+ margin-left: auto;
710
+ margin-right: auto;
711
+ margin-bottom: 10px;
712
+ width: 90%;
713
+ border: 1px solid black;
714
+ background-color: rgb(95%, 95%, 95%);
715
+ }
716
+ .label {
717
+ width: 50px;
718
+ white-space: nowrap;
719
+ font-size: 12px;
720
+ font-weight: bold;
721
+ }
722
+ .info {
723
+ font-size: 10px;
724
+ font-weight: normal;
725
+ }
726
+
727
+ .keybox {
728
+ border: 1px solid black;
729
+ background-color: #ffffff;
730
+ z-index: 5;
731
+ position: absolute;
732
+ top: 3px;
733
+ right: 3px;
734
+ }
735
+
736
+ .keylabel {
737
+ font-size: 8px;
738
+ }
739
+
740
+ .keytitle {
741
+ font-size: 9px;
742
+ font-weight: bold;
743
+ color: white;
744
+ text-align: center;
745
+ background-color: rgb(20%,20%,20%);
746
+ padding: 2px;
747
+ }
748
+
749
+ .demobox {
750
+ width: 20px;
751
+ height: 20px;
752
+ border: 1px solid black;
753
+ margin-left: auto;
754
+ margin-right: auto;
755
+ }
756
+
757
+ </style>\n
758
+ STYLESHEET
759
+ return(stylesheet)
760
+ end
761
+
762
+
763
+ # helper function to build a transfer information table
764
+ def http_transfer_table(transfer)
765
+ now = Time.now()
766
+
767
+ table = "<table class=\"transfersummary\" style=\""
768
+
769
+ # change the color of the table based on how close to on-schedule we are
770
+ if(transfer.lastfinished != nil)
771
+ green = transfer.lastfinished + transfer.frequency + transfer.frequency/10
772
+ yellow = transfer.lastfinished + transfer.frequency*2
773
+ if(now < green)
774
+ table << "background-color: #e1ffe1; "
775
+ elsif(now < yellow)
776
+ table << "background-color: #ffffe1; "
777
+ else
778
+ table << "background-color: #ffe1e1; "
779
+ end
780
+ end
781
+
782
+ # if it is running, mark it
783
+ if(transfer.running?())
784
+ table << "border: 3px solid #0042ff;"
785
+ end
786
+ table << "\">\n"
787
+
788
+ if(transfer.group != nil)
789
+ table << http_table_row("Group:", transfer.group)
790
+ end
791
+
792
+ table << http_table_row("Description:", transfer.description)
793
+ if(transfer.runbefore != '')
794
+ table << http_table_row("Run Before:", transfer.runbefore)
795
+ end
796
+ table << http_table_row("Command:", transfer.command)
797
+ if(transfer.runafter != '')
798
+ table << http_table_row("Run After:", transfer.runafter)
799
+ end
800
+ table << http_table_row("Ideal Frequency:", "#{(transfer.frequency/60).round()} minutes")
801
+ table << http_table_row("Running:", transfer.running?())
802
+ if(transfer.lastfinished)
803
+ table << http_table_row("Last Finished:", "#{transfer.lastfinished} (#{((now - transfer.lastfinished)/60).round()} minutes ago)")
804
+ else
805
+ table << http_table_row("Last Finished:", "")
806
+ end
807
+ table << http_table_row("Average Run Time:", "#{(transfer.averageruntime/60).round()} minutes")
808
+ table << http_table_row("Current Priority:", transfer.priority.to_f.round())
809
+ table << "</table>\n"
810
+
811
+ return(table)
812
+ end
813
+
814
+
815
+ # handles GET requests using the helper methods of the class
816
+ def do_GET(req, res)
817
+ begin
818
+ @configuration = @options[0]
819
+ if(!@configuration.kind_of?(Configuration))
820
+ raise(ArgumentError, "configuration must be a Configuration object")
821
+ end
822
+
823
+ timeout(30) {
824
+ res.body = ""
825
+ res.body << "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
826
+ res.body << "<html>\n"
827
+ res.body << "<head>\n"
828
+ res.body << "\t<meta http-equiv=\"content-type\" content=\"text/html; charset=ISO-8859-1\"/>\n"
829
+ res.body << "\t<title>Summary</title>\n"
830
+
831
+ # style sheet
832
+ res.body << http_stylesheet()
833
+
834
+ # reload the page automatically every 5 minutes
835
+ res.body << "\t<script type=\"text/JavaScript\">\n"
836
+ res.body << "\t\tfunction reloadpage()\n"
837
+ res.body << "\t\t{\n"
838
+ res.body << "\t\t\tdocument.location.href = document.location.href;\n"
839
+ res.body << "\t\t}\n\n"
840
+ res.body << "\t\tsetInterval('reloadpage()', 600000);\n"
841
+ res.body << "\t</script>\n"
842
+
843
+ res.body << "</head>\n"
844
+ res.body << "<body>\n"
845
+
846
+ # build a nice little key
847
+ res.body << http_key()
848
+
849
+ # now the information
850
+ @configuration.transfers.each() { |transfer|
851
+ # build the table
852
+ table = http_transfer_table(transfer)
853
+ res.body << table
854
+ }
855
+
856
+ res.body << "</body>\n"
857
+ res.body << "</html>\n"
858
+ }
859
+ rescue Exception
860
+ $logger.error("Error serving http information page - #{$!}")
861
+ end
862
+ end
863
+ end
864
+
865
+
866
+ # Collects the non-class methods for rsyncmanager
867
+ module RsyncManager
868
+
869
+ # Sets the $options variable. Using defaults where none
870
+ # are provided and entries from the command line where possible
871
+ def RsyncManager.set_options()
872
+ $options = Hash.new()
873
+ $options['config'] = 'rsyncmanager.xml'
874
+ $options['log'] = '/var/log/rsyncmanager.log'
875
+ $options['verbosity'] = Logger::INFO
876
+ $options['pidfile'] = '/var/run/rsyncmanager.pid'
877
+ $options['daemon'] = true
878
+
879
+
880
+ opts = OptionParser.new() { |opts|
881
+ opts.on_tail("-v", "--verbosity VERBOSITY", Integer, "(0-4) 0 being fatal errors only, default 3") { |verbosity|
882
+ if(verbosity == 0)
883
+ $options['verbosity'] = Logger::FATAL
884
+ elsif(verbosity == 1)
885
+ $options['verbosity'] = Logger::ERROR
886
+ elsif(verbosity == 2)
887
+ $options['verbosity'] = Logger::WARN
888
+ elsif(verbosity == 3)
889
+ $options['verbosity'] = Logger::INFO
890
+ elsif(verbosity == 4)
891
+ $options['verbosity'] = Logger::DEBUG
892
+ else
893
+ $stderr.print("verbosity must be set between 0 and 4\n")
894
+ exit(1)
895
+ end
896
+ }
897
+
898
+ opts.on_tail("-h", "--help", "Print this message") {
899
+ print(opts)
900
+ exit()
901
+ }
902
+
903
+ opts.on("-c", "--config CONFIGFILE", "Location of the configuration file") { |filename|
904
+ $options['config'] = filename
905
+ }
906
+
907
+ opts.on("-l", "--log LOGFILE", "Location of the log file") { |filename|
908
+ $options['log'] = filename
909
+ }
910
+
911
+ opts.on("-p", "--pidfile PIDFILE", "Location of the pid file") { |filename|
912
+ $options['pidfile'] = filename
913
+ }
914
+
915
+ opts.on("-d", "--nodaemon", "Do not fork into the background") {
916
+ $options['daemon'] = false
917
+ }
918
+ }
919
+
920
+ opts.parse(ARGV)
921
+ end
922
+
923
+
924
+ # forks the program into the background and detaches it from the terminal
925
+ def RsyncManager.daemonize()
926
+ # fork into the background
927
+ # Parent exits, child continues.
928
+ if(fork())
929
+ exit()
930
+ end
931
+
932
+ # Become session leader.
933
+ Process.setsid()
934
+
935
+ # Zap session leader.
936
+ if(fork())
937
+ exit()
938
+ end
939
+
940
+ # Release old working directory.
941
+ Dir.chdir "/"
942
+
943
+ # Ensure sensible umask. Adjust as needed.
944
+ File.umask 0000
945
+
946
+ # reopen the file descriptors to nothing
947
+ STDIN.reopen '/dev/null'
948
+ STDOUT.reopen '/dev/null', 'a'
949
+ STDERR.reopen STDOUT
950
+ end
951
+
952
+
953
+ # writes the PID to a file
954
+ def RsyncManager.writepid(pidfile)
955
+ File.open(pidfile, File::WRONLY|File::CREAT|File::TRUNC, 0644) { |fp|
956
+ fp.write("#{Process.pid}\n")
957
+ }
958
+ end
959
+
960
+
961
+ # looks in each string in *strings and returns the first ip-address or hostname that
962
+ # it can find (assumes the strings will be of the form used in rsync or scp to designate
963
+ # a remote host)
964
+ def RsyncManager.extract_ip(*strings)
965
+ strings.each() { |str|
966
+ ipaddress = str[/^([^\/:@]*@)?([^:]+):/, 2]
967
+ if(ipaddress != nil)
968
+ return(ipaddress)
969
+ end
970
+ }
971
+
972
+ return(nil)
973
+ end
974
+
975
+
976
+ # pulls down xml from the given url and adds the information to runningipaddresses and runninggroups
977
+ # so that it can be used in making decisions about which transfers to run on this machine
978
+ def RsyncManager.get_coordination_info(url, runningipaddresses, runninggroups)
979
+ begin
980
+ # we can't afford to wait around forever for an answer - we'll take what we can get
981
+ # and make a best guess if we don't get anything
982
+ timeout(10) {
983
+ # we only handle HTTP for now
984
+ response = Net::HTTP.get(URI.parse(url))
985
+ xmldoc = REXML::Document.new(response)
986
+ xmldoc.root.each_element('transfer') { |transfer|
987
+ if(transfer.elements['running'].text == 'true')
988
+ group = transfer.elements['group']
989
+ from = transfer.elements['from'].text
990
+ to = transfer.elements['to'].text
991
+ if(group != nil)
992
+ runninggroups[group] = true
993
+ end
994
+ runningipaddresses[extract_ip(from, to)] = 1
995
+ end
996
+ }
997
+ }
998
+ rescue Exception
999
+ $logger.warn("Unable to get coordination information from #{url} - #{$!}")
1000
+ end
1001
+ end
1002
+
1003
+
1004
+ # The main logic loop of the program
1005
+ def RsyncManager.main()
1006
+ # blow up the whole thing for unhandled exceptions
1007
+ Thread.abort_on_exception = true
1008
+
1009
+ # set $options
1010
+ set_options()
1011
+
1012
+ # create a logger
1013
+ $logger = Logger.new($options['log'], 10, 1024000)
1014
+ $logger.level = $options['verbosity']
1015
+ $logger.info("rsyncmanager started")
1016
+
1017
+ # read in the configuration
1018
+ configuration = Configuration.new($options['config'])
1019
+
1020
+ # go into the background
1021
+ if($options['daemon'] == true)
1022
+ daemonize()
1023
+ end
1024
+
1025
+ # write the pid to a file
1026
+ writepid($options['pidfile'])
1027
+
1028
+ # trap term so we can exit gracefully
1029
+ trap("TERM") {
1030
+ $logger.info("recieved TERM signal")
1031
+ begin
1032
+ # kill the child processes
1033
+ File.unlink($options['pidfile'])
1034
+ exit(0)
1035
+ rescue Exception
1036
+ # do nothing except note that we failed
1037
+ $stderr.print "Unable to delete pid file \"#{$options['pidfile']}\" - #{$!}\n"
1038
+ exit(1)
1039
+ end
1040
+ }
1041
+
1042
+ # start an informational webserver if it has been requested
1043
+ webserver = nil
1044
+ if(configuration.general['httpserver'] != nil)
1045
+ $logger.info("Starting HTTP server on #{configuration.general['httpserver']['listen']}:#{configuration.general['httpserver']['port']}")
1046
+
1047
+ config = Hash.new()
1048
+ config[:Port] = configuration.general['httpserver']['port']
1049
+ config[:BindAddress] = configuration.general['httpserver']['listen']
1050
+ webserver = WEBrick::HTTPServer.new(config)
1051
+ webserver.mount('/info', HTTPInformationServlet, configuration)
1052
+ webserver.mount('/xml', XMLInformationServlet, configuration)
1053
+
1054
+ # start the server in its own thread so it doesn't stop us from doing real work
1055
+ Thread.new() {
1056
+ begin
1057
+ webserver.start()
1058
+ rescue Exception
1059
+ $logger.error("Stopping web server - #{$!}")
1060
+ end
1061
+ }
1062
+ end
1063
+
1064
+ # run in a loop, scheduling and running transfers. Set foundtransfer == true at the beginning to "trick"
1065
+ # it into skipping the initial 1 minute sleep
1066
+ foundtransfer = true
1067
+ while(true)
1068
+ # sleep for a minute if we didn't find any transfers available to run
1069
+ if(!foundtransfer)
1070
+ $logger.debug("sleeping")
1071
+ sleep(60)
1072
+ $logger.debug("waking up and considering which transfers need to be run")
1073
+ end
1074
+
1075
+ $logger.debug("choosing a new transfer to run")
1076
+
1077
+ # set foundtransfer to false so we will sleep if there are no more available transfers.
1078
+ foundtransfer = false
1079
+
1080
+ # skip all this if we have a global cutoff program that returns > 0
1081
+ if(configuration.general['cutoffprogram'] != '' and !system(configuration.general['cutoffprogram']))
1082
+ next
1083
+ end
1084
+
1085
+ # collect information about all the currently running transfers
1086
+ begin
1087
+ numberrunning = 0
1088
+ runningipaddresses = Hash.new()
1089
+ runninggroups = Hash.new()
1090
+
1091
+ # if we are coordinating with any other information sources, bring them in here
1092
+ configuration.general['coordinate'].each() { |url|
1093
+ get_coordination_info(url, runningipaddresses, runninggroups)
1094
+ }
1095
+
1096
+ # first loop through the transfers to get a count of the running transfers as well as to tag any
1097
+ # remote addresses that are currently handling transfers so we don't overload them
1098
+ configuration.transfers.each() { |transfer|
1099
+ if(transfer.running?())
1100
+ numberrunning = numberrunning + 1
1101
+ runningipaddresses[extract_ip(transfer.from, transfer.to)] = 1
1102
+ if(transfer.group != nil)
1103
+ runninggroups[transfer.group] = true
1104
+ end
1105
+ end
1106
+ }
1107
+ rescue Exception
1108
+ $logger.fatal("Error calculating running transfers - #{$!} - Exiting")
1109
+ $@.each() { |line|
1110
+ $logger.fatal(line)
1111
+ }
1112
+ exit(1)
1113
+ end
1114
+
1115
+ # now collect the eligable transfers for running
1116
+ begin
1117
+ # collect the possible transfers in a hash of arrays. The arrays are hashed by priority
1118
+ # so that we can easily randomly choose from amongst groups of like priority transfers
1119
+ eligabletransfers = Hash.new()
1120
+ configuration.transfers.each() { |transfer|
1121
+ if(transfer.eligable?())
1122
+ # get the ip address from either the from or the to. If we can get an ip address, and we already
1123
+ # have a transfer running for that ip, then this transfer is not eligable either
1124
+ ipaddress = extract_ip(transfer.from, transfer.to)
1125
+ if(ipaddress != nil and runningipaddresses.has_key?(ipaddress))
1126
+ next
1127
+ end
1128
+
1129
+ # get the group id from the transfer and compare it to a list of already involved
1130
+ # groups - never run more than one transfer in the same group
1131
+ if(transfer.group != nil and runninggroups.has_key?(transfer.group))
1132
+ next
1133
+ end
1134
+
1135
+ $logger.debug("priority of \"#{transfer.description}\" is #{transfer.priority}")
1136
+
1137
+ # add it to the eligabletransfers array if the priority is greater than 100 (meaning
1138
+ # it is at it's ideal running time). NOTE - it is important to get a static priority
1139
+ # instead of calling transfer.priority each time because the priority can change over the
1140
+ # course of the checks... leading to be juju when we make a slot for it and then try to add
1141
+ # to a different slot. ;)
1142
+ priority = transfer.priority
1143
+ if(priority >= 100)
1144
+ if(eligabletransfers[priority] == nil)
1145
+ eligabletransfers[priority] = Array.new()
1146
+ end
1147
+ eligabletransfers[priority] << transfer
1148
+ end
1149
+ end
1150
+ }
1151
+ rescue Exception
1152
+ $logger.fatal("Error during check of eligable transfers - #{$!} - Exiting")
1153
+ $@.each() { |line|
1154
+ $logger.fatal(line)
1155
+ }
1156
+ exit(1)
1157
+ end
1158
+
1159
+
1160
+ # choose a transfer to actually run
1161
+ begin
1162
+ # check the maximum number of transfers and whether we still have any to run
1163
+ if(numberrunning < configuration.general['maxtransfers'] and eligabletransfers.length > 0)
1164
+ priorities = eligabletransfers.keys()
1165
+ priorities.sort() { |x,y|
1166
+ y <=> x
1167
+ }
1168
+
1169
+ priorities.each() { |priority|
1170
+ if(foundtransfer == true)
1171
+ break
1172
+ end
1173
+
1174
+ idealtransfers = eligabletransfers[priority]
1175
+
1176
+ while(idealtransfers.length > 0)
1177
+ idealtransferindex = rand(idealtransfers.length)
1178
+ idealtransfer = idealtransfers[idealtransferindex]
1179
+ idealtransfers.delete_at(idealtransferindex)
1180
+
1181
+ # check the per-transfer cutoffprogram
1182
+ if(idealtransfer.cutoffprogram != '' and !system(idealtransfer.cutoffprogram))
1183
+ foundtransfer = false
1184
+ next
1185
+ end
1186
+
1187
+ foundtransfer = true
1188
+ idealtransfer.run()
1189
+ break;
1190
+ end
1191
+ }
1192
+ end
1193
+
1194
+ if(foundtransfer == false)
1195
+ $logger.debug("no transfer currently ready to run")
1196
+ end
1197
+ rescue Exception
1198
+ $logger.fatal("Error choosing ideal transfer - #{$!} - Exiting")
1199
+ $@.each() { |line|
1200
+ $logger.fatal(line)
1201
+ }
1202
+ exit(1)
1203
+ end
1204
+ end
1205
+ end
1206
+ end
1207
+
1208
+
1209
+ # run the program
1210
+ RsyncManager.main()
@@ -0,0 +1,3 @@
1
+ #!/usr/local/bin/ruby
2
+
3
+ require 'test/unit'
metadata ADDED
@@ -0,0 +1,41 @@
1
+ --- !ruby/object:Gem::Specification
2
+ rubygems_version: 0.8.10
3
+ specification_version: 1
4
+ name: rsyncmanager
5
+ version: !ruby/object:Gem::Version
6
+ version: "1.1"
7
+ date: 2005-06-19
8
+ summary: RsyncManager is a daemon for controlling and monitoring rsync transfers
9
+ require_paths:
10
+ - lib
11
+ email: david@grayskies.net
12
+ homepage: http://rsyncmanager.rubyforge.org
13
+ rubyforge_project: rsyncmanager
14
+ description:
15
+ autorequire:
16
+ default_executable:
17
+ bindir: bin
18
+ has_rdoc: false
19
+ required_ruby_version: !ruby/object:Gem::Version::Requirement
20
+ requirements:
21
+ -
22
+ - ">"
23
+ - !ruby/object:Gem::Version
24
+ version: 0.0.0
25
+ version:
26
+ platform: ruby
27
+ authors:
28
+ - David Powers
29
+ files:
30
+ - bin/rsyncmanager.rb
31
+ test_files:
32
+ - tests/test_configuration.rb
33
+ rdoc_options: []
34
+ extra_rdoc_files: []
35
+ executables:
36
+ - rsyncmanager.rb
37
+ extensions: []
38
+ requirements:
39
+ - rsync
40
+ - scp
41
+ dependencies: []