six-rsync 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +1 -1
- data/bin/six-rsync +0 -1
- data/bin/six-rsync-mirror +0 -1
- data/lib/six/rsync/lib.rb +136 -144
- data/lib/six/rsync-app.rb +2 -2
- data/lib/six/rsync.rb +3 -7
- metadata +3 -3
data/Rakefile
CHANGED
data/bin/six-rsync
CHANGED
data/bin/six-rsync-mirror
CHANGED
data/lib/six/rsync/lib.rb
CHANGED
@@ -3,7 +3,6 @@
|
|
3
3
|
module Six
|
4
4
|
module Repositories
|
5
5
|
module Rsync
|
6
|
-
REGEX_FOLDER = /(.*)[\\|\/](.*)/
|
7
6
|
DIR_RSYNC = '.rsync'
|
8
7
|
DIR_PACK = File.join(DIR_RSYNC, '.pack')
|
9
8
|
|
@@ -195,12 +194,11 @@ module Six
|
|
195
194
|
change = true
|
196
195
|
@logger.info "Packing #{i}/#{@repos_local[:wd].size}: #{key}"
|
197
196
|
file = File.join(@rsync_work_dir, key)
|
198
|
-
file
|
199
|
-
folder = $1
|
197
|
+
folder = File.dirname(file)
|
200
198
|
folder.gsub!(@rsync_work_dir, '')
|
201
199
|
gzip(file)
|
202
200
|
@repos_local[:pack]["#{key}.gz"] = md5("#{file}.gz")
|
203
|
-
FileUtils.mkdir_p pack_path(folder) if folder
|
201
|
+
FileUtils.mkdir_p pack_path(folder) if folder.size > 0
|
204
202
|
FileUtils.mv("#{file}.gz", pack_path("#{key}.gz"))
|
205
203
|
end
|
206
204
|
end
|
@@ -215,8 +213,6 @@ module Six
|
|
215
213
|
packed = "#{key}.gz"
|
216
214
|
change = true
|
217
215
|
file = pack_path(packed)
|
218
|
-
file[REGEX_FOLDER]
|
219
|
-
folder = $2
|
220
216
|
|
221
217
|
@logger.info "Removing: #{packed}"
|
222
218
|
@repos_local[:wd].delete key
|
@@ -268,17 +264,13 @@ module Six
|
|
268
264
|
handle_hosts
|
269
265
|
|
270
266
|
host = config[:hosts].sample unless host
|
271
|
-
# TODO: UNCLUSTERFUCK
|
272
267
|
arr_opts = []
|
273
268
|
arr_opts << PARAMS
|
274
|
-
|
275
|
-
# Upload .pack changes
|
276
|
-
if host[/^(\w)*\@/]
|
277
|
-
arr_opts << @rsh
|
278
|
-
end
|
269
|
+
arr_opts << @rsh if host[/^(\w)*\@/]
|
279
270
|
arr_opts << esc(pack_path('.'))
|
280
271
|
arr_opts << esc(File.join(host, '.pack'))
|
281
272
|
|
273
|
+
# Upload .pack changes
|
282
274
|
command('', arr_opts)
|
283
275
|
end
|
284
276
|
|
@@ -300,21 +292,8 @@ module Six
|
|
300
292
|
arr_opts = []
|
301
293
|
arr_opts << "-I" if opts[:force]
|
302
294
|
|
303
|
-
|
304
|
-
|
305
|
-
begin
|
306
|
-
update('', arr_opts)
|
307
|
-
rescue RsyncError => e
|
308
|
-
@logger.error "Unable to sucessfully update... (#{e.class}: #{e.message})"
|
309
|
-
@logger.debug e.backtrace.join("\n")
|
310
|
-
end
|
311
|
-
rescue => e
|
312
|
-
@logger.error "Unable to initialize (#{e.class}: #{e.message})"
|
313
|
-
@logger.debug e.backtrace.join("\n")
|
314
|
-
FileUtils.rm_rf @rsync_work_dir if File.exists?(@rsync_work_dir)
|
315
|
-
raise RsyncError
|
316
|
-
end
|
317
|
-
|
295
|
+
init
|
296
|
+
update('', arr_opts, {:force => true})
|
318
297
|
opts[:bare] ? {:repository => @rsync_work_dir} : {:working_directory => @rsync_work_dir}
|
319
298
|
end
|
320
299
|
|
@@ -325,22 +304,28 @@ module Six
|
|
325
304
|
handle_hosts
|
326
305
|
|
327
306
|
load_repos(:local)
|
328
|
-
|
307
|
+
begin
|
308
|
+
load_repos(:remote)
|
309
|
+
rescue
|
310
|
+
@logger.warn "WARN: .pack/.repository.yml seems corrupt, forcing full check"
|
311
|
+
opts[:force] = true
|
312
|
+
end
|
329
313
|
|
330
314
|
hosts = config[:hosts].clone
|
331
315
|
host = hosts.sample
|
332
316
|
|
333
317
|
if opts[:force]
|
334
318
|
done = false
|
335
|
-
b = false
|
336
|
-
verbose = @verbose
|
337
|
-
|
319
|
+
b, i = false, 0
|
320
|
+
#verbose = @verbose
|
321
|
+
#@verbose = false
|
338
322
|
until hosts.empty? || done do
|
323
|
+
i += 1
|
339
324
|
# FIXME: Nasty
|
340
325
|
host = hosts.sample if b
|
341
326
|
b = true
|
342
327
|
hosts -= [host]
|
343
|
-
@logger.info "Trying #{host}"
|
328
|
+
@logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
|
344
329
|
begin
|
345
330
|
arr_opts = []
|
346
331
|
arr_opts << PARAMS
|
@@ -349,16 +334,25 @@ module Six
|
|
349
334
|
arr_opts << esc(File.join(host, '.pack/.'))
|
350
335
|
arr_opts << esc(pack_path)
|
351
336
|
command(cmd, arr_opts)
|
352
|
-
|
353
|
-
save_repos
|
337
|
+
load_repos(:remote)
|
354
338
|
done = true
|
355
339
|
rescue => e
|
356
|
-
@logger.
|
357
|
-
@logger.debug e.backtrace.join("\n")
|
340
|
+
@logger.debug "#{e.class}: #{e.message} #{e.backtrace.join("\n")}"
|
358
341
|
end
|
359
342
|
end
|
360
|
-
|
361
|
-
|
343
|
+
#@verbose = verbose
|
344
|
+
if done
|
345
|
+
calc
|
346
|
+
save_repos
|
347
|
+
@logger.info "Verifying Unpacked files..."
|
348
|
+
compare_set(:wd)
|
349
|
+
# Bump version and make final save
|
350
|
+
@repos_local[:version] = @repos_remote[:version]
|
351
|
+
save_repos
|
352
|
+
else
|
353
|
+
@logger.warn "Exhausted all mirrors, please retry!"
|
354
|
+
raise RsyncError
|
355
|
+
end
|
362
356
|
else
|
363
357
|
#reset(:hard => true)
|
364
358
|
calc
|
@@ -375,16 +369,17 @@ module Six
|
|
375
369
|
|
376
370
|
if online
|
377
371
|
hosts = config[:hosts].clone
|
378
|
-
b = false
|
372
|
+
b, i = false, 0
|
379
373
|
verbose = @verbose
|
380
374
|
@verbose = false
|
381
375
|
|
382
376
|
until hosts.empty? || done do
|
377
|
+
i += 1
|
383
378
|
# FIXME: Nasty
|
384
379
|
host = hosts.sample if b
|
385
380
|
b = true
|
386
381
|
hosts -= [host]
|
387
|
-
@logger.info "Trying #{host}"
|
382
|
+
@logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
|
388
383
|
|
389
384
|
begin
|
390
385
|
FileUtils.cp(pack_path(".repository.yml"), rsync_path(".repository-pack.yml")) if File.exists?(pack_path(".repository.yml"))
|
@@ -397,8 +392,7 @@ module Six
|
|
397
392
|
end
|
398
393
|
done = true
|
399
394
|
rescue => e
|
400
|
-
@logger.
|
401
|
-
@logger.debug e.backtrace.join("\n")
|
395
|
+
@logger.debug "#{e.class} #{e.message}: #{e.backtrace.join("\n")}"
|
402
396
|
FileUtils.cp(rsync_path(".repository-pack.yml"), pack_path(".repository.yml")) if File.exists?(rsync_path(".repository-pack.yml"))
|
403
397
|
ensure
|
404
398
|
FileUtils.rm(rsync_path(".repository-pack.yml")) if File.exists?(rsync_path(".repository-pack.yml"))
|
@@ -422,7 +416,7 @@ module Six
|
|
422
416
|
end
|
423
417
|
end
|
424
418
|
|
425
|
-
def compare_set(typ, host, online = true)
|
419
|
+
def compare_set(typ, host = nil, online = true)
|
426
420
|
#if local[typ][:md5] == remote[typ][:md5]
|
427
421
|
# @logger.info "#{typ} Match!"
|
428
422
|
#else
|
@@ -442,56 +436,57 @@ module Six
|
|
442
436
|
case typ
|
443
437
|
when :pack
|
444
438
|
# direct unpack of gz into working folder
|
445
|
-
|
439
|
+
done = false
|
440
|
+
|
441
|
+
## Pack
|
446
442
|
if online
|
447
443
|
hosts = config[:hosts].clone
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
arr_opts << "--files-from=#{win2cyg("\"#{slist}\"")}"
|
479
|
-
end
|
480
|
-
|
481
|
-
begin
|
482
|
-
arr_opts << esc(File.join(host, '.pack/.'))
|
483
|
-
arr_opts << esc(pack_path)
|
484
|
-
command('', arr_opts)
|
485
|
-
|
486
|
-
done = true
|
487
|
-
rescue => e
|
488
|
-
@logger.warn "Failure"
|
489
|
-
@logger.debug "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
|
490
|
-
ensure
|
491
|
-
FileUtils.rm_f slist if slist
|
492
|
-
end
|
444
|
+
host = hosts.sample unless host
|
445
|
+
b, i = false, 0
|
446
|
+
until hosts.empty? || done do
|
447
|
+
i += 1
|
448
|
+
# FIXME: Nasty
|
449
|
+
if b
|
450
|
+
host = hosts.sample
|
451
|
+
@logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
|
452
|
+
end
|
453
|
+
slist = nil
|
454
|
+
b = true
|
455
|
+
hosts -= [host]
|
456
|
+
|
457
|
+
# TODO: Progress bar
|
458
|
+
arr_opts = []
|
459
|
+
arr_opts << PARAMS
|
460
|
+
arr_opts << @rsh if host[/^(\w)*\@/]
|
461
|
+
|
462
|
+
if mismatch.size > (@repos_remote[typ].size / 2)
|
463
|
+
# Process full folder
|
464
|
+
@logger.info "Many files mismatched (#{mismatch.size}), running full update on .pack folder"
|
465
|
+
else
|
466
|
+
# Process only selective
|
467
|
+
@logger.info "Fetching #{mismatch.size} files... Please wait"
|
468
|
+
slist = File.join(TEMP_PATH, ".six-rsync_#{rand 9999}-list")
|
469
|
+
slist.gsub!("\\", "/")
|
470
|
+
File.open(slist, 'w') { |f| mismatch.each { |e| f.puts e } }
|
471
|
+
|
472
|
+
arr_opts << "--files-from=#{win2cyg("\"#{slist}\"")}"
|
493
473
|
end
|
494
|
-
|
474
|
+
|
475
|
+
begin
|
476
|
+
arr_opts << esc(File.join(host, '.pack/.'))
|
477
|
+
arr_opts << esc(pack_path)
|
478
|
+
command('', arr_opts)
|
479
|
+
|
480
|
+
done = true
|
481
|
+
rescue => e
|
482
|
+
@logger.debug "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
|
483
|
+
ensure
|
484
|
+
FileUtils.rm_f slist if slist
|
485
|
+
end
|
486
|
+
end
|
487
|
+
unless done
|
488
|
+
@logger.warn "Exhausted all mirrors, please retry!"
|
489
|
+
raise RsyncError
|
495
490
|
end
|
496
491
|
end
|
497
492
|
when :wd
|
@@ -567,10 +562,7 @@ module Six
|
|
567
562
|
end
|
568
563
|
|
569
564
|
def fetch_file(path, host)
|
570
|
-
path
|
571
|
-
folder, file = $1, $2
|
572
|
-
folder = "." unless folder
|
573
|
-
file = path unless file
|
565
|
+
folder = File.dirname(path)
|
574
566
|
# Only fetch a specific file
|
575
567
|
@logger.debug "Fetching #{path} from #{host}"
|
576
568
|
arr_opts = []
|
@@ -721,9 +713,7 @@ module Six
|
|
721
713
|
# TODO: Evaluate if this is actually wanted / useful at all..
|
722
714
|
=begin
|
723
715
|
if file[/\.tar\.?/]
|
724
|
-
file
|
725
|
-
fil = $2
|
726
|
-
fil = file unless fil
|
716
|
+
fil = File.basename(file)
|
727
717
|
f2 = fil.gsub('.gz', '')
|
728
718
|
zip7(f2)
|
729
719
|
FileUtils.rm_f f2
|
@@ -744,9 +734,8 @@ module Six
|
|
744
734
|
relative = file.clone
|
745
735
|
relative.gsub!(@rsync_work_dir, '')
|
746
736
|
relative.gsub!(/^[\\|\/]\.rsync[\\|\/]\.pack[\\|\/]/, '')
|
747
|
-
|
748
|
-
|
749
|
-
folder, fil = $1, $2 if relative[/(.*)\/(.*)/]
|
737
|
+
folder = File.dirname(relative)
|
738
|
+
fil = File.basename(relative)
|
750
739
|
#puts "Relative: #{relative}, Folder: #{folder}, File: #{fil} (Origin: #{file})"
|
751
740
|
|
752
741
|
path = File.join(@rsync_work_dir, folder)
|
@@ -767,7 +756,7 @@ module Six
|
|
767
756
|
opts << "--timeout=#{DEFAULT_TIMEOUT}"
|
768
757
|
|
769
758
|
opts = [opts].flatten.map {|s| s }.join(' ') # escape()
|
770
|
-
rsync_cmd = win2cyg("rsync #{cmd} #{opts} #{redirect} 2>&1
|
759
|
+
rsync_cmd = win2cyg("rsync #{cmd} #{opts} #{redirect}") # 2>&1
|
771
760
|
|
772
761
|
@logger.debug(rsync_cmd) if @logger
|
773
762
|
|
@@ -776,51 +765,48 @@ module Six
|
|
776
765
|
end
|
777
766
|
|
778
767
|
def run_command(rsync_cmd, &block)
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
out = ''
|
783
|
-
|
784
|
-
# Simpler method but on windows the !? exitstatus is not working properly..
|
785
|
-
# Does nicely display error output in logwindow though
|
786
|
-
io = IO.popen(rsync_cmd)
|
787
|
-
io.sync = true
|
768
|
+
out, err = '', ''
|
769
|
+
buff = []
|
770
|
+
status = nil
|
788
771
|
oldsync = STDOUT.sync
|
789
772
|
STDOUT.sync = true
|
790
773
|
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
774
|
+
po = Open3.popen3(rsync_cmd) do |io_in, io_out, io_err, waitth|
|
775
|
+
io_out.each_byte do |buffer|
|
776
|
+
char = buffer.chr
|
777
|
+
buff << char
|
778
|
+
if ["\n", "\r"].include?(char)
|
779
|
+
b = buff.join("")
|
780
|
+
print b if @verbose
|
781
|
+
out << b
|
782
|
+
buff = []
|
783
|
+
end
|
784
|
+
end
|
795
785
|
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
|
802
|
-
|
803
|
-
out << b
|
804
|
-
buff = []
|
786
|
+
io_err.each do |line|
|
787
|
+
print line
|
788
|
+
case line
|
789
|
+
when /max connections \((.*)\) reached/
|
790
|
+
@logger.warn "Server reached maximum connections."
|
791
|
+
end
|
792
|
+
err << line
|
805
793
|
end
|
794
|
+
status = waitth.value
|
806
795
|
end
|
807
796
|
|
808
797
|
unless buff.empty?
|
809
798
|
b = buff.join("")
|
810
|
-
|
799
|
+
print b if @verbose
|
811
800
|
out << b
|
812
|
-
end
|
801
|
+
end
|
813
802
|
|
814
|
-
|
815
|
-
|
803
|
+
@logger.debug "Status: #{status}"
|
804
|
+
@logger.debug "Err: #{err}" # TODO: Throw this into the info/error log?
|
805
|
+
@logger.debug "Output: #{out}"
|
816
806
|
|
817
|
-
if status > 0
|
818
|
-
return
|
819
|
-
|
820
|
-
when /max connections \((.*)\) reached/
|
821
|
-
@logger.warn "Server reached maximum connections."
|
822
|
-
end
|
823
|
-
raise Rsync::RsyncExecuteError.new(rsync_cmd + ':' + out)
|
807
|
+
if status.exitstatus > 0
|
808
|
+
#return 0 if status.exitstatus == 1 && out == ''
|
809
|
+
raise Rsync::RsyncExecuteError.new(rsync_cmd + ':' + err + ':' + out)
|
824
810
|
end
|
825
811
|
|
826
812
|
STDOUT.sync = false unless STDOUT.sync == oldsync
|
@@ -828,17 +814,23 @@ module Six
|
|
828
814
|
status
|
829
815
|
end
|
830
816
|
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
@logger.debug msg
|
837
|
-
print msg if @verbose
|
838
|
-
end
|
839
|
-
msg
|
840
|
-
end
|
817
|
+
=begin
|
818
|
+
# Simpler method but on windows the !? exitstatus is not working properly..
|
819
|
+
# Does nicely display error output in logwindow though
|
820
|
+
io = IO.popen(rsync_cmd)
|
821
|
+
io.sync = true
|
841
822
|
|
823
|
+
#io.each do |buffer|
|
824
|
+
# process_msg buffer
|
825
|
+
# out << buffer
|
826
|
+
#end
|
827
|
+
out[/rsync error: .* \(code ([0-9]*)\)/]
|
828
|
+
status = $1 ? $1.to_i : 0
|
829
|
+
case out
|
830
|
+
when /max connections \((.*)\) reached/
|
831
|
+
@logger.warn "Server reached maximum connections."
|
832
|
+
end
|
833
|
+
=end
|
842
834
|
end
|
843
835
|
end
|
844
836
|
end
|
data/lib/six/rsync-app.rb
CHANGED
@@ -90,7 +90,7 @@ module Six
|
|
90
90
|
if options[:logging]
|
91
91
|
o_file = Log4r::FileOutputter.new "#{COMPONENT}-file",
|
92
92
|
'level' => 0, # All
|
93
|
-
:filename => "#{COMPONENT}
|
93
|
+
:filename => File.join(DATA_PATH, 'logs', "#{COMPONENT}-file"),
|
94
94
|
'formatter' => format2
|
95
95
|
#:maxsize => 1024
|
96
96
|
@@log.outputters << o_file
|
@@ -101,7 +101,7 @@ module Six
|
|
101
101
|
'formatter' => format1
|
102
102
|
|
103
103
|
o_err = Log4r::StderrOutputter.new "#{COMPONENT}-stderr",
|
104
|
-
'level' =>
|
104
|
+
'level' => 5, # Error and Up
|
105
105
|
'formatter' => format1
|
106
106
|
|
107
107
|
@@log.outputters << o_out << o_err
|
data/lib/six/rsync.rb
CHANGED
@@ -3,17 +3,13 @@
|
|
3
3
|
require 'fileutils'
|
4
4
|
require 'digest/md5'
|
5
5
|
require 'yaml'
|
6
|
+
require 'open3'
|
6
7
|
|
7
|
-
BASE_PATH = Dir.pwd
|
8
8
|
require 'six/rsync/path'
|
9
9
|
require 'six/rsync/repository'
|
10
10
|
require 'six/rsync/working_directory'
|
11
11
|
require 'six/rsync/lib'
|
12
12
|
require 'six/rsync/base'
|
13
|
-
#require 'six/popen'
|
14
|
-
|
15
|
-
require 'open3'
|
16
|
-
#require 'win32/open3'
|
17
13
|
|
18
14
|
case RUBY_VERSION
|
19
15
|
when /1\.8\.[0-9]/
|
@@ -31,8 +27,8 @@ module Six
|
|
31
27
|
|
32
28
|
module Rsync
|
33
29
|
COMPONENT = 'six-rsync'
|
34
|
-
VERSION = '0.6.
|
35
|
-
|
30
|
+
VERSION = '0.6.1'
|
31
|
+
BASE_PATH = Dir.pwd
|
36
32
|
|
37
33
|
case RUBY_PLATFORM
|
38
34
|
when /-mingw32$/, /-mswin32$/
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 6
|
8
|
-
-
|
9
|
-
version: 0.6.
|
8
|
+
- 1
|
9
|
+
version: 0.6.1
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Sickboy
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-05-
|
17
|
+
date: 2010-05-08 00:00:00 +02:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|