six-rsync 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -12,7 +12,7 @@ require 'rake/testtask'
12
12
 
13
13
  spec = Gem::Specification.new do |s|
14
14
  s.name = 'six-rsync'
15
- s.version = '0.6.0'
15
+ s.version = '0.6.1'
16
16
  s.has_rdoc = true
17
17
  s.extra_rdoc_files = ['README', 'LICENSE']
18
18
  s.summary = 'Your summary here'
data/bin/six-rsync CHANGED
@@ -13,5 +13,4 @@ rescue => e
13
13
  rescue => e
14
14
  puts "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
15
15
  end
16
- sleep 5
17
16
  end
data/bin/six-rsync-mirror CHANGED
@@ -13,5 +13,4 @@ rescue => e
13
13
  rescue => e
14
14
  puts "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
15
15
  end
16
- sleep 5
17
16
  end
data/lib/six/rsync/lib.rb CHANGED
@@ -3,7 +3,6 @@
3
3
  module Six
4
4
  module Repositories
5
5
  module Rsync
6
- REGEX_FOLDER = /(.*)[\\|\/](.*)/
7
6
  DIR_RSYNC = '.rsync'
8
7
  DIR_PACK = File.join(DIR_RSYNC, '.pack')
9
8
 
@@ -195,12 +194,11 @@ module Six
195
194
  change = true
196
195
  @logger.info "Packing #{i}/#{@repos_local[:wd].size}: #{key}"
197
196
  file = File.join(@rsync_work_dir, key)
198
- file[REGEX_FOLDER]
199
- folder = $1
197
+ folder = File.dirname(file)
200
198
  folder.gsub!(@rsync_work_dir, '')
201
199
  gzip(file)
202
200
  @repos_local[:pack]["#{key}.gz"] = md5("#{file}.gz")
203
- FileUtils.mkdir_p pack_path(folder) if folder
201
+ FileUtils.mkdir_p pack_path(folder) if folder.size > 0
204
202
  FileUtils.mv("#{file}.gz", pack_path("#{key}.gz"))
205
203
  end
206
204
  end
@@ -215,8 +213,6 @@ module Six
215
213
  packed = "#{key}.gz"
216
214
  change = true
217
215
  file = pack_path(packed)
218
- file[REGEX_FOLDER]
219
- folder = $2
220
216
 
221
217
  @logger.info "Removing: #{packed}"
222
218
  @repos_local[:wd].delete key
@@ -268,17 +264,13 @@ module Six
268
264
  handle_hosts
269
265
 
270
266
  host = config[:hosts].sample unless host
271
- # TODO: UNCLUSTERFUCK
272
267
  arr_opts = []
273
268
  arr_opts << PARAMS
274
-
275
- # Upload .pack changes
276
- if host[/^(\w)*\@/]
277
- arr_opts << @rsh
278
- end
269
+ arr_opts << @rsh if host[/^(\w)*\@/]
279
270
  arr_opts << esc(pack_path('.'))
280
271
  arr_opts << esc(File.join(host, '.pack'))
281
272
 
273
+ # Upload .pack changes
282
274
  command('', arr_opts)
283
275
  end
284
276
 
@@ -300,21 +292,8 @@ module Six
300
292
  arr_opts = []
301
293
  arr_opts << "-I" if opts[:force]
302
294
 
303
- begin
304
- init
305
- begin
306
- update('', arr_opts)
307
- rescue RsyncError => e
308
- @logger.error "Unable to sucessfully update... (#{e.class}: #{e.message})"
309
- @logger.debug e.backtrace.join("\n")
310
- end
311
- rescue => e
312
- @logger.error "Unable to initialize (#{e.class}: #{e.message})"
313
- @logger.debug e.backtrace.join("\n")
314
- FileUtils.rm_rf @rsync_work_dir if File.exists?(@rsync_work_dir)
315
- raise RsyncError
316
- end
317
-
295
+ init
296
+ update('', arr_opts, {:force => true})
318
297
  opts[:bare] ? {:repository => @rsync_work_dir} : {:working_directory => @rsync_work_dir}
319
298
  end
320
299
 
@@ -325,22 +304,28 @@ module Six
325
304
  handle_hosts
326
305
 
327
306
  load_repos(:local)
328
- load_repos(:remote)
307
+ begin
308
+ load_repos(:remote)
309
+ rescue
310
+ @logger.warn "WARN: .pack/.repository.yml seems corrupt, forcing full check"
311
+ opts[:force] = true
312
+ end
329
313
 
330
314
  hosts = config[:hosts].clone
331
315
  host = hosts.sample
332
316
 
333
317
  if opts[:force]
334
318
  done = false
335
- b = false
336
- verbose = @verbose
337
- @verbose = false
319
+ b, i = false, 0
320
+ #verbose = @verbose
321
+ #@verbose = false
338
322
  until hosts.empty? || done do
323
+ i += 1
339
324
  # FIXME: Nasty
340
325
  host = hosts.sample if b
341
326
  b = true
342
327
  hosts -= [host]
343
- @logger.info "Trying #{host}"
328
+ @logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
344
329
  begin
345
330
  arr_opts = []
346
331
  arr_opts << PARAMS
@@ -349,16 +334,25 @@ module Six
349
334
  arr_opts << esc(File.join(host, '.pack/.'))
350
335
  arr_opts << esc(pack_path)
351
336
  command(cmd, arr_opts)
352
- calc
353
- save_repos
337
+ load_repos(:remote)
354
338
  done = true
355
339
  rescue => e
356
- @logger.warn "#{e.class}: #{e.message}"
357
- @logger.debug e.backtrace.join("\n")
340
+ @logger.debug "#{e.class}: #{e.message} #{e.backtrace.join("\n")}"
358
341
  end
359
342
  end
360
- @verbose = verbose
361
- raise RsyncError if !done
343
+ #@verbose = verbose
344
+ if done
345
+ calc
346
+ save_repos
347
+ @logger.info "Verifying Unpacked files..."
348
+ compare_set(:wd)
349
+ # Bump version and make final save
350
+ @repos_local[:version] = @repos_remote[:version]
351
+ save_repos
352
+ else
353
+ @logger.warn "Exhausted all mirrors, please retry!"
354
+ raise RsyncError
355
+ end
362
356
  else
363
357
  #reset(:hard => true)
364
358
  calc
@@ -375,16 +369,17 @@ module Six
375
369
 
376
370
  if online
377
371
  hosts = config[:hosts].clone
378
- b = false
372
+ b, i = false, 0
379
373
  verbose = @verbose
380
374
  @verbose = false
381
375
 
382
376
  until hosts.empty? || done do
377
+ i += 1
383
378
  # FIXME: Nasty
384
379
  host = hosts.sample if b
385
380
  b = true
386
381
  hosts -= [host]
387
- @logger.info "Trying #{host}"
382
+ @logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
388
383
 
389
384
  begin
390
385
  FileUtils.cp(pack_path(".repository.yml"), rsync_path(".repository-pack.yml")) if File.exists?(pack_path(".repository.yml"))
@@ -397,8 +392,7 @@ module Six
397
392
  end
398
393
  done = true
399
394
  rescue => e
400
- @logger.warn "#{e.class}: #{e.message}"
401
- @logger.debug e.backtrace.join("\n")
395
+ @logger.debug "#{e.class} #{e.message}: #{e.backtrace.join("\n")}"
402
396
  FileUtils.cp(rsync_path(".repository-pack.yml"), pack_path(".repository.yml")) if File.exists?(rsync_path(".repository-pack.yml"))
403
397
  ensure
404
398
  FileUtils.rm(rsync_path(".repository-pack.yml")) if File.exists?(rsync_path(".repository-pack.yml"))
@@ -422,7 +416,7 @@ module Six
422
416
  end
423
417
  end
424
418
 
425
- def compare_set(typ, host, online = true)
419
+ def compare_set(typ, host = nil, online = true)
426
420
  #if local[typ][:md5] == remote[typ][:md5]
427
421
  # @logger.info "#{typ} Match!"
428
422
  #else
@@ -442,56 +436,57 @@ module Six
442
436
  case typ
443
437
  when :pack
444
438
  # direct unpack of gz into working folder
445
- # Update file
439
+ done = false
440
+
441
+ ## Pack
446
442
  if online
447
443
  hosts = config[:hosts].clone
448
- done = false
449
-
450
- ## Pack
451
- if online
452
- b = false
453
- until hosts.empty? || done do
454
- # FIXME: Nasty
455
- if b
456
- host = hosts.sample
457
- @logger.info "Trying #{host}"
458
- end
459
- slist = nil
460
- b = true
461
- hosts -= [host]
462
-
463
- # TODO: Progress bar
464
- arr_opts = []
465
- arr_opts << PARAMS
466
- arr_opts << @rsh if host[/^(\w)*\@/]
467
-
468
- if mismatch.size > (@repos_remote[typ].size / 2)
469
- # Process full folder
470
- @logger.info "Many files mismatched (#{mismatch.size}), running full update on .pack folder"
471
- else
472
- # Process only selective
473
- @logger.info "Fetching #{mismatch.size} files... Please wait"
474
- slist = File.join(TEMP_PATH, ".six-rsync_#{rand 9999}-list")
475
- slist.gsub!("\\", "/")
476
- File.open(slist, 'w') { |f| mismatch.each { |e| f.puts e } }
477
-
478
- arr_opts << "--files-from=#{win2cyg("\"#{slist}\"")}"
479
- end
480
-
481
- begin
482
- arr_opts << esc(File.join(host, '.pack/.'))
483
- arr_opts << esc(pack_path)
484
- command('', arr_opts)
485
-
486
- done = true
487
- rescue => e
488
- @logger.warn "Failure"
489
- @logger.debug "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
490
- ensure
491
- FileUtils.rm_f slist if slist
492
- end
444
+ host = hosts.sample unless host
445
+ b, i = false, 0
446
+ until hosts.empty? || done do
447
+ i += 1
448
+ # FIXME: Nasty
449
+ if b
450
+ host = hosts.sample
451
+ @logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
452
+ end
453
+ slist = nil
454
+ b = true
455
+ hosts -= [host]
456
+
457
+ # TODO: Progress bar
458
+ arr_opts = []
459
+ arr_opts << PARAMS
460
+ arr_opts << @rsh if host[/^(\w)*\@/]
461
+
462
+ if mismatch.size > (@repos_remote[typ].size / 2)
463
+ # Process full folder
464
+ @logger.info "Many files mismatched (#{mismatch.size}), running full update on .pack folder"
465
+ else
466
+ # Process only selective
467
+ @logger.info "Fetching #{mismatch.size} files... Please wait"
468
+ slist = File.join(TEMP_PATH, ".six-rsync_#{rand 9999}-list")
469
+ slist.gsub!("\\", "/")
470
+ File.open(slist, 'w') { |f| mismatch.each { |e| f.puts e } }
471
+
472
+ arr_opts << "--files-from=#{win2cyg("\"#{slist}\"")}"
493
473
  end
494
- @logger.warn "There was a problem during updating, please retry!" unless done
474
+
475
+ begin
476
+ arr_opts << esc(File.join(host, '.pack/.'))
477
+ arr_opts << esc(pack_path)
478
+ command('', arr_opts)
479
+
480
+ done = true
481
+ rescue => e
482
+ @logger.debug "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
483
+ ensure
484
+ FileUtils.rm_f slist if slist
485
+ end
486
+ end
487
+ unless done
488
+ @logger.warn "Exhausted all mirrors, please retry!"
489
+ raise RsyncError
495
490
  end
496
491
  end
497
492
  when :wd
@@ -567,10 +562,7 @@ module Six
567
562
  end
568
563
 
569
564
  def fetch_file(path, host)
570
- path[/(.*)\/(.*)/]
571
- folder, file = $1, $2
572
- folder = "." unless folder
573
- file = path unless file
565
+ folder = File.dirname(path)
574
566
  # Only fetch a specific file
575
567
  @logger.debug "Fetching #{path} from #{host}"
576
568
  arr_opts = []
@@ -721,9 +713,7 @@ module Six
721
713
  # TODO: Evaluate if this is actually wanted / useful at all..
722
714
  =begin
723
715
  if file[/\.tar\.?/]
724
- file[/(.*)\/(.*)/]
725
- fil = $2
726
- fil = file unless fil
716
+ fil = File.basename(file)
727
717
  f2 = fil.gsub('.gz', '')
728
718
  zip7(f2)
729
719
  FileUtils.rm_f f2
@@ -744,9 +734,8 @@ module Six
744
734
  relative = file.clone
745
735
  relative.gsub!(@rsync_work_dir, '')
746
736
  relative.gsub!(/^[\\|\/]\.rsync[\\|\/]\.pack[\\|\/]/, '')
747
- fil = relative
748
- folder = "."
749
- folder, fil = $1, $2 if relative[/(.*)\/(.*)/]
737
+ folder = File.dirname(relative)
738
+ fil = File.basename(relative)
750
739
  #puts "Relative: #{relative}, Folder: #{folder}, File: #{fil} (Origin: #{file})"
751
740
 
752
741
  path = File.join(@rsync_work_dir, folder)
@@ -767,7 +756,7 @@ module Six
767
756
  opts << "--timeout=#{DEFAULT_TIMEOUT}"
768
757
 
769
758
  opts = [opts].flatten.map {|s| s }.join(' ') # escape()
770
- rsync_cmd = win2cyg("rsync #{cmd} #{opts} #{redirect} 2>&1")
759
+ rsync_cmd = win2cyg("rsync #{cmd} #{opts} #{redirect}") # 2>&1
771
760
 
772
761
  @logger.debug(rsync_cmd) if @logger
773
762
 
@@ -776,51 +765,48 @@ module Six
776
765
  end
777
766
 
778
767
  def run_command(rsync_cmd, &block)
779
- # TODO: Make this switchable? Verbosity ?
780
- # Or actually parse this live for own stats?
781
- s = nil
782
- out = ''
783
-
784
- # Simpler method but on windows the !? exitstatus is not working properly..
785
- # Does nicely display error output in logwindow though
786
- io = IO.popen(rsync_cmd)
787
- io.sync = true
768
+ out, err = '', ''
769
+ buff = []
770
+ status = nil
788
771
  oldsync = STDOUT.sync
789
772
  STDOUT.sync = true
790
773
 
791
- #io.each do |buffer|
792
- # process_msg buffer
793
- # out << buffer
794
- #end
774
+ po = Open3.popen3(rsync_cmd) do |io_in, io_out, io_err, waitth|
775
+ io_out.each_byte do |buffer|
776
+ char = buffer.chr
777
+ buff << char
778
+ if ["\n", "\r"].include?(char)
779
+ b = buff.join("")
780
+ print b if @verbose
781
+ out << b
782
+ buff = []
783
+ end
784
+ end
795
785
 
796
- buff = []
797
- io.each_byte do |buffer|
798
- buffer = buffer.chr
799
- buff << buffer
800
- if ["\n", "\r"].include?(buffer)
801
- b = buff.join("")
802
- process_msg b
803
- out << b
804
- buff = []
786
+ io_err.each do |line|
787
+ print line
788
+ case line
789
+ when /max connections \((.*)\) reached/
790
+ @logger.warn "Server reached maximum connections."
791
+ end
792
+ err << line
805
793
  end
794
+ status = waitth.value
806
795
  end
807
796
 
808
797
  unless buff.empty?
809
798
  b = buff.join("")
810
- process_msg b
799
+ print b if @verbose
811
800
  out << b
812
- end
801
+ end
813
802
 
814
- out[/rsync error: .* \(code ([0-9]*)\)/]
815
- status = $1 ? $1.to_i : 0
803
+ @logger.debug "Status: #{status}"
804
+ @logger.debug "Err: #{err}" # TODO: Throw this into the info/error log?
805
+ @logger.debug "Output: #{out}"
816
806
 
817
- if status > 0
818
- return '' if status == 1 && out == ''
819
- case out
820
- when /max connections \((.*)\) reached/
821
- @logger.warn "Server reached maximum connections."
822
- end
823
- raise Rsync::RsyncExecuteError.new(rsync_cmd + ':' + out)
807
+ if status.exitstatus > 0
808
+ #return 0 if status.exitstatus == 1 && out == ''
809
+ raise Rsync::RsyncExecuteError.new(rsync_cmd + ':' + err + ':' + out)
824
810
  end
825
811
 
826
812
  STDOUT.sync = false unless STDOUT.sync == oldsync
@@ -828,17 +814,23 @@ module Six
828
814
  status
829
815
  end
830
816
 
831
- def process_msg(msg)
832
- if msg[/\r/] #msg[/[k|m|g]?B\/s/i]
833
- #msg.gsub!("\n", '')
834
- print msg if @verbose
835
- else
836
- @logger.debug msg
837
- print msg if @verbose
838
- end
839
- msg
840
- end
817
+ =begin
818
+ # Simpler method but on windows the !? exitstatus is not working properly..
819
+ # Does nicely display error output in logwindow though
820
+ io = IO.popen(rsync_cmd)
821
+ io.sync = true
841
822
 
823
+ #io.each do |buffer|
824
+ # process_msg buffer
825
+ # out << buffer
826
+ #end
827
+ out[/rsync error: .* \(code ([0-9]*)\)/]
828
+ status = $1 ? $1.to_i : 0
829
+ case out
830
+ when /max connections \((.*)\) reached/
831
+ @logger.warn "Server reached maximum connections."
832
+ end
833
+ =end
842
834
  end
843
835
  end
844
836
  end
data/lib/six/rsync-app.rb CHANGED
@@ -90,7 +90,7 @@ module Six
90
90
  if options[:logging]
91
91
  o_file = Log4r::FileOutputter.new "#{COMPONENT}-file",
92
92
  'level' => 0, # All
93
- :filename => "#{COMPONENT}.log",
93
+ :filename => File.join(DATA_PATH, 'logs', "#{COMPONENT}-file"),
94
94
  'formatter' => format2
95
95
  #:maxsize => 1024
96
96
  @@log.outputters << o_file
@@ -101,7 +101,7 @@ module Six
101
101
  'formatter' => format1
102
102
 
103
103
  o_err = Log4r::StderrOutputter.new "#{COMPONENT}-stderr",
104
- 'level' => 4, # Error and Up
104
+ 'level' => 5, # Error and Up
105
105
  'formatter' => format1
106
106
 
107
107
  @@log.outputters << o_out << o_err
data/lib/six/rsync.rb CHANGED
@@ -3,17 +3,13 @@
3
3
  require 'fileutils'
4
4
  require 'digest/md5'
5
5
  require 'yaml'
6
+ require 'open3'
6
7
 
7
- BASE_PATH = Dir.pwd
8
8
  require 'six/rsync/path'
9
9
  require 'six/rsync/repository'
10
10
  require 'six/rsync/working_directory'
11
11
  require 'six/rsync/lib'
12
12
  require 'six/rsync/base'
13
- #require 'six/popen'
14
-
15
- require 'open3'
16
- #require 'win32/open3'
17
13
 
18
14
  case RUBY_VERSION
19
15
  when /1\.8\.[0-9]/
@@ -31,8 +27,8 @@ module Six
31
27
 
32
28
  module Rsync
33
29
  COMPONENT = 'six-rsync'
34
- VERSION = '0.6.0'
35
- FOLDER = /(.*)\/(.*)/
30
+ VERSION = '0.6.1'
31
+ BASE_PATH = Dir.pwd
36
32
 
37
33
  case RUBY_PLATFORM
38
34
  when /-mingw32$/, /-mswin32$/
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 0
7
7
  - 6
8
- - 0
9
- version: 0.6.0
8
+ - 1
9
+ version: 0.6.1
10
10
  platform: ruby
11
11
  authors:
12
12
  - Sickboy
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-05-07 00:00:00 +02:00
17
+ date: 2010-05-08 00:00:00 +02:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency