six-rsync 0.6.0 → 0.6.1

data/Rakefile

@@ -12,7 +12,7 @@ require 'rake/testtask'
 
 spec = Gem::Specification.new do |s|
   s.name = 'six-rsync'
-  s.version = '0.6.0'
+  s.version = '0.6.1'
   s.has_rdoc = true
   s.extra_rdoc_files = ['README', 'LICENSE']
   s.summary = 'Your summary here'

data/bin/six-rsync

@@ -13,5 +13,4 @@ rescue => e
   rescue => e
       puts "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
   end
-  sleep 5
 end

data/bin/six-rsync-mirror

@@ -13,5 +13,4 @@ rescue => e
   rescue => e
       puts "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
   end
-  sleep 5
 end

data/lib/six/rsync/lib.rb

@@ -3,7 +3,6 @@
 module Six
   module Repositories
     module Rsync
-      REGEX_FOLDER = /(.*)[\\|\/](.*)/
       DIR_RSYNC = '.rsync'
       DIR_PACK = File.join(DIR_RSYNC, '.pack')
 
@@ -195,12 +194,11 @@ module Six
               change = true
               @logger.info "Packing #{i}/#{@repos_local[:wd].size}: #{key}"
               file = File.join(@rsync_work_dir, key)
-              file[REGEX_FOLDER]
-              folder = $1
+              folder = File.dirname(file)
               folder.gsub!(@rsync_work_dir, '')
               gzip(file)
               @repos_local[:pack]["#{key}.gz"] = md5("#{file}.gz")
-              FileUtils.mkdir_p pack_path(folder) if folder
+              FileUtils.mkdir_p pack_path(folder) if folder.size > 0
               FileUtils.mv("#{file}.gz", pack_path("#{key}.gz"))
             end
           end
@@ -215,8 +213,6 @@ module Six
               packed = "#{key}.gz"
               change = true
               file = pack_path(packed)
-              file[REGEX_FOLDER]
-              folder = $2
 
               @logger.info "Removing: #{packed}"
               @repos_local[:wd].delete key
@@ -268,17 +264,13 @@ module Six
           handle_hosts
 
           host = config[:hosts].sample unless host
-          # TODO: UNCLUSTERFUCK
           arr_opts = []
           arr_opts << PARAMS
-
-          # Upload .pack changes
-          if host[/^(\w)*\@/]
-            arr_opts << @rsh
-          end
+          arr_opts << @rsh if host[/^(\w)*\@/]
           arr_opts << esc(pack_path('.'))
           arr_opts << esc(File.join(host, '.pack'))
 
+          # Upload .pack changes
           command('', arr_opts)
         end
 
@@ -300,21 +292,8 @@ module Six
           arr_opts = []
           arr_opts << "-I" if opts[:force]
 
-          begin
-            init
-            begin
-              update('', arr_opts)
-            rescue RsyncError => e
-              @logger.error "Unable to sucessfully update... (#{e.class}: #{e.message})"
-              @logger.debug e.backtrace.join("\n")
-            end
-          rescue => e
-            @logger.error "Unable to initialize (#{e.class}: #{e.message})"
-            @logger.debug e.backtrace.join("\n")
-            FileUtils.rm_rf @rsync_work_dir if File.exists?(@rsync_work_dir)
-            raise RsyncError
-          end
-
+          init
+          update('', arr_opts, {:force => true})
           opts[:bare] ? {:repository => @rsync_work_dir} : {:working_directory => @rsync_work_dir}
         end
 
@@ -325,22 +304,28 @@ module Six
           handle_hosts
 
           load_repos(:local)
-          load_repos(:remote)
+          begin
+            load_repos(:remote)
+          rescue
+            @logger.warn "WARN: .pack/.repository.yml seems corrupt, forcing full check"
+            opts[:force] = true
+          end
 
           hosts = config[:hosts].clone
           host = hosts.sample
 
           if opts[:force]
             done = false
-            b = false
-            verbose = @verbose
-            @verbose = false
+            b, i = false, 0
+            #verbose = @verbose
+            #@verbose = false
             until hosts.empty? || done do
+              i += 1
               # FIXME: Nasty
               host = hosts.sample if b
               b = true
               hosts -= [host]
-              @logger.info "Trying #{host}"
+              @logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
               begin
                 arr_opts = []
                 arr_opts << PARAMS
@@ -349,16 +334,25 @@ module Six
                 arr_opts << esc(File.join(host, '.pack/.'))
                 arr_opts << esc(pack_path)
                 command(cmd, arr_opts)
-                calc
-                save_repos
+                load_repos(:remote)
                 done = true
               rescue => e
-                @logger.warn "#{e.class}: #{e.message}"
-                @logger.debug e.backtrace.join("\n")
+                @logger.debug "#{e.class}: #{e.message} #{e.backtrace.join("\n")}"
               end
             end
-            @verbose = verbose
-            raise RsyncError if !done
+            #@verbose = verbose
+            if done
+              calc
+              save_repos
+              @logger.info "Verifying Unpacked files..."
+              compare_set(:wd)
+              # Bump version and make final save
+              @repos_local[:version] = @repos_remote[:version]
+              save_repos
+            else
+              @logger.warn "Exhausted all mirrors, please retry!"
+              raise RsyncError
+            end
           else
             #reset(:hard => true)
             calc
@@ -375,16 +369,17 @@ module Six
 
           if online
             hosts = config[:hosts].clone
-            b = false
+            b, i = false, 0
             verbose = @verbose
             @verbose = false
 
             until hosts.empty? || done do
+              i += 1
               # FIXME: Nasty
               host = hosts.sample if b
               b = true
               hosts -= [host]
-              @logger.info "Trying #{host}"
+              @logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
 
               begin
                 FileUtils.cp(pack_path(".repository.yml"), rsync_path(".repository-pack.yml")) if File.exists?(pack_path(".repository.yml"))
@@ -397,8 +392,7 @@ module Six
                 end
                 done = true
               rescue => e
-                @logger.warn "#{e.class}: #{e.message}"
-                @logger.debug e.backtrace.join("\n")
+                @logger.debug "#{e.class} #{e.message}: #{e.backtrace.join("\n")}"
                 FileUtils.cp(rsync_path(".repository-pack.yml"), pack_path(".repository.yml")) if File.exists?(rsync_path(".repository-pack.yml"))
               ensure
                 FileUtils.rm(rsync_path(".repository-pack.yml")) if File.exists?(rsync_path(".repository-pack.yml"))
@@ -422,7 +416,7 @@ module Six
           end
         end
 
-        def compare_set(typ, host, online = true)
+        def compare_set(typ, host = nil, online = true)
           #if local[typ][:md5] == remote[typ][:md5]
           #  @logger.info "#{typ} Match!"
           #else
@@ -442,56 +436,57 @@ module Six
             case typ
             when :pack
               # direct unpack of gz into working folder
-              # Update file
+              done = false
+
+              ## Pack
               if online
                 hosts = config[:hosts].clone
-                done = false
-
-                ## Pack
-                if online
-                  b = false
-                  until hosts.empty? || done do
-                    # FIXME: Nasty
-                    if b
-                      host = hosts.sample
-                      @logger.info "Trying #{host}"
-                    end
-                    slist = nil
-                    b = true
-                    hosts -= [host]
-
-                    # TODO: Progress bar
-                    arr_opts = []
-                    arr_opts << PARAMS
-                    arr_opts << @rsh if host[/^(\w)*\@/]
-
-                    if mismatch.size > (@repos_remote[typ].size / 2)
-                      # Process full folder
-                      @logger.info "Many files mismatched (#{mismatch.size}), running full update on .pack folder"
-                    else
-                      # Process only selective
-                      @logger.info "Fetching #{mismatch.size} files... Please wait"
-                      slist = File.join(TEMP_PATH, ".six-rsync_#{rand 9999}-list")
-                      slist.gsub!("\\", "/")
-                      File.open(slist, 'w') { |f| mismatch.each { |e| f.puts e } }
-
-                      arr_opts << "--files-from=#{win2cyg("\"#{slist}\"")}"
-                    end
-
-                    begin
-                      arr_opts << esc(File.join(host, '.pack/.'))
-                      arr_opts << esc(pack_path)
-                      command('', arr_opts)
-
-                      done = true
-                    rescue => e
-                      @logger.warn "Failure"
-                      @logger.debug "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
-                    ensure
-                      FileUtils.rm_f slist if slist
-                    end
+                host = hosts.sample unless host
+                b, i = false, 0
+                until hosts.empty? || done do
+                  i += 1
+                  # FIXME: Nasty
+                  if b
+                    host = hosts.sample
+                    @logger.info "Trying #{i}/#{config[:hosts].size}: #{host}"
+                  end
+                  slist = nil
+                  b = true
+                  hosts -= [host]
+
+                  # TODO: Progress bar
+                  arr_opts = []
+                  arr_opts << PARAMS
+                  arr_opts << @rsh if host[/^(\w)*\@/]
+
+                  if mismatch.size > (@repos_remote[typ].size / 2)
+                    # Process full folder
+                    @logger.info "Many files mismatched (#{mismatch.size}), running full update on .pack folder"
+                  else
+                    # Process only selective
+                    @logger.info "Fetching #{mismatch.size} files... Please wait"
+                    slist = File.join(TEMP_PATH, ".six-rsync_#{rand 9999}-list")
+                    slist.gsub!("\\", "/")
+                    File.open(slist, 'w') { |f| mismatch.each { |e| f.puts e } }
+
+                    arr_opts << "--files-from=#{win2cyg("\"#{slist}\"")}"
                   end
-                  @logger.warn "There was a problem during updating, please retry!" unless done
+
+                  begin
+                    arr_opts << esc(File.join(host, '.pack/.'))
+                    arr_opts << esc(pack_path)
+                    command('', arr_opts)
+
+                    done = true
+                  rescue => e
+                    @logger.debug "ERROR: #{e.class} #{e.message} #{e.backtrace.join("\n")}"
+                  ensure
+                    FileUtils.rm_f slist if slist
+                  end
+                end
+                unless done
+                  @logger.warn "Exhausted all mirrors, please retry!"
+                  raise RsyncError
                 end
               end
             when :wd
@@ -567,10 +562,7 @@ module Six
         end
 
         def fetch_file(path, host)
-          path[/(.*)\/(.*)/]
-          folder, file = $1, $2
-          folder = "." unless folder
-          file = path unless file
+          folder = File.dirname(path)
           # Only fetch a specific file
           @logger.debug "Fetching #{path} from  #{host}"
           arr_opts = []
@@ -721,9 +713,7 @@ module Six
             # TODO: Evaluate if this is actually wanted / useful at all..
 =begin
             if file[/\.tar\.?/]
-              file[/(.*)\/(.*)/]
-              fil = $2
-              fil = file unless fil
+              fil = File.basename(file)
               f2 = fil.gsub('.gz', '')
               zip7(f2)
               FileUtils.rm_f f2
@@ -744,9 +734,8 @@ module Six
               relative = file.clone
               relative.gsub!(@rsync_work_dir, '')
               relative.gsub!(/^[\\|\/]\.rsync[\\|\/]\.pack[\\|\/]/, '')
-              fil = relative
-              folder = "."
-              folder, fil = $1, $2 if relative[/(.*)\/(.*)/]
+              folder = File.dirname(relative)
+              fil = File.basename(relative)
               #puts "Relative: #{relative}, Folder: #{folder}, File: #{fil} (Origin: #{file})"
 
               path = File.join(@rsync_work_dir, folder)
@@ -767,7 +756,7 @@ module Six
           opts << "--timeout=#{DEFAULT_TIMEOUT}"
 
           opts = [opts].flatten.map {|s| s }.join(' ') # escape()
-          rsync_cmd = win2cyg("rsync #{cmd} #{opts} #{redirect} 2>&1")
+          rsync_cmd = win2cyg("rsync #{cmd} #{opts} #{redirect}") # 2>&1
 
           @logger.debug(rsync_cmd) if @logger
 
@@ -776,51 +765,48 @@ module Six
         end
 
         def run_command(rsync_cmd, &block)
-          # TODO: Make this switchable? Verbosity ?
-          # Or actually parse this live for own stats?
-          s = nil
-          out = ''
-
-          # Simpler method but on windows the !? exitstatus is not working properly..
-          # Does nicely display error output in logwindow though
-          io = IO.popen(rsync_cmd)
-          io.sync = true
+          out, err = '', ''
+          buff = []
+          status = nil
           oldsync = STDOUT.sync
           STDOUT.sync = true
 
-          #io.each do |buffer|
-          #  process_msg buffer
-          #  out << buffer
-          #end
+          po = Open3.popen3(rsync_cmd) do |io_in, io_out, io_err, waitth|
+            io_out.each_byte do |buffer|
+              char = buffer.chr
+              buff << char
+              if ["\n", "\r"].include?(char)
+                b = buff.join("")
+                print b if @verbose                
+                out << b
+                buff = []
+              end
+            end
 
-          buff = []
-          io.each_byte do |buffer|
-            buffer = buffer.chr
-            buff << buffer
-            if ["\n", "\r"].include?(buffer)
-              b = buff.join("")
-              process_msg b
-              out << b
-              buff = []
+            io_err.each do |line|
+              print line
+              case line
+              when /max connections \((.*)\) reached/
+                @logger.warn "Server reached maximum connections."
+              end
+              err << line
             end
+            status = waitth.value
           end
 
           unless buff.empty?
             b = buff.join("")
-            process_msg b
+            print b if @verbose
             out << b
-          end        
+          end
 
-          out[/rsync error: .* \(code ([0-9]*)\)/]
-          status = $1 ? $1.to_i : 0
+          @logger.debug "Status: #{status}"
+          @logger.debug "Err: #{err}" # TODO: Throw this into the info/error log?
+          @logger.debug "Output: #{out}"
 
-          if status > 0
-            return '' if status == 1 && out == ''
-            case out
-              when /max connections \((.*)\) reached/ 
-                @logger.warn "Server reached maximum connections."
-            end
-            raise Rsync::RsyncExecuteError.new(rsync_cmd + ':' + out)
+          if status.exitstatus > 0
+            #return 0 if status.exitstatus == 1 && out == ''
+            raise Rsync::RsyncExecuteError.new(rsync_cmd + ':' + err + ':' + out)
           end
 
           STDOUT.sync = false unless STDOUT.sync == oldsync
@@ -828,17 +814,23 @@ module Six
           status
         end
 
-        def process_msg(msg)
-          if msg[/\r/] #msg[/[k|m|g]?B\/s/i]
-            #msg.gsub!("\n", '')
-            print msg if @verbose
-          else
-            @logger.debug msg
-            print msg if @verbose
-          end
-          msg
-        end
+=begin
+          # Simpler method but on windows the !? exitstatus is not working properly..
+          # Does nicely display error output in logwindow though
+          io = IO.popen(rsync_cmd)
+          io.sync = true
 
+          #io.each do |buffer|
+          #  process_msg buffer
+          #  out << buffer
+          #end
+          out[/rsync error: .* \(code ([0-9]*)\)/]
+          status = $1 ? $1.to_i : 0
+            case out
+              when /max connections \((.*)\) reached/
+                @logger.warn "Server reached maximum connections."
+            end
+=end
       end
     end
   end

data/lib/six/rsync-app.rb

@@ -90,7 +90,7 @@ module Six
         if options[:logging]
           o_file = Log4r::FileOutputter.new "#{COMPONENT}-file",
                                             'level' => 0, # All
-                                            :filename => "#{COMPONENT}.log",
+                                            :filename => File.join(DATA_PATH, 'logs', "#{COMPONENT}-file"),
                                             'formatter' =>  format2
           #:maxsize => 1024
           @@log.outputters << o_file
@@ -101,7 +101,7 @@ module Six
                                            'formatter' =>  format1
 
         o_err = Log4r::StderrOutputter.new "#{COMPONENT}-stderr",
-                                           'level' => 4, # Error and Up
+                                           'level' => 5, # Error and Up
                                            'formatter' =>  format1
 
         @@log.outputters << o_out << o_err

data/lib/six/rsync.rb

@@ -3,17 +3,13 @@
 require 'fileutils'
 require 'digest/md5'
 require 'yaml'
+require 'open3'
 
-BASE_PATH = Dir.pwd
 require 'six/rsync/path'
 require 'six/rsync/repository'
 require 'six/rsync/working_directory'
 require 'six/rsync/lib'
 require 'six/rsync/base'
-#require 'six/popen'
-
-require 'open3'
-#require 'win32/open3'
 
 case RUBY_VERSION
 when /1\.8\.[0-9]/
@@ -31,8 +27,8 @@ module Six
 
     module Rsync
       COMPONENT = 'six-rsync'
-      VERSION = '0.6.0'
-      FOLDER = /(.*)\/(.*)/
+      VERSION = '0.6.1'
+      BASE_PATH = Dir.pwd      
 
       case RUBY_PLATFORM
         when /-mingw32$/, /-mswin32$/

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 6
-  - 0
-  version: 0.6.0
+  - 1
+  version: 0.6.1
 platform: ruby
 authors: 
 - Sickboy
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-07 00:00:00 +02:00
+date: 2010-05-08 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency