rubyyabt 0.0.5 → 0.1.6

data/bin/rubyyabt-backup.rb

@@ -36,6 +36,7 @@ OptionParser.new do |opts|
   opts.on("-g", "--gpg-key KEY", "Use KEY as GPG key for encryption and signing") { |v| options[:gpg_key] = v }
   opts.on("-h", "--gpg-pass PASS", "Password for GPG key") { |v| options[:gpg_pass] = v }
   opts.on("-H", "--gpg-passfile FILE", "Read password for GPG key from FILE") { |v| options[:gpg_pass] = File.read(v) }
+  opts.on("-l", "--log-file FILE", "Write log to FILE") { |v| options[:logfile] = v.to_s }
   opts.on("-c", "--cache-max-age DAYS", "Allow maximum of DAYS time for a positive cache hit. False hits are never cached.") { |v| options[:cache_max_age] = v.to_i *(24*60*60) }
   opts.on("",   "--http-timeout SECS", "Require all http requests to complete within SECS seconds.") { |v| options[:http_timeout] = v.to_i }
   opts.on("-C", "--empty-cache", "Start with an empty cache instead of restoring the cache from saved data") { options[:empty_cache] = true }

data/classes/Backup.rb

@@ -32,20 +32,20 @@ class Backup
 	@time = Time.now
 	@time.utc
 	@name = @time.strftime("%Y%m%dT%H%M%S") unless @name
-	@cui.message("Initialized backup #{@name}")
+	@cui.message("Initialized backup #{@name}", true, true)
   end
 
   def backup!()
 	$errors = 0 if not $errors
-	@cui.message("Scanning for files...")
+	@cui.message("Scanning for files...", true, true)
 	@cui.start
-	@cui.message("Found #{@source.files.count.to_s} files and directories")
+	@cui.message("Found #{@source.files.count.to_s} files and directories", true, true)
 	backup_meta = Array.new
 	backup_meta << '[Backup]'
 	cache_timer = Time.now
 	@source.files.each { | f |
 	  if (Time.now - cache_timer) > 900 then
-		@cui.message("Uploading cache data...")
+		@cui.message("Uploading cache data...", true, true)
 		@target.upload_cache
 		cache_timer = Time.now
 	  end
@@ -65,19 +65,19 @@ class Backup
 		$errors += 1
 	  end
 	}
-	@cui.message("Waiting for last chunk to finish uploading...")
+	@cui.message("Waiting for last chunk to finish uploading...", true, true)
 	@cui.error("DEBUG[#{Thread.current.inspect}]: Joining chunk threads") if $myDEBUG
 	chunk = Chunk.new
 	chunk.join
 	@cui.stop
 	@cui.update
-	@cui.message("Finished backing up data. Now uploading metadata for backup #{@name}.")
+	@cui.message("Finished backing up data. Now uploading metadata for backup #{@name}.", true, true)
 	backup_meta = backup_meta.join("\n")
 	@cui.error("DEBUG[#{Thread.current.inspect}]: Writing backup info") if $myDEBUG
 	@target.write(:backup, @name, backup_meta)
-	@cui.message("Uploading cache data...")
+	@cui.message("Uploading cache data...", true, true)
 	@target.upload_cache
-	@cui.error("Errors: A total of #{$errors} errors have been encountered.") if $errors > 0
+	@cui.error("Errors: A total of #{$errors} errors have been encountered.", true, true) if $errors > 0
 	return $errors
   end
 

data/classes/Chunk.rb

@@ -10,8 +10,6 @@ require 'classes/Target'
 #noinspection RubyResolve
 require 'classes/cui'
 
-$myDEBUG = false unless $myDEBUG
-
 class Chunk
   attr_reader :data, :length
   @@upload_mutex = Mutex.new
@@ -27,16 +25,12 @@ class Chunk
   end
 
   def join()
-    @cui.error("DEBUG[#{Thread.current.inspect}]: Joining chunk thread") if $myDEBUG
     @@upload_thread.join if @@upload_thread
   end
 
   def set_data(data)
-    @cui.error("DEBUG[#{Thread.current.inspect}: Entered chunk set_data") if $myDEBUG
     @data = data
-    @cui.error("DEBUG[#{Thread.current.inspect}: Set data") if $myDEBUG
     @length = @data.length
-    @cui.error("DEBUG[#{Thread.current.inspect}: Calculated length") if $myDEBUG
     @md5 = nil
     @md5 = Digest::MD5.hexdigest(@data)
     @sha256 = nil
@@ -64,26 +58,20 @@ class Chunk
   end
 
   def backup!()
-    @cui.error("DEBUG[#{Thread.current.inspect}]: Locking chunk upload mutex in thread #{Thread.current.inspect}") if $myDEBUG
     @@upload_mutex.synchronize {
-      @cui.error("DEBUG[#{Thread.current.inspect}]: Got lock of chunk upload mutex in thread #{Thread.current.inspect}") if $myDEBUG
-      @cui.error("DEBUG[#{Thread.current.inspect}]: Trying to join earlier upload threads if any (@@uploadThread: #{@@upload_thread.inspect})") if $myDEBUG
       @@upload_thread.join if @@upload_thread
       @@upload_thread = Thread.new {
         target = Target.instance # Get a target object, then ...
-        @cui.error("DEBUG[#{Thread.current.inspect}]: Checking if target chunk exists (#{sha256}, #{md5})") if $myDEBUG
         if not target.exists?(:chunk, "#{sha256}.#{md5}") then
-          @cui.error("Uploading chunk #{sha256}.#{md5}...") if $myDEBUG
           target.write(:chunk, "#{sha256}.#{md5}", @data)
+		  @cui.message("Chunk #{sha256}.#{md5} (#{@data.length} bytes) uploaded.", true, false)
         end
-        @cui.error("DEBUG[#{Thread.current.inspect}]: Upload chunk thread finished (thread id: #{Thread.current.inspect})") if $myDEBUG
       }
-      @cui.error("DEBUG[#{Thread.current.inspect}]: New upload thread: @@upload_thread = #{@@upload_thread.inspect}") if $myDEBUG
     }
   end
 
   def restore!()
-    @cui.message("Downloading chunk #{sha256}.#{md5}...")
+    @cui.message("Downloading chunk #{sha256}.#{md5}...", true, true)
     target = Target.instance
     @data = target.read(:chunk, "#{sha256}.#{md5}")
     @length = @data.length
@@ -92,9 +80,9 @@ class Chunk
     @md5 = nil
     @sha256 = nil
     if (expected_md5 == md5) and (expected_sha256 == sha256) then
-      @cui.message("chunk verified")
+      @cui.message("Chunk verified", true, true)
     else
-      @cui.message("checksum error in chunk #{sha256}.#{md5}")
+      @cui.error("checksum error in chunk #{sha256}.#{md5}", true, true)
       raise 'checksum error'
     end
   end

data/classes/ProxyHTTP.rb

@@ -8,8 +8,6 @@ require 'thread'
 #noinspection RubyResolve
 require 'classes/proxy_http_cache_hash'
 
-$myDEBUG = false if not $myDEBUG
-$myVERBOSE = false if not $myVERBOSE
 $options = nil unless $options
 
 class ProxyHTTP
@@ -60,48 +58,43 @@ class ProxyHTTP
 		@cui.error("Cache is not compatible. Starting with an empty cache.")
 		@cache = ProxyHTTPCache_Hash.new
 	  end
+	  @cui.message("Successfully restored cache with a size of #{cache_data.length} bytes.", true, true)
 	rescue Exception # Just start out with an empty cache
 	  @cache = ProxyHTTPCache_Hash.new
+	  @cui.message("Unable to restore cache. Starting with an empty cache.", true, true)
 	end
 
   end
 
   #noinspection RubyScope
   def request(http_request, timeout = 300, tries = 20)
-	@cui.error("DEBUG[#{Thread.current.inspect}]: Trying to lock ProxyHTTP request mutex in thread #{Thread.current.inspect}") if $myDEBUG
 	@mutex.synchronize {
-	  @cui.error("DEBUG[#{Thread.current.inspect}]: Got ProxyHTTP request mutex in thread #{Thread.current.inspect}") if $myDEBUG
 	  response = ''
 	  while tries > 0 do
-		@cui.error("DEBUG[#{Thread.current.inspect}]: Got #{tries} tries left for request #{http_request.inspect} #{http_request.path}, timeout #{timeout}") if $myDEBUG
 		begin
 		  Timeout::timeout(timeout) {
-			@cui.error("DEBUG[#{Thread.current.inspect}]: Sending request #{http_request.inspect} for #{http_request.path}....") if $myDEBUG
 			response = @http.request(http_request)
 		  }
 		rescue Timeout::Error
 		  tries -= 1
-		  @cui.error("Timeout during request. #{tries} tries left")
+		  @cui.error("Timeout during request. #{tries} tries left", true, true)
 		  retry if tries > 0
 		  raise
 		rescue Exception => ex
-		  @cui.error("Caught exception #{ex} with message #{ex.message} during request. #{tries} tries left") if $myVERBOSE
+		  @cui.error("Caught exception #{ex} with message #{ex.message} during request. #{tries} tries left", true, false)
 		  tries -= 1
 		  retry if tries > 0
 		  raise
 		end
 		code = response.code.to_i
-		@cui.error("DEBUG[#{Thread.current.inspect}]: Response code: #{code}") if $myDEBUG
 		return response if (code >= 200) and (code < 300)
 		if (code >= 400) and (code < 500) # No retries for these errors...
 		  # Yeehaw! we need an exception for code 401 since humyo sometimes replies with it with correct authorization
 		  tries = 0 if (code != 401)
 		end
 		tries -= 1
-		@cui.error("DEBUG[#{Thread.current.inspect}]: #{tries} tries left for this request. Retrying after sleep if > 0") if $myDEBUG
 		sleep(rand) if tries > 0 # Sleep up to one second before retrying
 	  end
-	  @cui.error("DEBUG[#{Thread.current.inspect}]: Raising exception for failed request: #{response.code} #{response.message}") if $myDEBUG
 	  raise 'HTTP request failed: ' + response.code + ' ' + response.message
 	}
   end
@@ -118,7 +111,6 @@ class ProxyHTTP
 	get = Net::HTTP::Get.new(uri.request_uri)
 	get.initialize_http_header({"User-Agent" => $options[:user_agent]})
 	get.basic_auth($options[:username], $options[:password])
-	@cui.error("DEBUG[#{Thread.current.inspect}]: Reading from #{uri.to_s}") if $myDEBUG
 	begin
 	  response = request(get, $options[:http_timeout])
 	rescue Exception => ex
@@ -142,22 +134,19 @@ class ProxyHTTP
 	put.initialize_http_header({"User-Agent" => $options[:user_agent], "Content-Type" => "application/octet-stream"})
 	put.basic_auth($options[:username], $options[:password])
 	put.body = data
-	@cui.error("DEBUG[#{Thread.current.inspect}]: Writing to #{uri.to_s}") if $myDEBUG
 	begin
 	  response = request(put, 180)
 	rescue RuntimeError => ex
 	  if ex.message == "HTTP request failed: 409 Conflict" then
-		@cui.error("DEBUG[#{Thread.current.inspect}]: Caught a 409 Conflict error") if $myDEBUG
 		if (subdir_type != type) then
 		  begin
-			@cui.error("DEBUG[#{Thread.current.inspect}]: Trying to create the directory #{@url.merge(@types[type] + '/').to_s}") if $myDEBUG
 			mkdir(@url.merge(@types[type] + '/'))
 		  rescue RuntimeError => rt
 			@cui.error("DEBUG[#{Thread.current.inspect}]: Caught exception during mkdir: #{rt.message}") if $myDEBUG
 			raise if rt.message[0..23] != "HTTP request failed: 405"
 		  end
 		end
-		@cui.error("DEBUG[#{Thread.current.inspect}]: Trying to create the directory #{@url.merge(@types[type] + '/').merge(subdir).to_s}") if $myDEBUG
+		@cui.message("Trying to create the directory #{@url.merge(@types[type] + '/').merge(subdir).to_s}", true, false)
 		mkdir(@url.merge(@types[type] + '/').merge(subdir))
 	  end
 	  retry
@@ -166,7 +155,6 @@ class ProxyHTTP
 	  raise
 	else
 	  @cache.add(type, file) if @@valid_caches.include?(type)
-	  @cui.error("DEBUG[#{Thread.current.inspect}]: Added #{file} to cache list for #{subdir_type}") if $myDEBUG
 	  return true
 	end
   end
@@ -175,7 +163,7 @@ class ProxyHTTP
 	mkcol = Net::HTTP::Mkcol.new(uri.request_uri)
 	mkcol.initialize_http_header({"User-Agent" => $options[:user_agent]})
 	mkcol.basic_auth($options[:username], $options[:password])
-	@cui.error("DEBUG[#{Thread.current.inspect}]: Trying to create directory #{uri.to_s}") if $myDEBUG
+	@cui.message("Trying to create directory #{uri.to_s}", true, false)
 	begin
 	  request(mkcol)
 	rescue Exception => ex

data/classes/SMGFile.rb

@@ -11,8 +11,6 @@ require 'digest/md5'
 require 'digest/sha2'
 require 'time'
 
-$myDEBUG = false if not $myDEBUG
-$myVERBOSE = false if not $myVERBOSE
 $options = nil unless $options
 
 class SMGFile
@@ -69,33 +67,23 @@ class SMGFile
     size = 0
     md5hashing = Digest::MD5.new
     sha256hashing = Digest::SHA2.new(256)
-    @cui.message("DEBUG[#{Thread.current.inspect}: Created hashing objects") if $myDEBUG
     chunks = ''
+	gc_counter = -1 # Initialize a counter for the garbage collection
     while !fd.eof?
-      @cui.message("DEBUG[#{Thread.current.inspect}: Checked for EOF") if $myDEBUG
+	  GC.start if ((gc_counter += 1) % 30) == 0 # Run the garbage collection every now and then
       chunk = Chunk.new
-      @cui.message("DEBUG[#{Thread.current.inspect}: Created new chunk") if $myDEBUG
       data = fd.read($options[:chunk_size])
-      @cui.message("DEBUG[#{Thread.current.inspect}: Read data") if $myDEBUG
       chunk.set_data(data)
-      @cui.message("DEBUG[#{Thread.current.inspect}: Saved data in chunk") if $myDEBUG
       md5hashing << data
-      @cui.message("DEBUG[#{Thread.current.inspect}: hashed for MD5") if $myDEBUG
       sha256hashing << data
-      @cui.message("DEBUG[#{Thread.current.inspect}: hashed for sha256") if $myDEBUG
       size += data.length
-      @cui.message("DEBUG[#{Thread.current.inspect}: Updated read size") if $myDEBUG
       chunk.backup! # Upload this chunk
-      @cui.message("DEBUG[#{Thread.current.inspect}: backed up chunk") if $myDEBUG
       chunks += chunk.sha256 + '.' + chunk.md5 + "\n"
       @cui.finished_size_add(data.length)
     end
     fd.close()
-    @cui.message("DEBUG[#{Thread.current.inspect}: Closed FD") if $myDEBUG
     md5 = md5hashing.hexdigest
-    @cui.message("DEBUG[#{Thread.current.inspect}: Got MD5") if $myDEBUG
     sha256 = sha256hashing.hexdigest
-    @cui.message("DEBUG[#{Thread.current.inspect}: Got sha256") if $myDEBUG
     # Build the metadata file
     @metadata = "[stat]\n"
     @metadata += "filename = " + @filename + "\n"
@@ -112,14 +100,12 @@ class SMGFile
     @metadata += "\n"
     @metadata += "[chunks]\n"
     @metadata += chunks
-    @cui.message("Size mismatch: expected #{expected_size} but found #{size}!") if expected_size != size
+    @cui.message("Size mismatch: expected #{expected_size} but found #{size}!", true, true) if expected_size != size
     @size = size
     @mtime = mtime
-    @cui.message("DEBUG[#{Thread.current.inspect}: Checking if file already exists on server") if $myDEBUG
     return true if @target.exists?(:file, "#{meta_sha256}.#{meta_md5}") # if this chunk already exists, no need to upload it
-    @cui.message("Uploading file metadata #{@filename}: #{meta_sha256}.#{meta_md5}...") if $myDEBUG
+    @cui.message("Uploading file metadata #{@filename}: #{meta_sha256}.#{meta_md5}...", true, true)
     @target.write(:file, "#{meta_sha256}.#{meta_md5}", @metadata)
-    @cui.message("DEBUG[#{Thread.current.inspect}: Uploaded file metadata") if $myDEBUG
   end
 
   def backup_dir()
@@ -141,12 +127,11 @@ class SMGFile
     @size = 0
     @mtime = mtime
     return true if @target.exists?(:file, "#{meta_sha256}.#{meta_md5}") # if this chunk already exists, no need to upload it
-    @cui.message("Uploading directory metadata #{@filename}: #{meta_sha256}.#{meta_md5}...") if $myVERBOSE
+    @cui.message("Uploading directory metadata #{@filename}: #{meta_sha256}.#{meta_md5}...", true, true)
     @target.write(:file, "#{meta_sha256}.#{meta_md5}", @metadata)
   end
 
   def backup_link()
-    @cui.message("Backing up link #{@filename}") if $myVERBOSE
     stat = @source.lstat(@filename)
     ftype = stat.ftype
     mtime = stat.mtime.utc.rfc2822
@@ -164,7 +149,7 @@ class SMGFile
     @size = 0
     @mtime = mtime
     return true if @target.exists?(:file, "#{meta_sha256}.#{meta_md5}") # if this chunk already exists, no need to upload it
-    @cui.message("Uploading file metadata #{@filename}: #{meta_sha256}.#{meta_md5}...") if $myVERBOSE
+    @cui.message("Uploading file metadata #{@filename}: #{meta_sha256}.#{meta_md5}...", true, true)
     @target.write(:file, "#{meta_sha256}.#{meta_md5}", @metadata)
   end
 
@@ -218,9 +203,9 @@ class SMGFile
     md5 = md5hashing.hexdigest
     sha256 = sha256hashing.hexdigest
     if (@md5 = md5) and (@sha256 = sha256) then
-      @cui.message("File verified")
+      @cui.message("File verified", true, true)
     else
-      @cui.error("Checksum error in file #{@filename}: #{sha256}.#{md5}")
+      @cui.error("Checksum error in file #{@filename}: #{sha256}.#{md5}", true, true)
     end
   end
 

data/classes/Target.rb

@@ -43,16 +43,15 @@ class Target
 	  else
 		raise "No idea how to connect to #{$options[:target]}"
 	end
-	@cui.message("Initialized proxy as #{@proxy.class}.") if $myVERBOSE
+	@cui.message("Initialized proxy as #{@proxy.class}.", true, true)
 	begin
 	  if @proxy.caching? then
 		if not $options[:empty_cache] then
-		  @cui.message("Trying to restore cache...") if $myVERBOSE
+		  @cui.message('Trying to restore cache...', true, false)
 		  compressed_cache_data = read(:cache, "flist")
 		  begin
 			# Check if the data is compressed and uncompress the data
 			cache_data = Zlib::Inflate.inflate(compressed_cache_data)
-			@cui.message("Uncompressed cache data from #{compressed_cache_data.length} bytes to #{cache_data.length} bytes.")
 		  rescue Exception
 			# Uncompress failed, so the data was probably uncompressed from the beginning.
 			cache_data = compressed_cache_data
@@ -102,15 +101,15 @@ class Target
 	  rescue Exception
 		# Seems like compression does not work. We'll just use the uncompressed data then.
 		compressed_cache_data = cache_data
-		@cui.message('Unable to compress the cache data for upload. Using uncompressed data.')
+		@cui.message('Unable to compress the cache data for upload. Using uncompressed data.', true, false)
 	  end
-	  @cui.error("DEBUG[#{Thread.current.inspect}]: Cache has a length of #{compressed_cache_data.length}. Uploading...") if $myDEBUG
+	  @cui.message("Cache has a length of #{compressed_cache_data.length}. Uploading...", true, false)
 	  write(:cache, 'flist', compressed_cache_data)
 	end
   end
 
   def export_key()
-	@cui.error("DEBUG[#{Thread.current.inspect}]: Exporting keys...") if $myDEBUG
+	@cui.message("Exporting keys...", true, true)
 	IO.popen('"' + $options[:gpg_bin] + '" ' + '--export-ownertrust', 'w+') do | gpg |
 	  gpg.close_write()
 	  trust = gpg.read()

data/classes/cui.rb

@@ -2,6 +2,7 @@ require 'singleton'
 #noinspection RubyResolve
 require 'thread'
 
+$options = nil unless $options
 
 # Usage of stdout and stderr:
 # stdout will be used for normal status messages like the status bar.
@@ -29,6 +30,7 @@ class Cui
 	@changed = false
 	@active = false
 	@exists = true
+	@logfile = nil
   end
   def exists(e)
 	@exists = e
@@ -66,6 +68,7 @@ class Cui
 	  @last_file_name = @current_file_name
 	  @current_file_name = file.to_str
 	  update
+	  write_log("File: #{file.to_s} (#{@finished_files + 1}/#{@total_files}; #{(@finished_size/1024/1024).to_i.to_s} MB/#{(@total_size/1024/1024).to_i.to_s} MB)")
 	}
   end
   def start()
@@ -108,34 +111,40 @@ class Cui
 	$stdout.print(backspace + line)
 	@changed = false
   end
-  def message(text)
+  def message(text, logging = false, write_stdout = true)
 	@mutex.synchronize {
-	  case @active
-		when true
-		  backspace = ("\b" * @last_length)
-		  text += " " * (@last_length - text.length) if (@last_length - text.length) > 0
-		  @last_length = 0
-		  $stdout.print(backspace + text + "\n")
-		  update
-		when false
-		  # If there's no regular update, we can just write out a new line
-		  $stdout.print("\n" + text + "\n")
+	  write_log(text) if logging
+	  if write_stdout then
+		case @active
+		  when true
+			backspace = ("\b" * @last_length)
+			text += " " * (@last_length - text.length) if (@last_length - text.length) > 0
+			@last_length = 0
+			$stdout.print(backspace + text + "\n")
+			update
+		  when false
+			# If there's no regular update, we can just write out a new line
+			$stdout.print("\n" + text + "\n")
+		end
 	  end
 	}
   end
-  def error(text)
+  def error(text, logging = true, write_stderr = true)
 	@mutex.synchronize {
-	  case @active
-		when true
-		  backspace = ("\b" * @last_length) # Calculate the amount of backspaces required
-		  spaces = ""
-		  spaces = " " * (@last_length - text.length) if (@last_length - text.length) > 0
-		  $stderr.print(backspace + text + spaces + "\n")
-		  @last_length = 0
-		  update
-		when false
-		  # If there's no regular update, we can just write out a new line
-		  $stdout.print("\n" + text + "\n")
+	  write_log(text) if logging
+	  if write_stderr then
+		case @active
+		  when true
+			backspace = ("\b" * @last_length) # Calculate the amount of backspaces required
+			spaces = ""
+			spaces = " " * (@last_length - text.length) if (@last_length - text.length) > 0
+			$stderr.print(backspace + text + spaces + "\n")
+			@last_length = 0
+			update
+		  when false
+			# If there's no regular update, we can just write out a new line
+			$stdout.print("\n" + text + "\n")
+		end
 	  end
 	}
   end
@@ -145,4 +154,12 @@ class Cui
 	@update_thread.join
 	@update_thread = nil
   end
+  def write_log(text)
+	return if $options[:logfile].nil?
+	@logfile = File.new($options[:logfile], "w:utf-8") if @logfile.nil?
+	date_time = Time.now.strftime('%Y-%m-%d %H:%M:%S')
+	text = date_time + " " + text + "\n"
+	@logfile.write(text)
+	@logfile.flush
+  end
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 0
-  - 5
-  version: 0.0.5
+  - 1
+  - 6
+  version: 0.1.6
 platform: ruby
 authors: 
 - Daniel Frank