rubyyabt 0.0.5

data/classes/Target.rb

@@ -0,0 +1,130 @@
+#!/usr/bin/env ruby
+
+require 'singleton'
+require 'uri'
+require 'zlib'
+
+#noinspection RubyResolve
+require 'classes/ProxyFile'
+#noinspection RubyResolve
+require 'classes/ProxyHTTP'
+#noinspection RubyResolve
+require 'classes/GPG'
+#noinspection RubyResolve
+require 'classes/cui'
+
+$myVERBOSE = false unless $myVERBOSE
+$myDEBUG = false unless $myDEBUG
+
+$options = nil unless $options
+
+class Target
+  include Singleton
+
+  def initialize()
+	@cui = Cui.instance
+	connect
+	if $options[:mode] == :backup then
+	  # Export the key only if a backup is in progress.
+	  export_key unless exists?(:root, 'key')
+	  export_key unless exists?(:root, 'trust')
+	end
+  end
+
+  def connect()
+	@proxy = ProxyFile.new() if !($options[:target].include?(":"))
+	case URI.parse($options[:target]).scheme
+	  when 'file'
+		@proxy = ProxyFile.new()
+	  when 'http'
+		@proxy = ProxyHTTP.new()
+	  when 'https'
+		@proxy = ProxyHTTP.new()
+	  else
+		raise "No idea how to connect to #{$options[:target]}"
+	end
+	@cui.message("Initialized proxy as #{@proxy.class}.") if $myVERBOSE
+	begin
+	  if @proxy.caching? then
+		if not $options[:empty_cache] then
+		  @cui.message("Trying to restore cache...") if $myVERBOSE
+		  compressed_cache_data = read(:cache, "flist")
+		  begin
+			# Check if the data is compressed and uncompress the data
+			cache_data = Zlib::Inflate.inflate(compressed_cache_data)
+			@cui.message("Uncompressed cache data from #{compressed_cache_data.length} bytes to #{cache_data.length} bytes.")
+		  rescue Exception
+			# Uncompress failed, so the data was probably uncompressed from the beginning.
+			cache_data = compressed_cache_data
+		  end
+		  #noinspection RubyUnusedLocalVariable
+		  compressed_cache_data = nil # Free up the memory
+		  @proxy.load_cache(cache_data)
+		end
+	  end
+	rescue Exception
+	  # Nothing to do here.
+	end
+	GC.start
+  end
+
+  def connected?()
+	if @proxy then
+	  true
+	else
+	  false
+	end
+  end
+
+  def exists?(type, file)
+	@cui.exists(@proxy.exists?($options[:target], type, file + '.gpg'))
+  end
+
+  def read(type, file)
+	gpg = GPG.new()
+	gpg.decrypt(@proxy.read($options[:target], type, file + '.gpg'))
+  end
+
+  def write(type, file, data)
+	gpg = GPG.new()
+	encrypted_data = gpg.encrypt(data)
+	@proxy.write($options[:target], type, file + '.gpg', encrypted_data)
+  end
+
+  def upload_cache()
+	# Upload the cache to save time for the next runs...
+	if @proxy.caching? then
+	  cache_data = @proxy.dump_cache
+	  return if cache_data.nil? # Skip dumping if the data has not changed
+	  # Compress the data to save some space...
+	  begin
+		compressed_cache_data = Zlib::Deflate.deflate(cache_data, 9)
+	  rescue Exception
+		# Seems like compression does not work. We'll just use the uncompressed data then.
+		compressed_cache_data = cache_data
+		@cui.message('Unable to compress the cache data for upload. Using uncompressed data.')
+	  end
+	  @cui.error("DEBUG[#{Thread.current.inspect}]: Cache has a length of #{compressed_cache_data.length}. Uploading...") if $myDEBUG
+	  write(:cache, 'flist', compressed_cache_data)
+	end
+  end
+
+  def export_key()
+	@cui.error("DEBUG[#{Thread.current.inspect}]: Exporting keys...") if $myDEBUG
+	IO.popen('"' + $options[:gpg_bin] + '" ' + '--export-ownertrust', 'w+') do | gpg |
+	  gpg.close_write()
+	  trust = gpg.read()
+	  # Need to use the proxy directly as the data would get encrypted otherwise
+	  @proxy.write($options[:target], :root, 'trust.gpg', trust)
+	end
+	IO.popen('"' + $options[:gpg_bin] + '" ' + '--export-secret-keys --armor --default-key ' + $options[:gpg_key], 'w+') do | gpg |
+	  gpg.close_write()
+	  key = gpg.read()
+	  @proxy.write($options[:target], :root, 'key.gpg', key)
+	end
+  end
+
+  def list(type)
+	@proxy.list($options[:target], type)
+  end
+end

data/classes/cui.rb

@@ -0,0 +1,148 @@
+require 'singleton'
+#noinspection RubyResolve
+require 'thread'
+
+
+# Usage of stdout and stderr:
+# stdout will be used for normal status messages like the status bar.
+# In short, messages that could be redirected to /dev/null without impact.
+# stderr will be used for all messages that a user is supposed to read,
+# or that should show up in logfiles (e.g. error messages). By definition
+# this includes DEBUG messages. Cui.error will automatically append
+# information like the current file, if it's known.
+
+
+class Cui
+  include Singleton
+  def initialize()
+	@update_thread = nil
+	@mutex = Mutex.new
+	@total_size = 0 # These are bytes!
+	@finished_size = 0 # These are bytes!
+	@total_files = 0
+	@finished_files = 0
+	@current_file_size = 0
+	@current_file_finished_size = 0
+	@current_file_name = ""
+	@last_file_name = ""
+	@last_length = 0
+	@changed = false
+	@active = false
+	@exists = true
+  end
+  def exists(e)
+	@exists = e
+  end
+  def total_size_add(size)
+	@mutex.synchronize {
+	  @total_size += size
+	  @changed = true
+	}
+  end
+  def finished_size_add(size)
+	@mutex.synchronize {
+	  @finished_size += size
+	  @current_file_finished_size += size
+	  @changed = true
+	}
+  end
+  def total_files_inc()
+	@mutex.synchronize {
+	  @total_files += 1
+	  @changed = true
+	}
+  end
+  def current_file_size(size)
+	@mutex.synchronize {
+	  @current_file_size = size
+	  @changed = true
+	}
+  end
+  def current_file_name(file)
+	@mutex.synchronize {
+	  @finished_files += 1 unless @last_file_name == ""
+	  @current_file_finished_size = 0
+	  @current_file_size = 0
+	  @last_file_name = @current_file_name
+	  @current_file_name = file.to_str
+	  update
+	}
+  end
+  def start()
+	stop if @update_thread # Stop the thread if it's already running
+	@changed = true
+	@active = true
+	@update_thread = Thread.new {
+	  while @active
+		sleep(0.3)
+		@mutex.synchronize {
+		  update if @changed
+		}
+	  end
+	}
+  end
+  def update()
+	finished_size = @finished_size / 1024 / 1024
+	total_size = @total_size / 1024 / 1024
+	current_file_size = ("%.1f" % (@current_file_size.to_f / 1024 / 1024)).to_f
+	current_file_finished_size = ("%.1f" % (@current_file_finished_size.to_f / 1024 / 1024)).to_f
+	size_length = total_size.to_s.length
+	files_length = @total_files.to_s.length
+	current_file_size_length = current_file_size.to_s.length
+	uploading = "Uploading..."
+	uploading = "Checking..." if @exists
+	template = "Total %#{size_length}d/%#{size_length}d MB | Files %#{files_length}d/%#{files_length}d | Current %#{current_file_size_length}.1f/%#{current_file_size_length}.1f MB " + uploading
+	if @current_file_name != @last_file_name then
+	  text = ""
+	  text += "\b" * @last_length if @last_length > 0
+	  text += @current_file_name
+	  text += " " * (@last_length - @current_file_name.length) if (@last_length - @current_file_name.length) > 0
+	  text += "\n"
+	  $stdout.print(text)
+	  @last_file_name = @current_file_name
+	  @last_length = 0
+	end
+	backspace = ("\b" * @last_length)
+	line = template % [ finished_size, total_size, @finished_files, @total_files, current_file_finished_size, current_file_size ]
+	@last_length = line.length
+	$stdout.print(backspace + line)
+	@changed = false
+  end
+  def message(text)
+	@mutex.synchronize {
+	  case @active
+		when true
+		  backspace = ("\b" * @last_length)
+		  text += " " * (@last_length - text.length) if (@last_length - text.length) > 0
+		  @last_length = 0
+		  $stdout.print(backspace + text + "\n")
+		  update
+		when false
+		  # If there's no regular update, we can just write out a new line
+		  $stdout.print("\n" + text + "\n")
+	  end
+	}
+  end
+  def error(text)
+	@mutex.synchronize {
+	  case @active
+		when true
+		  backspace = ("\b" * @last_length) # Calculate the amount of backspaces required
+		  spaces = ""
+		  spaces = " " * (@last_length - text.length) if (@last_length - text.length) > 0
+		  $stderr.print(backspace + text + spaces + "\n")
+		  @last_length = 0
+		  update
+		when false
+		  # If there's no regular update, we can just write out a new line
+		  $stdout.print("\n" + text + "\n")
+	  end
+	}
+  end
+  def stop()
+	@active = false
+	@update_thread.run
+	@update_thread.join
+	@update_thread = nil
+  end
+end

data/classes/proxy_http_cache_data.rb

@@ -0,0 +1,16 @@
+$options = nil unless $options
+
+class ProxyHTTPCacheData
+  attr_accessor :time_validated
+
+  def initialize()
+	# Create an entry that's valid at least 1 second...
+	@time_validated = Time.now + ( Kernel.rand($options[:cache_max_age] - 1) + 1 )
+  end
+
+  def valid?()
+	return false if @time_validated.nil?
+	return false if (Time.now - @time_validated) > $options[:cache_max_age]
+	return true
+  end
+end

data/classes/proxy_http_cache_hash.rb

@@ -0,0 +1,72 @@
+#noinspection RubyResolve
+require 'classes/proxy_http_cache_data.rb'
+
+$options = nil unless $options
+
+class ProxyHTTPCache_Hash
+  def initialize()
+	@chunks = Hash.new
+	@files = Hash.new
+	@changed = false
+  end
+
+  def add(type, key)
+	case type
+	  when :chunk
+		add_key(@chunks, key)
+	  when :file
+		add_key(@files, key)
+	  else
+		raise "no such cache - #{type.to_s}"
+	end
+	@changed = true
+  end
+
+  def mark_valid(type, key)
+	case type
+	  when :chunk
+		set_validated(@chunks, key)
+	  when :file
+		set_validated(@files, key)
+	  else
+		raise "no such cache - #{type.to_s}"
+	end
+	@changed = true
+  end
+
+  def search(type, key)
+	case type
+	  when :chunk
+		result = search_tree(@chunks, key)
+		return result.valid? unless result.nil?
+		return false
+	  when :file
+		result = search_tree(@files, key)
+		return result.valid? unless result.nil?
+		return false
+	  else
+		raise "no such cache - #{type.to_s}"
+	end
+  end
+
+  def dump()
+	return nil if not @changed
+	return Marshal.dump(self)
+  end
+
+  private
+  def add_key(hash, key)
+	data = ProxyHTTPCacheData.new
+	hash[key] = data
+  end
+
+  def set_validated(hash, key)
+	data = hash[key]
+	data.time_validated = Time.now
+  end
+
+  def search_tree(hash, key)
+	data = hash[key]
+	return data
+  end
+end

data/doc/dev/options.txt

@@ -0,0 +1,69 @@
+This document describes the options hash.
+
+$options = {
+        :program_name => 'rubyyabt',
+        :version => [0, 0, 1],
+        :verbose => false,
+        :debug => false,
+        :excl_incl => [],
+        :username => "",
+        :password => "",
+        :chunk_size => 2 ** 20, # 1 Megabyte chunk size by default
+        :gpg_bin => '/usr/bin/gpg',
+        :source => '/home/user',
+        :target => 'https://dav.example.org/path/to/backupfiles',
+        :gpg_key => 'ABCDEF12',
+        :gpg_pass => 'mysecretpass',
+        :user_agent => options[:program_name] + " " + options[:version].join(".")
+        :ssl_cert_dir => '/etc/ssl/certs',
+        :backup_name => '20100410T113956',
+        :mode => :backup | :restore,
+		:empty_cache => false
+}
+
+The hash contains the above listed symbols. An explanation follows:
+:program_name
+    String containing the name of the program.
+:version
+    Array containing 3 numbers that define the version number of the program
+:verbose
+    Boolean that shows if the user wants verbose or just minimum output
+:debug
+    Boolean defining if full scale debugging output is requested
+:excl_incl
+    Array that contains a list of includes and excludes (Strings). The first character of the string has to be "-" or
+    "+" to define what happens if there is a match. A "-" will exclude the file or in case of a directory, the directory
+    and all of its children. The last match decides the fate of the file/dir.
+:username
+    The username (string) required to connect to :target.
+:password
+    The password (string) required to connect to :target.
+:chunk_size
+    Defines the maximum size of a chunk. Every file that is backed up will be split in chunks of this size. The last
+    chunk can of course be smaller. Changing this value when a backup already exists in :target is unsupported and
+    should not be done.
+    Restores are not affected by this option.
+:gpg_bin
+    The path of the GPG binary that will be used to encrypt the data.
+:gpg_key
+    The GPG key that will be used to encrypt the data. Accepts whatever gpg accepts with the --recipient and
+    --local-user option.
+:gpg_pass
+    The password required for the gpg key. Required both for backup and restore as during the backup the data is
+    signed, which requires the password.
+:source
+    The local directory that contains the data to be backed up or acts as the destination directory for the restore.
+    This should have been called :local_dir instead...
+:target
+    The remote directory or URL that the backup will be written to. In case of a restore the data will be read from this
+    location. Should have been called :remote_location instead...
+:user_agent
+    The HTTP User Agent sent to the server.
+:ssl_cert_dir
+    Directory that contains valid root certificates for verifying certificates of https servers.
+:backup_name
+    Needs to be set for a restore. Defines which backup will be restored.
+:mode
+    Is either :backup or :restore, depending on what the user is requesting.
+:empty_cache
+    Prevents the load of the cache on the server

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: rubyyabt
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 5
+  version: 0.0.5
+platform: ruby
+authors: 
+- Daniel Frank
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-06 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: rubyyabt backups file to a WebDAV location. The data is splitted into chunks and encrypted with GPG.
+email: tokudan@rubyforge.org
+executables: 
+- rubyyabt-backup.rb
+- rubyyabt-restore.rb
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/rubyyabt-backup.rb
+- bin/rubyyabt-restore.rb
+- classes/Backup.rb
+- classes/Chunk.rb
+- classes/cui.rb
+- classes/GPG.rb
+- classes/ProxyFile.rb
+- classes/ProxyHTTP.rb
+- classes/proxy_http_cache_data.rb
+- classes/proxy_http_cache_hash.rb
+- classes/SMGFile.rb
+- classes/Source.rb
+- classes/Target.rb
+- doc/dev/options.txt
+has_rdoc: true
+homepage: http://rubyforge.org/projects/rubyyabt/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- .
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: rubyyabt
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: backs up data to WebDAV
+test_files: []
+