rubyyabt 0.0.5

data/bin/rubyyabt-backup.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+$stdout.sync = true
+$stderr.sync = true
+
+require 'optparse'
+
+# Defaults for the command line options
+options = {
+        :program_name => 'rubyyabt',
+        :version => [0, 0, 1],
+        :verbose => false,
+        :debug => false,
+        :excl_incl => [],
+        :username => "",
+        :password => "",
+        :chunk_size => 2 ** 20, # 1 Megabyte chunk size by default
+        :gpg_bin => '/usr/bin/gpg',
+        :ssl_cert_dir => '/etc/ssl/certs',
+        :mode => :backup,
+		:empty_cache => false,
+		:cache_max_age => 90 * (24*60*60), # By default allow 30 days cache time
+		:http_timeout => 5 * 60 # By default allow a http timeout of 5 minutes
+}
+
+# Parse the command line...
+OptionParser.new do |opts|
+  opts.banner = "Usage: rubyyabt-backup.rb [options] --source <source> --target <target>"
+  opts.on("-s", "--source SOURCE", "Specify the source path") { |v| options[:source] = v }
+  opts.on("-e", "--exclincl FILE", "Load excludes/includes from FILE") { |v| options[:excl_incl] = File.readlines(v) }
+  opts.on("-t", "--target TARGET", "Specify the target path") { |v| options[:target] = v }
+  opts.on("-u", "--user USER", "Username for target") { |v| options[:username] = v }
+  opts.on("-p", "--pass PASS", "Password for target") { |v| options[:password] = v }
+  opts.on("-P", "--passfile FILE", "Read password for target from FILE") { |v| options[:password] = File.read(v) }
+  opts.on("-G", "--gpg-bin FILE", "use FILE as gpg binary") { |v| options[:gpg_bin] = v }
+  opts.on("-g", "--gpg-key KEY", "Use KEY as GPG key for encryption and signing") { |v| options[:gpg_key] = v }
+  opts.on("-h", "--gpg-pass PASS", "Password for GPG key") { |v| options[:gpg_pass] = v }
+  opts.on("-H", "--gpg-passfile FILE", "Read password for GPG key from FILE") { |v| options[:gpg_pass] = File.read(v) }
+  opts.on("-c", "--cache-max-age DAYS", "Allow maximum of DAYS time for a positive cache hit. False hits are never cached.") { |v| options[:cache_max_age] = v.to_i *(24*60*60) }
+  opts.on("",   "--http-timeout SECS", "Require all http requests to complete within SECS seconds.") { |v| options[:http_timeout] = v.to_i }
+  opts.on("-C", "--empty-cache", "Start with an empty cache instead of restoring the cache from saved data") { options[:empty_cache] = true }
+  opts.on("-v", "--[no-]verbose", "Run verbosely") { |v| options[:verbose] = v }
+      opts.on("-d", "--[no-]debug", "Show debug messages") { |v| options[:debug] = v }
+  opts.on_tail("-?", "--help", "Show this message") {
+    puts opts
+    exit
+  }
+end.parse!
+
+abort('--source must be specified') if not options.include?(:source)
+abort('--target must be specified') if not options.include?(:target)
+abort('--gpg-key must be specified') if not options.include?(:gpg_key)
+abort('--gpg-pass must be specified') if not options.include?(:gpg_pass)
+
+# Now load the main program
+#noinspection RubyResolve
+require 'classes/Backup'
+
+Thread.abort_on_exception = true
+$myDEBUG = options[:debug]
+$myVERBOSE = options[:verbose]
+
+backup = Backup.new(options)
+
+exit 1 if backup.backup! > 0

data/bin/rubyyabt-restore.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+$stdout.sync = true
+$stderr.sync = true
+
+require 'optparse'
+
+# Defaults for the command line options
+options = {
+        :program_name => 'rubyyabt',
+        :version => [0, 0, 1],
+        :verbose => false,
+        :debug => false,
+        :excl_incl => [],
+        :username => "",
+        :password => "",
+        :chunk_size => 2 ** 20, # 1 Megabyte chunk size by default
+        :gpg_bin => '/usr/bin/gpg',
+        :ssl_cert_dir => '/etc/ssl/certs',
+        :mode => :restore
+        }
+# Parse the command line...
+OptionParser.new do |opts|
+  opts.banner = "Usage: rubyyabt-restore.rb [options] --source <LOCAL> --target <REMOTE> --backup <BACKUPNAME>"
+  opts.on("-s", "--source LOCAL", "Specify the local path (restore will write to this directory)") { |v| options[:source] = v }
+  opts.on("-e", "--excl_incl FILE", "Load excludes/includes from FILE") { |v| options[:excl_incl] = File.readlines(v) }
+  opts.on("-t", "--target REMOTE", "Specifies the remote path that contains the backed up data") { |v| options[:target] = v }
+  opts.on("-b", "--backup BACKUPNAME", "Specifies which backup should be restored (no .gpg extension)") { |v| options[:backup_name] = v }
+  opts.on("-u", "--user USER", "Username for target") { |v| options[:username] = v }
+  opts.on("-p", "--pass PASS", "Password for target") { |v| options[:password] = v }
+  opts.on("-P", "--passfile FILE", "Read password for target from FILE") { |v| options[:password] = File.read(v) }
+  opts.on("-g", "--gpg-key KEY", "Use KEY as GPG key for encryption and signing") { |v| options[:gpg_key] = v }
+  opts.on("-h", "--gpg-pass PASS", "Password for GPG key") { |v| options[:gpg_pass] = v }
+  opts.on("-H", "--gpg-passfile FILE", "Read password for GPG key from FILE") { |v| options[:gpg_pass] = File.read(v) }
+  opts.on("-v", "--[no-]verbose", "Run verbosely") { |v| options[:verbose] = v }
+  opts.on("-d", "--[no-]debug", "Show debug messages") { |v| options[:debug] = v }
+  opts.on_tail("-?", "--help", "Show this message") {
+    puts opts
+    exit
+  }
+end.parse!
+
+abort('--source must be specified') if not options.include?(:source)
+abort('--target must be specified') if not options.include?(:target)
+abort('--gpg-key must be specified') if not options.include?(:gpg_key)
+abort('--gpg-pass must be specified') if not options.include?(:gpg_pass)
+
+# Now load the main program
+#noinspection RubyResolve
+require 'classes/Backup'
+
+Thread.abort_on_exception = true
+$myDEBUG = options[:debug]
+$myVERBOSE = options[:verbose]
+
+backup = Backup.new(options)
+backup.restore!

data/classes/Backup.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+#noinspection RubyResolve
+require 'classes/SMGFile'
+#noinspection RubyResolve
+require 'classes/Source'
+#noinspection RubyResolve
+require 'classes/Target'
+#noinspection RubyResolve
+require 'classes/cui'
+
+$stderr.sync = true
+$stdout.sync = true
+
+$myDEBUG = false unless $myDEBUG
+
+# TODO: Implement a trap for SIGINT to catch CTRL+c
+
+class Backup
+  def initialize(options)
+	$options = options
+	@cui = Cui.instance()
+	@target = Target.instance()
+	@source = Source.instance()
+	@metadata = Array.new
+	# Find a good name for the backup. If one has been given through options[:backup_name] then use it,
+	# else create one by using the current date and time.
+	@name = $options[:backup_name] if options.include?(:backup_name)
+	if (options[:mode] == :restore) and not (options.include?(:backup_name)) then
+	  raise("No backup to restore given.")
+	end
+	@time = Time.now
+	@time.utc
+	@name = @time.strftime("%Y%m%dT%H%M%S") unless @name
+	@cui.message("Initialized backup #{@name}")
+  end
+
+  def backup!()
+	$errors = 0 if not $errors
+	@cui.message("Scanning for files...")
+	@cui.start
+	@cui.message("Found #{@source.files.count.to_s} files and directories")
+	backup_meta = Array.new
+	backup_meta << '[Backup]'
+	cache_timer = Time.now
+	@source.files.each { | f |
+	  if (Time.now - cache_timer) > 900 then
+		@cui.message("Uploading cache data...")
+		@target.upload_cache
+		cache_timer = Time.now
+	  end
+	  begin
+		file = SMGFile.new(f)
+		@cui.error("DEBUG[#{Thread.current.inspect}]: created new SMGFile object from #{f}") if $myDEBUG
+		file.backup!
+		file_meta = Array.new
+		file_meta << 'file=' + file.meta_sha256 + '.' + file.meta_md5
+		file_meta << 'size=' + file.size.to_s
+		file_meta << 'mtime=' + file.mtime
+		file_meta << 'name=' + f
+		file_meta = file_meta.join(";")
+		backup_meta << file_meta
+	  rescue Exception
+		@cui.error("Error: #{$!.to_s}. Skipping this object.")
+		$errors += 1
+	  end
+	}
+	@cui.message("Waiting for last chunk to finish uploading...")
+	@cui.error("DEBUG[#{Thread.current.inspect}]: Joining chunk threads") if $myDEBUG
+	chunk = Chunk.new
+	chunk.join
+	@cui.stop
+	@cui.update
+	@cui.message("Finished backing up data. Now uploading metadata for backup #{@name}.")
+	backup_meta = backup_meta.join("\n")
+	@cui.error("DEBUG[#{Thread.current.inspect}]: Writing backup info") if $myDEBUG
+	@target.write(:backup, @name, backup_meta)
+	@cui.message("Uploading cache data...")
+	@target.upload_cache
+	@cui.error("Errors: A total of #{$errors} errors have been encountered.") if $errors > 0
+	return $errors
+  end
+
+  def restore!()
+	backup_meta = @target.read(:backup, @name)
+	backup_meta.lines { | line |
+	  line.chomp!
+	  if !(line.eql?('[Backup]')) then
+		sha256 = line[5..68]
+		md5 = line[70..101]
+		size = /^[^;]*;size=([0-9]*);/.match(line).captures[0].to_i
+		mtime = Time.rfc2822(/^[^;]*;[^;]*;mtime=([^;]*);/.match(line).captures[0])
+		name = /^[^;]*;[^;]*;[^;]*;name=(.*)$/.match(line).captures[0]
+		@cui.message("Restoring file: size=#{size} mtime=#{mtime} name=#{name}")
+		f = SMGFile.new(nil, md5, sha256)
+		f.restore!
+	  end
+	}
+  end
+end

data/classes/Chunk.rb

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+require 'digest/md5'
+#noinspection RubyResolve
+require 'digest/sha2'
+#noinspection RubyResolve
+require 'thread'
+#noinspection RubyResolve
+require 'classes/Target'
+#noinspection RubyResolve
+require 'classes/cui'
+
+$myDEBUG = false unless $myDEBUG
+
+class Chunk
+  attr_reader :data, :length
+  @@upload_mutex = Mutex.new
+  @@upload_thread = nil
+
+  def initialize()
+    @md5 = nil
+    @md5thread = nil
+    @sha256 = nil
+    @sha256thread = nil
+    @data = nil
+    @cui = Cui.instance
+  end
+
+  def join()
+    @cui.error("DEBUG[#{Thread.current.inspect}]: Joining chunk thread") if $myDEBUG
+    @@upload_thread.join if @@upload_thread
+  end
+
+  def set_data(data)
+    @cui.error("DEBUG[#{Thread.current.inspect}: Entered chunk set_data") if $myDEBUG
+    @data = data
+    @cui.error("DEBUG[#{Thread.current.inspect}: Set data") if $myDEBUG
+    @length = @data.length
+    @cui.error("DEBUG[#{Thread.current.inspect}: Calculated length") if $myDEBUG
+    @md5 = nil
+    @md5 = Digest::MD5.hexdigest(@data)
+    @sha256 = nil
+    @sha256 = Digest::SHA2.new(256).hexdigest(@data)
+  end
+
+  def md5()
+    return @md5 if @md5
+    @md5 = Digest::MD5.hexdigest(@data)
+  end
+
+  def md5=(md5)
+    @data = nil
+    @md5 = md5
+  end
+
+  def sha256()
+    return @sha256 if @sha256
+    @sha256 = Digest::SHA2.new(256).hexdigest(@data)
+  end
+
+  def sha256=(sha256)
+    @data = nil
+    @sha256 = sha256
+  end
+
+  def backup!()
+    @cui.error("DEBUG[#{Thread.current.inspect}]: Locking chunk upload mutex in thread #{Thread.current.inspect}") if $myDEBUG
+    @@upload_mutex.synchronize {
+      @cui.error("DEBUG[#{Thread.current.inspect}]: Got lock of chunk upload mutex in thread #{Thread.current.inspect}") if $myDEBUG
+      @cui.error("DEBUG[#{Thread.current.inspect}]: Trying to join earlier upload threads if any (@@uploadThread: #{@@upload_thread.inspect})") if $myDEBUG
+      @@upload_thread.join if @@upload_thread
+      @@upload_thread = Thread.new {
+        target = Target.instance # Get a target object, then ...
+        @cui.error("DEBUG[#{Thread.current.inspect}]: Checking if target chunk exists (#{sha256}, #{md5})") if $myDEBUG
+        if not target.exists?(:chunk, "#{sha256}.#{md5}") then
+          @cui.error("Uploading chunk #{sha256}.#{md5}...") if $myDEBUG
+          target.write(:chunk, "#{sha256}.#{md5}", @data)
+        end
+        @cui.error("DEBUG[#{Thread.current.inspect}]: Upload chunk thread finished (thread id: #{Thread.current.inspect})") if $myDEBUG
+      }
+      @cui.error("DEBUG[#{Thread.current.inspect}]: New upload thread: @@upload_thread = #{@@upload_thread.inspect}") if $myDEBUG
+    }
+  end
+
+  def restore!()
+    @cui.message("Downloading chunk #{sha256}.#{md5}...")
+    target = Target.instance
+    @data = target.read(:chunk, "#{sha256}.#{md5}")
+    @length = @data.length
+    expected_md5 = @md5
+    expected_sha256 = @sha256
+    @md5 = nil
+    @sha256 = nil
+    if (expected_md5 == md5) and (expected_sha256 == sha256) then
+      @cui.message("chunk verified")
+    else
+      @cui.message("checksum error in chunk #{sha256}.#{md5}")
+      raise 'checksum error'
+    end
+  end
+end

data/classes/GPG.rb

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+
+require 'tempfile'
+#noinspection RubyResolve
+require 'classes/cui'
+
+$options = nil unless $options
+
+class GPG
+  def initialize()
+	@cui = Cui.instance
+  end
+
+  def encrypt(data)
+	# Setup the pipes
+	pass = IO::pipe	# Pipe for providing the password
+	pi = IO::pipe		# Data will be written to this pipe (stdin for gpg)
+	po = IO::pipe		# Encrypted data can be read from this pipe (stdout for gpg)
+
+	gpg_proc = Kernel.fork do
+	  # Cleanup unused end of the pipes
+	  pass[1].close
+	  pi[1].close
+	  po[0].close
+
+	  # Remap the STDIN and STDOUT pipes
+	  STDIN.reopen(pi[0])
+	  pi[0].close
+	  STDOUT.reopen(po[1])
+	  po[1].close
+
+	  # Execute gpg
+	  Kernel.exec($options[:gpg_bin], '--batch', '--encrypt', '--sign', '--passphrase-fd', pass[0].fileno.to_s, '--recipient', $options[:gpg_key], '--local-user', $options[:gpg_key])
+	  raise 'Error running GPG'
+	end
+
+	# Cleanup unused end of the pipes for ruby
+	pass[0].close
+	pi[0].close
+	po[1].close
+
+	# Start new threads to send the data to GPG. This hopefully avoids any blocking.
+	thread_pass = Thread.new do
+	  pass[1].write($options[:gpg_pass])
+	  pass[1].close
+	end
+	thread_pi = Thread.new do
+	  pi[1].write(data)
+	  pi[1].close
+	end
+
+	# Run a new thread to read data from GPG.
+	encrypted_data = String.new # Initialize the variable for reading from GPG
+	thread_po = Thread.new do
+	  # Read from GPG until the pipe is closed
+	  #while (new_encrypted_data = po[0].read)
+		#encrypted_data += new_encrypted_data
+	  while (!po[0].eof?)
+		encrypted_data += po[0].read
+	  end
+	  po[0].close
+	end
+	# Now cleanup the process from GPG and the threads
+	Process.wait(gpg_proc)
+	thread_pi.join
+	#noinspection RubyUnusedLocalVariable
+	data = nil	# Free up the variable and hopefully some memory while we're waiting for the other threads
+	thread_pass.join
+	thread_po.join
+	# Return the data, if there's no data, GPG has probably thrown an error.
+	return encrypted_data if encrypted_data.length > 0
+	@cui.error("Failed to run gpg to encrypt data. Command run: " + [$options[:gpg_bin], '--batch', '--encrypt', '--sign', '--passphrase-fd', pass[0].fileno.to_s, '--recipient', $options[:gpg_key], '--local-user', $options[:gpg_key]].inspect)
+	raise 'GPG error'
+  end
+
+  def decrypt(encrypted_data)
+	# Setup the pipes
+	pass = IO::pipe	# Pipe for providing the password
+	pi = IO::pipe		# Data will be written to this pipe (stdin for gpg)
+	po = IO::pipe		# Encrypted data can be read from this pipe (stdout for gpg)
+
+	gpg_proc = Kernel.fork do
+	  # Cleanup unused end of the pipes
+	  pass[1].close
+	  pi[1].close
+	  po[0].close
+
+	  # Remap the STDIN and STDOUT pipes
+	  STDIN.reopen(pi[0])
+	  pi[0].close
+	  STDOUT.reopen(po[1])
+	  po[1].close
+
+	  # Execute gpg
+	  Kernel.exec($options[:gpg_bin], '--batch', '--decrypt', '--passphrase-fd', pass[0].fileno.to_s)
+	  raise 'Error running GPG'
+	end
+
+	# Cleanup unused end of the pipes for ruby
+	pass[0].close
+	pi[0].close
+	po[1].close
+
+	# Start new threads to send the data to GPG. This hopefully avoids any blocking.
+	thread_pass = Thread.new do
+	  pass[1].write($options[:gpg_pass])
+	  pass[1].close
+	end
+	thread_pi = Thread.new do
+	  pi[1].write(encrypted_data)
+	  pi[1].close
+	end
+
+	# Run a new thread to read data from GPG.
+	data = String.new # Initialize the variable for reading from GPG
+	thread_po = Thread.new do
+	  # Read from GPG until the pipe is closed
+	  #while (new_data = po[0].read)
+	    #data += new_data
+	  while (!po[0].eof?)
+		data += po[0].read
+	  end
+	  po[0].close
+	end
+	# Now cleanup the process from GPG and the threads
+	Process.wait(gpg_proc)
+	thread_pi.join
+	#noinspection RubyUnusedLocalVariable
+	encrypted_data = nil	# Free up the variable and hopefully some memory while we're waiting for the other threads
+	thread_pass.join
+	thread_po.join
+	# Return the data, if there's no data, GPG has probably thrown an error.
+	return data if data.length > 0
+	@cui.error("Failed to run gpg to encrypt data. Command run: " + [$options[:gpg_bin], '--batch', '--decrypt', '--passphrase-fd', pass[0].fileno.to_s].inspect)
+	raise 'GPG error'
+  end
+end

data/classes/ProxyFile.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+class ProxyFile
+  def initialize()
+    @types = {
+            :chunk    => 'chunks',
+            :file    => 'files',
+            :backup    => 'backups',
+            :root    => ''
+    }
+  end
+
+  def caching?()
+	return false
+  end
+
+  def read(target_dir, type, file)
+    target_file = File.expand_path(target_dir.to_s + '/' + @types[type] + '/' + file)
+    open(target_file, "rb") {|io| io.read }
+  end
+
+  def write(target_dir, type, file, data)
+    target file = File.expand_path(target_dir.to_s + '/' + @types[type] + '/' + file)
+    open(target file, "wb") {|io| io.write(data) }
+  end
+
+  def exists?(target_dir, type, file)
+    target_file = File.expand_path(target_dir.to_s + '/' + @types[type] + '/' + file)
+    if File.exists?(target_file) then
+      if File.directory?(target_file) then
+        false
+      else
+        true
+      end
+    else
+      false
+    end
+  end
+
+  def list(target_dir, type)
+    Dir.entries(target_dir + '/' + @types[type])
+  end
+end