s3-tar-backup 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 842cea517d3276df06766424c4354d644069df99
-  data.tar.gz: 1ad93ccb5dd87a00b794692643cdf9dfc4936bcd
+  metadata.gz: aa76416c2881739829d7dde209216071f4889caa
+  data.tar.gz: b98758ddff06cf3125180a851a94a2b5f4a97277
 SHA512:
-  metadata.gz: bc7a8b17a71626eef16fc974d4725a2cb11482d4cdb51201093c3a51e1fe12565228666a2302f5d1498f298b8579ceb23c2e913a75682961e5425f00e842ca84
-  data.tar.gz: f7b8d94f5f23c8f234c2f1003b6a48132baacba2df14db0b9754b45cdea812f783e20ffee995b20595864fbf8141280ffd6d65160b33ef097abc1d5ed3966d02
+  metadata.gz: f4c2b4a6fe2035c3461c1745e1b82d273dea3603d096dbcef6f49673111907b0166ea50c95737141912d5de1bae4d19d9ff22f7882dbf1b07633636eff5071a9
+  data.tar.gz: fbc2f9093c5cc8df641316fdde22a484c340f487a14a854ba72805ef902bba32ab807167ce1a0c42b7ccc4d27100adddba2c09067d1d7f2c465318090207bec8

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) <year> <copyright holders>
+Copyright (c) 2012 Antony Male
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -9,16 +9,18 @@ You can then restore the files at a later date.
 
 This tool was built as a replacement for duplicity, after duplicity started throwing errors and generally being a bit unreliable.
 It uses command-line tar to create incremental backup snapshots, and the aws-s3 gem to upload these to S3.
+It can also optionally use command-ling gpg to encrypt backups.
 
 In practice, it turns out that this tool has few lower bandwidth and CPU requirements, and can restore a backup in a fraction of the time that duplicity would take.
 
 Installation
 ------------
 
-This tool is not yet a ruby gem, and unless people ask me, it will remain that way.
+This tool is available as a ruby gem, or you can build it youself.
 
-Therefore, to install:
+To install from rubygems: `gem install s3-tar-backup`.
 
+To build it yourself:
 ```
 $ git clone git://github.com/canton7/s3-tar-backup.git
 $ cd s3-tar-backup
@@ -58,13 +60,20 @@ remove_all_but_n_full = <number>
 
 backup_dir = </path/to/dir>
 dest = <bucket_name>/<path>
+
+; Optional: specifies commands to run before and after each backup
 pre-backup = <some command>
 post-backup = <some command>
 
+; Optional: defaults to bzip2
 compression = <compression_type>
 
+; Optional: defaults to false
 always_full = <bool>
 
+; Optional: defaults to no key
+gpg_key = <key ID>
+
 ; You have have multiple lines of the following types.
 ; Values from here and from your profiles will be combined
 source = </path/to/another/source>
@@ -105,6 +114,10 @@ s3-tar-backup is capable of auto-detecting the format of a previously-backed-up
 `always_full` is an optional key which have have the value `True` or `False`.
 This is used to say that incremental backups should never be used.
 
+`gpg_key` is an optional GPG Key ID to use to encrypt backups.
+This key must exist in your keyring.
+By default, no key is used and backups are not encrypted.
+
 `source` contains the folders to be backed up.
 
 `exclude` lines specify files/dirs to exclude from the backup.
@@ -131,10 +144,23 @@ dest = <bucket_name>/<path>
 exclude = </some/dir>
 pre-backup = <some command>
 post-backup = <some command>
+gpg_key = <key ID>
 ```
 
 `profile_name` is the name of the profile. You'll use this later.
 
+### Encryption
+
+`s3-tar-backup` will encrypt your backups if you specify the config key `gpg_key`, which is the ID of the key to use for encrypting backups. 
+In order to create an encrypted backup, the public key with this ID must exist in your keyring: it doesn't matter if it has a passphrase or not.
+In order to restore an encrypted backup, the private key corresponding to the public key which encrypted the backup must exist in your keyring: your `gpg-agent` will prompt you for the passphrase if required.
+The `gpg_key` option is not used when restoring from backup (instead gpg works out which key to use to decrypt the backup by looking at the backup itself), which means that you can safely change the key that `s3-tar-backup` uses to encrypt backups without losing access to older backups.
+
+`s3-tar-backup` works out whether or not to try and decrypt a backup by looking at its file extension, which means you can safely enable or disable encryption without losing access to older backups.
+
+To create a key, run `gpg --gen-key`, and follow the prompts.
+Make sure you create a backup of the private key using `gpg -a --export-secret-keys <key ID> > s3-tar-backup-secret-key.asc`.
+
 ### Example config file
 
 ```ini
@@ -150,9 +176,12 @@ backup_dir = /root/.backup
 dest = my-backups/tar
 ; You may prefer bzip2, as it has a much lower CPU cost
 compression = lzma2
+gpg_key = ABCD1234
 
 [profile "www"]
 source = /srv/http
+; Don't encrypt this (for some reason)
+gpg_key =
 
 [profile "home"]
 source = /home/me

data/lib/s3_tar_backup.rb

@@ -84,9 +84,11 @@ module S3TarBackup
 
 		def gen_backup_config(profile, config)
 			bucket, dest_prefix = (config.get("profile.#{profile}.dest", false) || config['settings.dest']).split('/', 2)
+			gpg_key = config.get("profile.#{profile}.gpg_key", false) || config['settings.gpg_key']
 			backup_config = {
 				:backup_dir => config.get("profile.#{profile}.backup_dir", false) || config['settings.backup_dir'],
 				:name => profile,
+				:gpg_key => gpg_key && !gpg_key.empty? ? gpg_key : nil,
 				:sources => [*config.get("profile.#{profile}.source", [])] + [*config.get("settings.source", [])],
 				:exclude => [*config.get("profile.#{profile}.exclude", [])] + [*config.get("settings.exclude", [])],
 				:bucket => bucket,
@@ -96,7 +98,7 @@ module S3TarBackup
 				:full_if_older_than => config.get("profile.#{profile}.full_if_older_than", false) || config['settings.full_if_older_than'],
 				:remove_older_than => config.get("profile.#{profile}.remove_older_than", false) || config.get('settings.remove_older_than', false),
 				:remove_all_but_n_full => config.get("profile.#{profile}.remove_all_but_n_full", false) || config.get('settings.remove_all_but_n_full', false),
-				:compression => (config.get("profile.#{profile}.compression", false) || config.get('settings.compression', 'gzip')).to_sym,
+				:compression => (config.get("profile.#{profile}.compression", false) || config.get('settings.compression', 'bzip2')).to_sym,
 				:always_full => config.get('settings.always_full', false) || config.get("profile.#{profile}.always_full", false),
 			}
 			backup_config
@@ -106,7 +108,7 @@ module S3TarBackup
 			puts "===== Backing up profile #{backup_config[:name]} ====="
 			backup_config[:pre_backup].each_with_index do |cmd, i|
 				puts "Executing pre-backup hook #{i+1}"
-				system(cmd)
+				exec(cmd)
 			end
 			full_required = full_required?(backup_config[:full_if_older_than], prev_backups)
 			puts "Last full backup is too old. Forcing a full backup" if full_required && !opts[:full] && backup_config[:always_full]
@@ -117,7 +119,7 @@ module S3TarBackup
 			end
 			backup_config[:post_backup].each_with_index do |cmd, i|
 				puts "Executing post-backup hook #{i+1}"
-				system(cmd)
+				exec(cmd)
 			end
 		end
 
@@ -155,7 +157,7 @@ module S3TarBackup
 		# backup_dir, name, soruces, exclude, bucket, dest_prefix
 		def backup_incr(config, verbose=false)
 			puts "Starting new incremental backup"
-			backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression])
+			backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:gpg_key])
 
 			# Try and get hold of the snar file
 			unless backup.snar_exists?
@@ -179,14 +181,14 @@ module S3TarBackup
 
 		def backup_full(config, verbose=false)
 			puts "Starting new full backup"
-			backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression])
+			backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:gpg_key])
 			# Nuke the snar file -- forces a full backup
 			File.delete(backup.snar_path) if File.exists?(backup.snar_path)
 			backup(config, backup, verbose)
 		end
 
 		def backup(config, backup, verbose=false)
-			system(backup.backup_cmd(verbose))
+			exec(backup.backup_cmd(verbose))
 			puts "Uploading #{config[:bucket]}/#{config[:dest_prefix]}/#{File.basename(backup.archive)} (#{bytes_to_human(File.size(backup.archive))})"
 			upload(backup.archive, config[:bucket], "#{config[:dest_prefix]}/#{File.basename(backup.archive)}")
 			puts "Uploading snar (#{bytes_to_human(File.size(backup.snar_path))})"
@@ -232,7 +234,7 @@ module S3TarBackup
 			restore_dir = opts[:restore].chomp('/') << '/'
 
 			Dir.mkdir(restore_dir) unless Dir.exists?(restore_dir)
-			raise "Detination dir is not a directory" unless File.directory?(restore_dir)
+			raise "Destination dir is not a directory" unless File.directory?(restore_dir)
 
 			prev_backups[restore_start_index..restore_end_index].each do |object|
 				puts "Fetching #{backup_config[:bucket]}/#{backup_config[:dest_prefix]}/#{object[:name]} (#{bytes_to_human(object[:size])})"
@@ -243,7 +245,7 @@ module S3TarBackup
 					end
 				end
 				puts "Extracting..."
-				system(Backup.restore_cmd(restore_dir, dl_file, opts[:verbose]))
+				exec(Backup.restore_cmd(restore_dir, dl_file, opts[:verbose]))
 
 				File.delete(dl_file)
 			end
@@ -252,7 +254,7 @@ module S3TarBackup
 		def perform_list_backups(prev_backups, backup_config)
 			# prev_backups alreays contains just the files for the current profile
 			puts "===== Backups list for #{backup_config[:name]} ====="
-			puts "Type: N:  Date:#{' '*18}Size:       Chain Size:   Format:\n\n"
+			puts "Type: N:  Date:#{' '*18}Size:       Chain Size:   Format:   Encrypted:\n\n"
 			prev_type = ''
 			total_size = 0
 			chain_length = 0
@@ -268,7 +270,7 @@ module S3TarBackup
 				chain_length_str = (chain_length == 0 ? '' : chain_length.to_s).ljust(3)
 				chain_cum_size_str = (object[:type] == :full ? '' : bytes_to_human(chain_cum_size)).ljust(8)
 				puts "#{type}  #{chain_length_str} #{object[:date].strftime('%F %T')}    #{bytes_to_human(object[:size]).ljust(8)}    " \
-					"#{chain_cum_size_str}      (#{object[:compression]})"
+					"#{chain_cum_size_str}      #{object[:compression].to_s.ljust(7)}   #{object[:encryption] ? 'Y' : 'N'}"
 				total_size += object[:size]
 			end
 			puts "\n"
@@ -308,5 +310,10 @@ module S3TarBackup
 			end
 			format("%.2f", n) << %w(B KB MB GB TB)[count]
 		end
+
+		def exec(cmd)
+			puts "Executing: #{cmd}"
+			system(cmd)
+		end
 	end
 end

data/lib/s3_tar_backup/backup.rb

@@ -1,4 +1,4 @@
-require 'aws/s3'
+require 'aws-sdk'
 
 module S3TarBackup
 	class Backup
@@ -10,6 +10,7 @@ module S3TarBackup
 		@archive
 		@compression_flag
 		@compression_ext
+		@gpg_key
 
 		COMPRESSIONS = {
 			:gzip => {:flag => '-z', :ext => 'tar.gz'},
@@ -18,13 +19,16 @@ module S3TarBackup
 			:lzma2 => {:flag => '-J', :ext => 'tar.xz'}
 		}
 
+		ENCRYPTED_EXTENSION = 'asc'
 
-		def initialize(backup_dir, name, sources, exclude, compression=:bzip2)
+
+		def initialize(backup_dir, name, sources, exclude, compression=:bzip2, gpg_key=nil)
 			@backup_dir, @name, @sources, @exclude = backup_dir, name, [*sources], [*exclude]
 			raise "Unknown compression #{compression}. Valid options are #{COMPRESSIONS.keys.join(', ')}" unless COMPRESSIONS.has_key?(compression)
 			@compression_flag = COMPRESSIONS[compression][:flag]
 			@compression_ext = COMPRESSIONS[compression][:ext]
 			@time = Time.now
+			@gpg_key = gpg_key
 
 			Dir.mkdir(@backup_dir) unless File.directory?(@backup_dir)
 		end
@@ -44,20 +48,24 @@ module S3TarBackup
 		def archive
 			return @archive if @archive
 			type = snar_exists? ? 'incr' : 'full'
-			File.join(@backup_dir, "backup-#{@name}-#{@time.strftime('%Y%m%d_%H%M%S')}-#{type}.#{@compression_ext}")
+			encrypted_bit = @gpg_key ? ".#{ENCRYPTED_EXTENSION}" : ''
+			File.join(@backup_dir, "backup-#{@name}-#{@time.strftime('%Y%m%d_%H%M%S')}-#{type}.#{@compression_ext}#{encrypted_bit}")
 		end
 
 		def backup_cmd(verbose=false)
-			exclude = @exclude.map{ |e| "\"#{e}\""}.join(' ')
+			exclude = @exclude.map{ |e| " --exclude \"#{e}\""}.join
 			sources = @sources.map{ |s| "\"#{s}\""}.join(' ')
 			@archive = archive
-			"tar c#{verbose ? 'v' : ''}f \"#{@archive}\" #{@compression_flag} -g \"#{snar_path}\" --exclude #{exclude} --no-check-device #{sources}"
+			tar_archive = @gpg_key ? '' : "f \"#{@archive}\""
+			gpg_cmd = @gpg_key ? " | gpg -r #{@gpg_key} -o \"#{@archive}\" --always-trust --yes --batch --no-tty -e" : ''
+			"tar c#{verbose ? 'v' : ''}#{tar_archive} #{@compression_flag} -g \"#{snar_path}\"#{exclude} --no-check-device #{sources}#{gpg_cmd}"
 		end
 
 		def self.parse_object(object, profile)
 			name = File.basename(object.key)
-			match = name.match(/^backup-([\w\-]+)-(\d\d\d\d)(\d\d)(\d\d)_(\d\d)(\d\d)(\d\d)-(\w+)\.(.*)$/)
+			match = name.match(/^backup-([\w\-]+)-(\d\d\d\d)(\d\d)(\d\d)_(\d\d)(\d\d)(\d\d)-(\w+)\.(.*?)(\.#{ENCRYPTED_EXTENSION})?$/)
 			return nil unless match && match[1] == profile
+
 			return {
 				:type => match[8].to_sym,
 				:date => Time.new(match[2].to_i, match[3].to_i, match[4].to_i, match[5].to_i, match[6].to_i, match[7].to_i),
@@ -66,14 +74,17 @@ module S3TarBackup
 				:size => object.content_length,
 				:profile => match[1],
 				:compression => COMPRESSIONS.find{ |k,v| v[:ext] == match[9] }[0],
+				:encryption => !match[10].nil?
 			}
 		end
 
 		# No real point in creating a whole new class for this one
 		def self.restore_cmd(restore_into, restore_from, verbose=false)
-			ext = restore_from.match(/[^\.\\\/]+\.(.*)$/)[1]
+			ext, encrypted = restore_from.match(/[^\.\\\/]+\.(.*?)(\.#{ENCRYPTED_EXTENSION})?$/)[1..2]
 			compression_flag = COMPRESSIONS.find{ |k,v| v[:ext] == ext }[1][:flag]
-			"tar xp#{verbose ? 'v' : ''}f #{restore_from} #{compression_flag} -G -C #{restore_into}"
+			tar_archive = encrypted ? '' : "f \"#{restore_from}\""
+			gpg_cmd = encrypted ? "gpg --yes --batch --no-tty -d \"#{restore_from}\" | " : ''
+			"#{gpg_cmd}tar xp#{verbose ? 'v' : ''}#{tar_archive} #{compression_flag} -G -C #{restore_into}"
 		end
 
 	end

data/lib/s3_tar_backup/version.rb

@@ -1,3 +1,3 @@
 module S3TarBackup
-	VERSION = '1.1.0'
+	VERSION = '1.1.1'
 end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: s3-tar-backup
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Antony Male
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-16 00:00:00.000000000 Z
+date: 2016-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 description: Uses tar's incremental backups to backup data to, and restore from, Amazon's
@@ -50,17 +50,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.5.0
 signing_key: 
 specification_version: 4
 summary: 's3-tar-backup: Incrementally backup/restore to Amazon S3 using tar'