s3streambackup 0.1.0 → 0.2.0

data/README.md

@@ -1,6 +1,7 @@
 # S3 Stream Backup
 
 Stores data from STDIN in S3 object using multipart upload and removes oldest backups to keep maximum desired backup object count.
+Restore tool included.
 
 ## Installing
 
@@ -15,12 +16,20 @@ gem install s3streambackup
 ## Usage
 
 ```bash
-s3streambackup mybucket some-backup < some-backup.file
+# store some-backup.file in mybucket bucket and name it my-backup
+s3streambackup mybucket my-backup < some-backup.file
+
+# list available backups of my-backup
+s3streamrestore mybucket my-backup
+
+# restore my-backup backup from 2013-08-06 09:03:17 UTC
+s3streamrestore mybucket my-backup 130806_090317 > some-backup.file
 ```
 
-Note that you should have your S3 key and secret set in environment `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` or you can specify them in command line with `--key` and `--secret`.
+Note that you should have your S3 key and secret set in environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` or you can specify them in command line with `--key` and `--secret` options.
 
 You can store your backup object within prefix by using `--prefix` option e.g.: `--prefix backups/server1/`.
+Additionally you can postfix your backup objects by using `--postfix` option e.g: `--postfix .sql.gz`.
 
 By default two backup copies will be kept. You can change this number by using `--keep` options.
 
@@ -29,7 +38,11 @@ For other usage information use `--help`.
 ### PostgreSQL backup example
 
 ```bash
-/bin/su - postgres -c '/usr/bin/pg_dumpall' | xz -2 | s3streambackup --keep 7 --prefix backups/zabbix/ mybucket postgress-all
+# backup to S3
+su - postgres -c 'pg_dumpall' | xz -2 | s3streambackup --keep 7 --prefix backups/zabbix/ mybucket postgress-all
+
+# restore could look like this
+s3streambackup --prefix backups/zabbix/ mybucket postgress-all 130806_090317 | xz -d | su - postgres -c 'psql'
 ```
 
 ## Contributing to S3 Stream Backup

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/bin/s3streambackup

@@ -1,117 +1,55 @@
 #!/usr/bin/env ruby
 
-require 'cli'
-require 'aws-sdk'
-require 'logger'
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 's3streambackup'
 
-## HACK: Auto select region based on location_constraint
-module AWS
-	class S3
-		class BucketCollection
-			def [](name)
-				# if name is DNS compatible we still cannot use it for writes if it does contain dots
-				return S3::Bucket.new(name.to_s, :owner => nil, :config => config) if client.dns_compatible_bucket_name?(name) and not name.include? '.'
-
-				# save region mapping for bucket for futher requests
-				@@location_cache = {} unless defined? @@location_cache
-				# if we have it cased use it; else try to fetch it and if it is nil bucket is in standard region
-				region = @@location_cache[name] || @@location_cache[name] = S3::Bucket.new(name.to_s, :owner => nil, :config => config).location_constraint || @@location_cache[name] = :standard
-
-				# no need to specify region if bucket is in standard region
-				return S3::Bucket.new(name.to_s, :owner => nil, :config => config) if region == :standard
-
-				# use same config but with region specified for buckets that are not DNS compatible or have dots and are not in standard region
-				S3::Bucket.new(name.to_s, :owner => nil, :config => config.with(region: region))
-			end
-		end
-	end
-end
-
-settings = CLI.new do
+S3StreamBackup.new do
+	cli do
+		description 'Store backup from STDIN to S3'
 		stdin :data
-	    option :key, 
-			short: :k,
-			description: 'S3 key',
-			default: ENV['AWS_ACCESS_KEY_ID']
-		option :secret, 
-			short: :s,
-			description: 'S3 key secret',
-			default_label: '<secret>',
-			default: ENV['AWS_SECRET_ACCESS_KEY']
-		option :prefix,
-			short: :p,
-			description: 'prefix under which the objects will be kept',
-			default: ''
 		option :keep,
 			short: :K,
 			description: 'how many backup file to keep',
 			cast: Integer,
 			default: 2
-		option :log_file,
-			short: :l,
-			description: 'location of log file'
-		switch :plain,
-			description: 'use plain connections instead of SSL to S3'
-		switch :debug,
-			description: 'log debug messages'
-		argument :bucket,
-			description: 'name of bucket to upload data to'
-		argument :name,
-			description: 'name under which the object will be stored'
-end.parse! do |settings|
-	fail 'AWS_ACCESS_KEY_ID environment not set and --key not used' unless settings.key
-	fail 'AWS_SECRET_ACCESS_KEY environment not set and --secret not used' unless settings.secret
-end
-
-log = Logger.new(settings.log_file ? settings.log_file : STDOUT)
-log.formatter = proc do |severity, datetime, progname, msg|
-	"[#{datetime.utc.strftime "%Y-%m-%d %H:%M:%S.%6N %Z"}] [#{$$}] #{severity}: #{msg.strip}\n"
-end
-
-log.level = Logger::INFO
-log.level = Logger::DEBUG if settings.debug
-
-begin
-	s3 = AWS::S3.new(
-		access_key_id: settings.key,
-		secret_access_key: settings.secret,
-		logger: log,
-		log_level: :debug,
-		use_ssl: ! settings.plain
-	)
-
-	upload_date = Time.now.utc.strftime "%y%m%d_%H%M%S"
-	prefix = "#{settings.prefix}#{settings.name}.backup."
-	path = "#{prefix}#{upload_date}"
-
-	bucket = s3.buckets[settings.bucket]
-	backup = bucket.objects[path] 
-	log.info "writting to: #{path}"
-
-	# make sure we use multipart upload
-	total_bytes = 0
-	backup.write(estimated_content_length: 10 * 1024 ** 3) do |buffer, bytes|
-		log.info "#{total_bytes} bytes written..."
-		data = settings.stdin.read(bytes)
-		total_bytes += data.bytesize
-		buffer.write data
+		option :postfix,
+			short: :P,
+			description: 'postfix which is appended to backup objects',
+			default: ''
 	end
-	log.info "total upload size: #{total_bytes}"
 
-	backups = bucket.objects.with_prefix(prefix).to_a
+	main do |settings, log, s3|
+		upload_date = Time.now.utc.strftime "%y%m%d_%H%M%S"
+		prefix = "#{settings.prefix}#{settings.name}.backup."
+		path = "#{prefix}#{upload_date}#{settings.postfix}"
+
+		bucket = s3.buckets[settings.bucket]
+		backup = bucket.objects[path] 
+		log.info "writting to: #{path}"
+
+		# make sure we use multipart upload
+		total_bytes = ProgeressLogger.new
+		backup.write(
+			content_type: 'application/octet-stream',
+			estimated_content_length: 10 * 1024 ** 3
+		) do |buffer, bytes|
+			total_bytes.log(log, ' written...')
+			data = settings.stdin.read(bytes)
+			total_bytes << data.bytesize
+			buffer.write data
+		end
+		log.info "total upload size: #{total_bytes.in_bytes_auto}"
 
-	log.info "keeping maximum #{settings.keep} latest buckups of #{backups.length} storred"
+		backups = bucket.objects.with_prefix(prefix).to_a
 
-	if backups.length > settings.keep
-		backups.take(backups.length - settings.keep).each do |backup|
-			log.info "removing oldest backup: #{backup.key}"
-			backup.delete
+		log.info "keeping maximum #{settings.keep} latest buckups of #{backups.length} storred"
+
+		if backups.length > settings.keep
+			backups.take(backups.length - settings.keep).each do |backup|
+				log.info "removing oldest backup: #{backup.key}"
+				backup.delete
+			end
 		end
 	end
-rescue => error
-	msg = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")}"
-	log.error msg
-	STDERR.write msg 
-	exit 10
 end
 

data/bin/s3streamrestore

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 's3streambackup'
+
+S3StreamBackup.new do
+	cli do
+		description 'Restore backup from S3 to STDOUT'
+		option :keep,
+			short: :K,
+			description: 'how many backup file to keep',
+			cast: Integer,
+			default: 2
+		option :postfix,
+			short: :P,
+			description: 'ignored',
+			default: ''
+		argument :date,
+			description: 'date of the backup to restore e.g 130806_090317; if not specified list available backups',
+			required: false
+	end
+
+	main do |settings, log, s3|
+		prefix = "#{settings.prefix}#{settings.name}.backup."
+
+		bucket = s3.buckets[settings.bucket]
+		backups = bucket.objects.with_prefix(prefix).to_a
+		if not settings.date
+			dates =  backups.map do |backup|
+				backup.key.to_s.match(/.backup.([0-9]{6}_[0-9]{6})/).captures.first
+			end
+			dates.each do |date|
+				puts "#{date} (#{Time.parse(date + ' +0000')})"
+			end
+			exit 0
+		end
+
+		backup = backups.select do |backup|
+			backup.key =~ /.backup.#{settings.date}/
+		end.first
+
+		fail "backup from date #{settings.date} not found" unless backup
+
+		length = backup.content_length
+		log.info "sourcing from: #{backup.key} length: #{length}"
+
+		total_bytes = ProgeressLogger.new
+		backup.read do |data|
+			total_bytes.log log, ' read...'
+			total_bytes << data.bytesize
+			begin
+				STDOUT.write data
+			rescue Errno::EPIPE
+				log.warn "STDOUT closed prematurely"
+				exit 1
+			end
+		end
+		if total_bytes != length
+			log.warn "got differnet amount of data (#{total_bytes} bytes) than expected (#{length} bytes)"
+			exit 1
+		else
+			log.info "total restore size: #{total_bytes.in_bytes_auto}"
+		end
+	end
+end
+

data/lib/s3streambackup.rb

@@ -0,0 +1,137 @@
+require 'cli'
+require 'aws-sdk'
+require 'logger'
+require 's3streambackup/units'
+
+## HACK: Auto select region based on location_constraint
+module AWS
+	class S3
+		class BucketCollection
+			def [](name)
+				# if name is DNS compatible we still cannot use it for writes if it does contain dots
+				return S3::Bucket.new(name.to_s, :owner => nil, :config => config) if client.dns_compatible_bucket_name?(name) and not name.include? '.'
+
+				# save region mapping for bucket for futher requests
+				@@location_cache = {} unless defined? @@location_cache
+				# if we have it cased use it; else try to fetch it and if it is nil bucket is in standard region
+				region = @@location_cache[name] || @@location_cache[name] = S3::Bucket.new(name.to_s, :owner => nil, :config => config).location_constraint || @@location_cache[name] = :standard
+
+				# no need to specify region if bucket is in standard region
+				return S3::Bucket.new(name.to_s, :owner => nil, :config => config) if region == :standard
+
+				# use same config but with region specified for buckets that are not DNS compatible or have dots and are not in standard region
+				S3::Bucket.new(name.to_s, :owner => nil, :config => config.with(region: region))
+			end
+		end
+	end
+end
+
+class S3StreamBackup
+	def initialize(&block)
+		instance_eval &block
+		cli_setup = @cli_setup
+		cli_verify_setup = @cli_verify_setup
+		settings = CLI.new do
+				option :key, 
+					short: :k,
+					description: 'S3 key',
+					default: ENV['AWS_ACCESS_KEY_ID']
+				option :secret, 
+					short: :s,
+					description: 'S3 key secret',
+					default_label: '<secret>',
+					default: ENV['AWS_SECRET_ACCESS_KEY']
+				option :prefix,
+					short: :p,
+					description: 'prefix under which the backup objects are kept',
+					default: ''
+				option :log_file,
+					short: :l,
+					description: 'location of log file; if not specifed log to STDERR'
+				switch :plain,
+					description: 'use plain connections instead of SSL to S3'
+				switch :verbose,
+					short: :v,
+					description: 'log debug messages'
+				switch :debug,
+					short: :d,
+					description: 'log AWS SDK debug messages'
+				argument :bucket,
+					description: 'name of bucket to upload data to'
+				argument :name,
+					description: 'name under which the object will be stored'
+				instance_eval &cli_setup if cli_setup
+		end.parse! do |settings|
+			fail 'AWS_ACCESS_KEY_ID environment not set and --key not used' unless settings.key
+			fail 'AWS_SECRET_ACCESS_KEY environment not set and --secret not used' unless settings.secret
+			instance_eval &cli_verify_setup if cli_verify_setup
+		end
+
+		log = Logger.new(settings.log_file ? settings.log_file : STDERR)
+		log.formatter = proc do |severity, datetime, progname, msg|
+			"[#{datetime.utc.strftime "%Y-%m-%d %H:%M:%S.%6N %Z"}] [#{$$}] #{severity}: #{msg.strip}\n"
+		end
+
+		log.level = Logger::INFO
+		log.level = Logger::DEBUG if settings.verbose or settings.debug
+
+		begin
+			s3 = AWS::S3.new(
+				access_key_id: settings.key,
+				secret_access_key: settings.secret,
+				logger: settings.debug ? log : nil,
+				log_level: :debug,
+				use_ssl: ! settings.plain
+			)
+			@main.call(settings, log, s3)
+		rescue => error
+			msg = "#{error.class.name}: #{error.message}\n#{error.backtrace.join("\n")}"
+			log.error msg
+			STDERR.write msg 
+			exit 10
+		end
+	end
+
+	def cli(&block)
+		@cli_setup = block
+	end
+
+	def cli_verify(&block)
+		@cli_verify_setup = block
+	end
+
+	def main(&block)
+		@main = block
+	end
+end
+
+class ProgeressLogger
+	def initialize
+		@bytes = 0
+	end
+
+	def log(logger, postfix)
+		if logger.debug?
+			was, @output_bytes = @output_bytes || '', @bytes.in_bytes_auto
+			logger.debug "#{@output_bytes}#{postfix}"
+		end
+	end
+
+	def <<(bytes)
+		@bytes += bytes
+	end
+
+	def to_s
+		@bytes.to_s
+	end
+
+	include Comparable
+	def <=>(value)
+		@bytes <=> value
+	end
+
+	def in_bytes_auto
+		@bytes.in_bytes_auto
+	end
+end
+

data/lib/s3streambackup/units.rb

@@ -0,0 +1,19 @@
+class Numeric
+	UNITS = %w{bytes KiB MiB GiB TiB PiB EiB}
+	def in_bytes_auto
+		size = self.to_f
+		units = UNITS.dup
+
+		while size > 999
+			size /= 1024
+			units.shift
+		end
+
+		if units.length == UNITS.length
+			"#{'%d' % size} #{units.first}"
+		else
+			"#{'%.1f' % size} #{units.first}"
+		end
+	end
+end
+

data/s3streambackup.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = "s3streambackup"
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jakub Pastuszek"]
-  s.date = "2013-08-05"
+  s.date = "2013-08-06"
   s.description = "Stores data from STDIN in S3 object using multipart upload and removes oldest backups to keep maximum desired backup object count."
   s.email = "jpastuszek@gmail.com"
-  s.executables = ["s3streambackup"]
+  s.executables = ["s3streambackup", "s3streamrestore"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -27,9 +27,12 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/s3streambackup",
+    "bin/s3streamrestore",
     "features/s3streambackup.feature",
     "features/step_definitions/s3streambackup_steps.rb",
     "features/support/env.rb",
+    "lib/s3streambackup.rb",
+    "lib/s3streambackup/units.rb",
     "s3streambackup.gemspec",
     "spec/s3streambackup_spec.rb",
     "spec/spec_helper.rb"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3streambackup
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-05 00:00:00.000000000 Z
+date: 2013-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cli
@@ -144,6 +144,7 @@ description: Stores data from STDIN in S3 object using multipart upload and remo
 email: jpastuszek@gmail.com
 executables:
 - s3streambackup
+- s3streamrestore
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
@@ -158,9 +159,12 @@ files:
 - Rakefile
 - VERSION
 - bin/s3streambackup
+- bin/s3streamrestore
 - features/s3streambackup.feature
 - features/step_definitions/s3streambackup_steps.rb
 - features/support/env.rb
+- lib/s3streambackup.rb
+- lib/s3streambackup/units.rb
 - s3streambackup.gemspec
 - spec/s3streambackup_spec.rb
 - spec/spec_helper.rb
@@ -179,7 +183,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1306132470382039118
+      hash: 4107987020335858105
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: