jemmyw-backs3 0.0.1

data/Manifest.txt

@@ -0,0 +1,17 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+example.conf
+bin/backs3
+bin/res3
+lib/backs3.rb
+lib/backs3/backs3.rb
+lib/backs3/backup.rb
+lib/backs3/restore.rb
+lib/backs3/version.rb
+spec/spec.opts
+spec/spec_helper.rb
+spec/backs3/backup_spec.rb
+spec/backs3/restore_spec.rb
+tasks/rspec.rake

data/README.txt

@@ -0,0 +1,51 @@
+= Backs3
+
+* http://github.com/jemmyw/backs3
+
+== DESCRIPTION:
+
+A simple backup and restore program for S3
+
+== FEATURES/PROBLEMS:
+
+* FIX (list of features or problems)
+
+== SYNOPSIS:
+
+  backs3 -c s3.conf test
+  res3 -c s3.conf available => [12345, 54321]
+  res3 -c s3.conf restore 12345
+
+== REQUIREMENTS:
+
+* ActiveSupport
+* AWS/S3
+
+== INSTALL:
+
+* sudo gem install jemmyw-backs3
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Jeremy Wells / Boost Limited
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/bin/backs3

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rubygems'
+  require 'backs3/backup'
+rescue LoadError => e
+  require 'lib/backs3/backup'
+end
+
+require 'active_support'
+require 'getoptlong'
+
+option_parser = GetoptLong.new(
+  ['--help', '-h', GetoptLong::NO_ARGUMENT],
+  ['--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  ['--exclude', GetoptLong::REQUIRED_ARGUMENT],
+  ['--id', '-i', GetoptLong::REQUIRED_ARGUMENT],
+  ['--key', '-k', GetoptLong::REQUIRED_ARGUMENT],
+  ['--bucket', '-b', GetoptLong::REQUIRED_ARGUMENT],
+  ['--prefix', '-p', GetoptLong::REQUIRED_ARGUMENT],
+  ['--full', '-f', GetoptLong::REQUIRED_ARGUMENT],
+  ['--force-full', GetoptLong::NO_ARGUMENT]
+)
+
+$options = {}
+
+def usage(message = nil)
+  $stderr.puts message unless message.blank?
+
+  name = $0.split('/').last
+  $stderr.puts <<"ENDUSAGE"
+  #{name} [options] <directory>
+  --help        -h
+  --config      -c      Configuration file
+  --id          -i      AWS Access Key ID
+  --key         -k      AWS Secret Key
+  --bucket      -b      AWS Bucket name
+  --full=d      -f d    Number of days between full backups (default: 7)
+  --force-full          Force a full backup
+  --prefix      -p
+  --exclude="regex" Exclude files based on regex
+
+ENDUSAGE
+
+  $stderr.puts "Current configuration:"
+  $options.each do |key, value|
+    $stderr.puts "  #{key}:    \t#{value}"
+  end
+
+  exit!
+end #usage
+
+begin
+  option_parser.each do |opt, arg|
+    $options[opt.gsub(/^-*/, '')] = (arg || true)
+  end
+
+  usage if $options['help']
+  $options['folder'] = ARGV[0] unless ARGV[0].blank?
+
+  raise Exception.new("Invalid configuration file #{$options['config']}") unless $options['config'].blank? || File.exists?($options['config'])
+  $options['config'] ||= '/etc/backs3.conf'
+
+  if File.exists?($options['config'])
+    begin
+      puts "Reading configuration from #{$options['config']}"
+      config = YAML::load_file($options['config'])
+      $options = config.merge($options)
+    rescue
+      raise Exception.new("Invalid configuration file #{$options['config']}")
+    end
+  end
+
+  raise Exception.new("You must specify a directory to backup") if $options['folder'].blank?
+  raise Exception.new("You must specify a bucket") if $options['bucket'].blank?
+  raise Exception.new("You must specify an AWS ID") if $options['id'].blank?
+  raise Exception.new("You must specify an AWS Secret Key") if $options['key'].blank?
+rescue Exception => e
+  usage(e.to_s)
+end
+
+backs3 = Backs3::Backup.new($options)
+backs3.backup

data/bin/res3

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rubygems'
+  require 'backs3/restore'
+rescue LoadError => e
+  require 'lib/backs3/restore'
+end
+
+require 'active_support'
+require 'getoptlong'
+
+option_parser = GetoptLong.new(
+  ['--help', '-h', GetoptLong::NO_ARGUMENT],
+  ['--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  ['--id', '-i', GetoptLong::REQUIRED_ARGUMENT],
+  ['--key', '-k', GetoptLong::REQUIRED_ARGUMENT],
+  ['--bucket', '-b', GetoptLong::REQUIRED_ARGUMENT],
+  ['--prefix', '-p', GetoptLong::REQUIRED_ARGUMENT]
+)
+
+$options = {}
+
+def usage(message = nil)
+  $stderr.puts message unless message.blank?
+
+  name = $0.split('/').last
+  $stderr.puts <<"ENDUSAGE"
+  #{name} [options] <directory> <command> [command options]
+  Commands:
+    #{Backs3::Restore.commands.join(", ")}
+
+  --help        -h
+  --config      -c      Configuration file
+  --id          -i      AWS Access Key ID
+  --key         -k      AWS Secret Key
+  --bucket      -b      AWS Bucket name
+  --prefix      -p
+
+ENDUSAGE
+
+  $stderr.puts "Current configuration:"
+  $options.each do |key, value|
+    $stderr.puts "  #{key}:    \t#{value}"
+  end
+
+  exit!
+end #usage
+
+begin
+  option_parser.each do |opt, arg|
+    $options[opt.gsub(/^-*/, '')] = (arg || true)
+  end
+
+  usage if $options['help']
+
+  raise Exception.new("Invalid configuration file #{$options['config']}") unless $options['config'].blank? || File.exists?($options['config'])
+  $options['config'] ||= '/etc/backs3.conf'
+
+  if File.exists?($options['config'])
+    begin
+      puts "Reading configuration from #{$options['config']}"
+      config = YAML::load_file($options['config'])
+      $options = config.merge($options)
+    rescue
+      raise Exception.new("Invalid configuration file #{$options['config']}")
+    end
+  end
+
+  args = ARGV
+  $options['folder'] = args.shift if $options['folder'].blank?
+  $command = args.shift
+
+  raise Exception.new("You must specify a directory to restore from") if $options['folder'].blank?
+  raise Exception.new("You must specify a bucket") if $options['bucket'].blank?
+  raise Exception.new("You must specify an AWS ID") if $options['id'].blank?
+  raise Exception.new("You must specify an AWS Secret Key") if $options['key'].blank?
+  raise Exception.new("You must specify a valid command") unless Backs3::Restore.commands.include?($command)
+rescue Exception => e
+  usage(e.to_s)
+end
+
+res3 = Backs3::Restore.new($options)
+res3.send($command, *args)

data/example.conf

@@ -0,0 +1,5 @@
+---
+id: xxx
+key: xxx
+folder: xxx
+bucket: xxx

data/lib/backs3/backs3.rb

@@ -0,0 +1,39 @@
+# #!/usr/bin/env ruby
+
+require 'rubygems' rescue nil
+require 'aws/s3'
+require 'active_support'
+require 'digest/md5'
+require 'time'
+
+$has_md5 = !(`which md5`).blank?
+
+module Backs3
+  include AWS::S3
+
+  def establish_connection
+    AWS::S3::Base.establish_connection!(
+      :access_key_id => @options['id'],
+      :secret_access_key => @options['key']
+    )
+  end
+
+  def md5(filename)
+    if $has_md5
+      `md5 #{filename}`
+    else
+      Digest::MD5.hexdigest(filename)
+    end
+  end
+
+  def backup_info
+    @backup_info ||= begin
+      backup_info_file = S3Object.find(@options['prefix'] + 's3backup', @options['bucket'])
+      backup_info_data = backup_info_file.value(:reload)
+      YAML.load(backup_info_data) || {}
+    rescue Exception => e
+      puts e.to_s
+      {}
+    end
+  end
+end

data/lib/backs3/backup.rb

@@ -0,0 +1,82 @@
+require File.dirname(__FILE__) + '/backs3'
+
+module Backs3
+  class Backup
+    include Backs3
+    include AWS::S3
+
+    def initialize(options = {})
+      @options = options
+      @options['prefix'] ||= ''
+    end
+
+    def first_backup?
+      last_backup.nil?
+    end
+
+    def full_backup
+      @options['force-full'] || first_backup? || Time.now.to_i - last_backup > (@options['full'] || 7).days
+    end
+
+    def last_backup
+      backup_info['last_backup'].to_i
+    end
+
+    def backup_key
+      @options['prefix'] + @current_backup
+    end
+
+    def last_key
+      @options['prefix'] + last_backup.to_s
+    end
+
+    def files_to_backup
+      @files_to_backup ||= begin
+        Dir.glob(File.join(@options['folder'], '**', '**')).select do |file|
+          if @options['exclude'].blank? || file !~ /#{@options['exclude']}/
+            if full_backup || File.mtime(file).to_i > last_backup
+              true
+            else
+              false
+            end
+          else
+            false
+          end
+        end
+      end
+    end
+
+    def backup
+      @current_backup = Time.now.to_i.to_s
+      @files_to_backup = nil
+      @backup_info = nil
+
+      establish_connection
+
+      puts "Backup started at #{Time.now}"
+      puts "Last backup happened at #{backup_info['last_backup']}"
+      puts "Performing %s" % (full_backup ? "full backup" : "incremental backup")
+
+      files_to_backup.each do |filename|
+        puts "Backing up #{filename}"
+        file_md5 = md5(filename)
+        aws_filename = File.join(backup_key, filename)
+
+        object = S3Object.find(aws_filename, @options['bucket']) rescue nil
+
+        if object.nil? || object.metadata[:md5sum] != file_md5
+          S3Object.store(aws_filename, open(filename), @options['bucket'])
+          object = S3Object.find(aws_filename, @options['bucket'])
+          object.metadata[:md5sum] = file_md5
+          object.save
+        end
+      end
+
+      backup_info['last_backup'] = @current_backup
+      backup_info['backups'] ||= []
+      backup_info['backups'] << @current_backup
+      S3Object.store(@options['prefix'] + 's3backup', YAML.dump(backup_info), @options['bucket'])
+      puts "Backup completed, #{files_to_backup.size} files backed up"
+    end
+  end
+end

data/lib/backs3/restore.rb

@@ -0,0 +1,89 @@
+require File.dirname(__FILE__) + '/backs3'
+require 'fileutils'
+
+module Backs3
+  class Restore
+    include Backs3
+    include AWS::S3
+
+    def self.commands
+      %w(ls available restore cat)
+    end
+
+    def initialize(options = {})
+      @options = options
+      @options['prefix'] ||= ''
+      establish_connection
+    end
+
+    def available(file = nil)
+      if file.nil?
+        puts "Backups available: #{backup_info['backups'].join(", ")}"
+      else
+        info = backup_info['backups'].collect do |backup|
+          backup_key = @options['prefix'] + backup.to_s
+          object = S3Object.find(File.join(backup_key, file), @options['bucket']) rescue nil
+
+          if object
+            {
+              :backup => backup,
+              :md5 => object.metadata[:md5sum]
+            }
+          else
+            nil
+          end
+
+        end.compact
+
+        puts "Backup information for #{file}:"
+        info.each do |i|
+          puts "\tBackup #{i[:backup]}: #{i[:md5]}"
+        end
+
+      end
+    end
+
+    def ls(backup)
+      Bucket.objects(@options['bucket'], :prefix => @options['prefix'] + backup.to_s).each do |object|
+        puts object.key
+      end
+    end
+
+    def cat(backup, file)
+      backup_key = @options['prefix'] + backup.to_s
+      object = S3Object.find(File.join(backup_key, file), @options['bucket']) rescue nil
+      
+      if object.nil?
+        puts "Cannot find file #{file}"
+      else
+        puts object.value(:reload)
+      end
+    end
+
+    def restore(backup, file = nil)
+      backup_key = @options['prefix'] + backup.to_s
+      objects = []
+
+      if file.nil?
+        objects = Bucket.objects(@options['bucket'], :prefix => @options['prefix'] + backup.to_s)
+      else
+        objects << S3Object.find(File.join(backup_key, file), @options['bucket']) rescue nil
+      end
+
+      objects.compact!
+
+      objects.each do |object|
+        $stdout.write "Restoring file #{object.key} to /tmp/#{object.key}"
+        filename = "/tmp/#{object.key}"
+        FileUtils.mkdir_p File.dirname(filename)
+        File.open(filename, 'w') do |f|
+          object.value do |segment|
+            $stdout.write "."
+            f.write segment
+          end
+        end
+        $stdout.write "\n"
+      end
+    end
+  end
+end

data/lib/backs3/version.rb

@@ -0,0 +1,9 @@
+module Backs3 #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/backs3.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/backs3/backs3'

data/spec/backs3/backup_spec.rb

@@ -0,0 +1,73 @@
+require File.dirname(__FILE__) + "/../spec_helper"
+require 'backs3/backup'
+include Backs3
+
+describe Backup do
+  before(:each) do
+    AWS::S3::Base.stub!(:establish_connection!)
+    @bucket = 'test_bucket'
+    @backup = Backup.new({'bucket' => @bucket})
+  end
+
+  describe 'backup' do
+    it 'should backup all the files returned by files_to_backup'
+  end
+
+  describe 'first_backup?' do
+    it 'should be true if there is no previous backup' do
+      @backup.should_receive(:last_backup).and_return(nil)
+      @backup.first_backup?.should be_true
+    end
+
+    it 'should be false if there is a previous backup' do
+      @backup.should_receive(:last_backup).and_return(12345)
+      @backup.first_backup?.should be_false
+    end
+  end
+
+  describe 'full_backup' do
+    it 'should be true if force-full option is set' do
+      @backup = Backup.new('force-full' => true)
+      @backup.should_not_receive(:last_backup)
+      @backup.should_not_receive(:first_backup?)
+      @backup.full_backup.should be_true
+    end
+    it 'should be true if this is the first backup' do
+      @backup.should_receive(:first_backup?).and_return(true)
+      @backup.full_backup.should be_true
+    end
+    it 'should be true if the last backup was more than 7 days ago' do
+      @backup.should_receive(:first_backup?).and_return(false)
+      @backup.should_receive(:last_backup).and_return((Time.now - 8.days).to_i)
+      @backup.full_backup.should be_true
+    end
+    it 'should be false' do
+      @backup.should_receive(:first_backup?).and_return(false)
+      @backup.should_receive(:last_backup).and_return(Time.now.to_i)
+      @backup.full_backup.should be_false
+    end
+  end
+
+  describe 'last_backup' do
+    it 'should return the integer time of the last backup' do
+      time = Time.now
+      time.should_receive(:to_i).and_return(12345)
+      @backup.should_receive(:backup_info).and_return({'last_backup' => time})
+      @backup.last_backup.should == 12345
+    end
+  end
+
+  describe 'backup_key' do
+    it 'should return the prefix + the backup time'
+  end
+
+  describe 'last_key' do
+    it 'should return the prefix + the last backup time'
+  end
+
+  describe 'files_to_backup' do
+    it 'should return all the files in the backup folder'
+    it 'should not return excluded files'
+    it 'should not return files that have not changed if this is an incremental backup'
+  end
+end

data/spec/backs3/restore_spec.rb

@@ -0,0 +1,54 @@
+require File.dirname(__FILE__) + "/../spec_helper"
+require 'backs3/restore'
+
+include Backs3
+
+describe Restore do
+  before(:each) do
+    AWS::S3::Base.stub!(:establish_connection!)
+    @bucket = 'test_bucket'
+    @restore = Restore.new('bucket' => @bucket)
+
+    @file_1 = mock(:s3object, :metadata => {})
+  end
+
+  describe 'self.commands' do
+    it 'should return an array' do
+      Restore.commands.should be_a(Array)
+    end
+  end
+
+  describe 'available' do
+    it 'should list all of the backups available if no file is specified' do
+      @restore.should_receive(:backup_info).and_return({'backups' => [12345, 54321]})
+      @restore.should_receive(:puts).with('Backups available: 12345, 54321')
+      @restore.available
+    end
+    it 'should list all of the backups a file is in' do
+      file = 'test/file_1'
+      @restore.should_receive(:backup_info).and_return({'backups' => [12345, 54321]})
+      S3Object.should_receive(:find).with('12345/test/file_1', @bucket).and_return(@file_1)
+      S3Object.should_receive(:find).with('54321/test/file_1', @bucket).and_return(nil)
+
+      @restore.should_receive(:puts).with('Backup information for test/file_1:')
+      @restore.should_receive(:puts).with("\tBackup 12345: ")
+      @restore.should_not_receive(:puts).with("\tBackup 54321: ")
+
+      @restore.available(file)
+    end
+  end
+
+  describe 'ls' do
+    it 'should list all of the files in a directory'
+  end
+
+  describe 'cat' do
+    it 'should show an error if the file specified does not exist'
+    it 'should output the contents of a file'
+  end
+
+  describe 'restore' do
+    it 'should restore a whole backup if no file is specified'
+    it 'should restore a file'
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  gem 'rspec'
+  require 'spec'
+end
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'backs3'
+
+include Spec::Matchers
+
+module Kernel
+  def logger
+    @@__log_file__ ||= StringIO.new
+    @@__log__ = ActiveSupport::BufferedLogger.new @@__log_file__
+  end
+end

data/tasks/rspec.rake

@@ -0,0 +1,21 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  require 'spec'
+end
+begin
+  require 'spec/rake/spectask'
+rescue LoadError
+  puts <<-EOS
+To use rspec for testing you must install rspec gem:
+    gem install rspec
+EOS
+  exit(0)
+end
+
+desc "Run the specs under spec"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+name: jemmyw-backs3
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jeremy Wells
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-25 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.5.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.0
+    version: 
+description: S3 backup and restore program
+email: 
+- jeremy@boost.co.nz
+executables: 
+- backs3
+- res3
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- example.conf
+- bin/backs3
+- bin/res3
+- lib/backs3.rb
+- lib/backs3/backs3.rb
+- lib/backs3/backup.rb
+- lib/backs3/restore.rb
+- lib/backs3/version.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/backs3/backup_spec.rb
+- spec/backs3/restore_spec.rb
+- tasks/rspec.rake
+has_rdoc: true
+homepage: http://backs3.rubyforge.org
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: backs3
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: S3 backup and restore program
+test_files: []
+