tens3 0.1

data/README

@@ -0,0 +1,46 @@
+tens3 : dead simple s3 backups
+
+* uses s3 to backup a directory of files
+* uses fadvise to be easy on filesystem caches and disks
+* purges files after X days
+* streams files rather than loading them entirely into memory
+
+
+license:
+
+This code is available as Open Source Software under the MIT license.
+
+
+ruby dependencies:
+
+fadvise
+right_aws
+
+
+configuration:
+
+tens3.yml:
+
+amazon_access_key_id: "someid"
+amazon_secret_access_key: "somekey"
+backup_dir: "/some/path/"
+purge_threshold: 3
+bucket_name: "somebucket"
+
+
+usage:
+
+backup a directory of files:
+
+$ ./tens3_put tens3.conf
+
+restore a file from a backup:
+
+$ ./tens3_get tens3.conf date somefile ./somefile
+
+The date is the date that the file was backed up in a YYYYMMDD format.
+
+
+caveats:
+
+The script will only backup files at one directory level, it does not recurse into subdirectories.

data/bin/tens3_get

@@ -0,0 +1,60 @@
+#! /usr/bin/ruby
+
+## Copyright 2009, Joe Williams <joe@joetify.com>
+##
+## Permission is hereby granted, free of charge, to any person
+## obtaining a copy of this software and associated documentation
+## files (the "Software"), to deal in the Software without
+## restriction, including without limitation the rights to use,
+## copy, modify, merge, publish, distribute, sublicense, and/or sell
+## copies of the Software, and to permit persons to whom the
+## Software is furnished to do so, subject to the following
+## conditions:
+##
+## The above copyright notice and this permission notice shall be
+## included in all copies or substantial portions of the Software.
+##
+## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+## EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+## OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+## NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+## HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+## WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+## OTHER DEALINGS IN THE SOFTWARE.
+
+
+require 'rubygems'
+require 'right_aws'
+require 'socket'
+require 'fadvise'
+
+tens3_config = YAML.load(File.open(ARGV[0]))
+
+AWS_KEY = tens3_config['amazon_access_key_id']
+AWS_SECRET = tens3_config['amazon_secret_access_key']
+BUCKET_NAME = tens3_config['bucket_name']
+
+S3 = RightAws::S3Interface.new(AWS_KEY, AWS_SECRET)
+def main
+  if ARGV.length != 4
+    puts "tens3_get usage:\n# tens3_get CONFIG BACKUPDATE REMOTEFILENAME LOCALFILEWITHPATH"
+  else
+    s3 = RightAws::S3Interface.new(AWS_KEY, AWS_SECRET)
+    remote_file_name = "#{ARGV[1]}_#{ARGV[2]}"
+    local_file_name = "#{ARGV[3]}"
+
+    puts "[ #{Time.now} ] IFNO :: Creating local file at #{local_file_name}"
+    file = File.open(local_file_name, File::CREAT|File::RDWR)
+    file.fadvise(0, 0, :dont_need)
+
+    puts "[ #{Time.now} ] INFO :: Streaming #{remote_file_name} from S3 to #{local_file_name}"
+    rhdr = s3.get(BUCKET_NAME, remote_file_name) do |chunk|
+      file.write(chunk)
+    end
+
+    file.close
+  end
+end
+
+main

data/bin/tens3_put

@@ -0,0 +1,104 @@
+#! /usr/bin/ruby
+
+## Copyright 2009, Joe Williams <joe@joetify.com>
+##
+## Permission is hereby granted, free of charge, to any person
+## obtaining a copy of this software and associated documentation
+## files (the "Software"), to deal in the Software without
+## restriction, including without limitation the rights to use,
+## copy, modify, merge, publish, distribute, sublicense, and/or sell
+## copies of the Software, and to permit persons to whom the
+## Software is furnished to do so, subject to the following
+## conditions:
+##
+## The above copyright notice and this permission notice shall be
+## included in all copies or substantial portions of the Software.
+##
+## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+## EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+## OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+## NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+## HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+## WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+## OTHER DEALINGS IN THE SOFTWARE.
+
+
+require 'rubygems'
+require 'right_aws'
+require 'socket'
+require 'fadvise'
+require 'time'
+require 'ftools'
+
+tens3_config = YAML.load(File.open(ARGV[0]))
+
+BACKUP_DIR = tens3_config['backup_dir']
+AWS_KEY = tens3_config['amazon_access_key_id']
+AWS_SECRET = tens3_config['amazon_secret_access_key']
+PURGE_THRESHOLD = tens3_config['purge_threshold']
+BUCKET_NAME = tens3_config['bucket_name']
+
+S3 = RightAws::S3Interface.new(AWS_KEY, AWS_SECRET)
+
+def file_list
+  ls = Dir.entries(BACKUP_DIR)
+  ls.delete(".")
+  ls.delete("..")
+  ls
+end
+
+def backup
+  puts "[ #{Time.now} ] INFO :: Creating bucket #{BUCKET_NAME}"
+  S3.create_bucket(BUCKET_NAME)
+
+  file_list.each do |file|
+    puts "[ #{Time.now} ]"
+    full_path = "#{BACKUP_DIR}#{file}"
+
+    unless File.directory?(full_path)
+      backup_file = File.open(full_path, 'rb')
+      backup_file.fadvise(0, file.size, :sequential)
+
+      puts "[ #{Time.now} ] INFO :: Calculating SHA1 for #{full_path}"
+      sha1_output = `sha1sum #{full_path}`
+      sha1 = sha1_output.split[0]
+
+      puts "[ #{Time.now} ] INFO :: Backing up #{full_path} (#{File.size(full_path)} bytes) - #{sha1}"
+      S3.put(BUCKET_NAME, "#{Time.now.strftime("%Y%m%d")}_#{file}", backup_file)
+
+      puts "[ #{Time.now} ] INFO :: #{Time.now.strftime("%Y%m%d")}_#{file} has been uploaded to bucket #{BUCKET_NAME}"
+    end
+  end
+end
+
+def purge
+  today = Time.now
+  seconds_in_day = 86400
+  bucket_list = S3.list_bucket(BUCKET_NAME)
+  purge_list = Hash.new
+
+  bucket_list.each do |file|
+    purge_list.store(file[:key], file[:last_modified])
+  end
+
+  purge_list.each do |key, value|
+    time_diff = (today - Time.parse(value))/seconds_in_day
+    if time_diff > PURGE_THRESHOLD
+      puts "[ #{Time.now} ] INFO :: #{key} is #{time_diff} days old (>#{PURGE_THRESHOLD}), purging from s3"
+      S3.delete(BUCKET_NAME, key)
+    end
+  end
+
+end
+
+def main
+  if ARGV.length != 1
+    puts "tens3_put usage:\n# tens3_put CONFIG"
+  else
+    backup
+    purge
+  end
+end
+
+main

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification 
+name: tens3
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- joe williams
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-03 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fadvise
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: joe@joetify.com
+executables: 
+- tens3_put
+- tens3_get
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- bin/tens3_get
+- bin/tens3_put
+- README
+has_rdoc: true
+homepage: http://github.com/joewilliams/tens3
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: easy backups to s3
+test_files: []
+