s3snapshot 0.0.2

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+/pkg
+/testcreds.txt
+/html

data/.loadpath

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<loadpath>
+	<pathentry path="" type="src"/>
+	<pathentry path="org.rubypeople.rdt.launching.RUBY_CONTAINER" type="con"/>
+</loadpath>

data/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>s3snapshot</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.rubypeople.rdt.core.rubybuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.rubypeople.rdt.core.rubynature</nature>
+	</natures>
+</projectDescription>

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in s3snapshot.gemspec
+gemspec

data/Gemfile.lock.bak

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    s3snapshot (0.0.1)
+      fog (~> 0.8.1)
+      thor (~> 0.14.6)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    builder (3.0.0)
+    excon (0.6.3)
+    fog (0.8.1)
+      builder
+      excon (~> 0.6.1)
+      formatador (>= 0.1.3)
+      json
+      mime-types
+      net-ssh (>= 2.1.3)
+      nokogiri (>= 1.4.4)
+      ruby-hmac
+    formatador (0.1.3)
+    json (1.5.1)
+    mime-types (1.16)
+    net-ssh (2.1.4)
+    nokogiri (1.4.4)
+    ruby-hmac (0.4.0)
+    thor (0.14.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  s3snapshot!

data/Rakefile

@@ -0,0 +1,11 @@
+gem 'rdoc'
+require 'bundler'
+require 'rdoc/task'
+
+Bundler::GemHelper.install_tasks
+
+
+RDoc::Task.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+end

data/Readme.markdown

@@ -0,0 +1,44 @@
+# Overview
+
+This gem is designed to sync an immutable directory to a timestamped prefix on Amazon S3.  In principal it is similar to time machine for Mac.  We use this utility to create snapshots of our Cassandra data.
+
+# Installing
+
+		gem install s3snapshot
+
+# Operations
+
+* Backup a directory
+* Restore a specific snapshot time to a directory
+* Restore latest complete snapshot to a directory
+* List all prefixes
+* List all times for prefixes
+* Clean incomplete uploads (Use wisely, can delete a backup in progress)
+* Perform rolling cleanup.  Can keep a user defined number of daily and weekly backups with user specified day.  Ally days and weeks are deltas calculated from the timestamp of the last successful backup to s3.
+
+# Algorithm
+
+Below is a general outline on how the plugin was designed to work.  No meta data is stored on S3.  Every time this plugin is launched it performs an analysis of S3 to ensure it is always using a correct state of backups.  2 instances should never access the same prefix concurrently, this could cause issues with data consistency.
+
+## Snapshot path
+
+All snapshot paths are of the format [prefix]/[snapshot utc time].  In our usage, we typically use the prefix of node+directory, such as test-cass-west-1_snapshot
+
+## Rolling cleanup
+
+Capture the current time UTC, and truncate to 00:00 hours
+
+Remove all incomplete backups before start time
+
+Analyze all time stamps, if more than one backup is present per day, only keep the latest complete backup for that day
+
+For all backups, if older than the max weekly, remove it , of between oldest weekly and oldest daily, only keep it if it falls on the day specified
+
+
+# Notes
+
+Occasionally a cleanup operation will miss a time stamp.  From my testing this appears to be due to the amazon eventual consistency, and the timestamp not being returned on a delim search.  On the next run it is usually deleted
+
+#Future features
+
+* Use native s3 copying for unmodified files (similar to s3sync).  If a file exists in the previous completed backup and has not changed, copy it on the new path in s3, this will avoid the need to upload unmodified files.

data/bin/s3snapshot

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 's3snapshot/cli'
+S3snapshot::CLI.start

data/lib/s3snapshot/backup_manager.rb

@@ -0,0 +1,280 @@
+require 's3snapshot/time_factory'
+require 'dictionary'
+
+module S3snapshot
+  #
+  #Handles retrieving all current backups and performing operations on them.  This object is stateful, once snapshots are loaded it is cached.
+  #Create a new instance or call clear_snapshots to force a reload from aws
+  #
+  class BackupManager < SyncOp
+    
+    #Number of seconds in a day
+    SECONDS_DAY = 60*60*24
+    SECONDS_WEEK = SECONDS_DAY * 7
+    
+    #Instance variable of snapshots by prefix.  This is cached because loading this information is expensive and slow
+    @snapshots = nil
+    
+    
+    def initialize(aws_id, aws_key, bucket_name)
+      super(aws_id, aws_key, bucket_name)
+    end
+    
+    ##
+    #Get all prefixes in the bucket
+    #
+    def prefixes
+      bucket.files.all(:delimiter => "/").common_prefixes
+    end
+    
+    ##
+    # returns a map of snapshots.  The key is the time, the value is a boolean signaling if it's complete
+    ##
+    def snapshots(prefix)
+      
+      prefix_snap = get_snapshot(prefix)
+      
+      unless prefix_snap.nil?
+        return prefix_snap
+      end
+      
+      prefix_snap = Dictionary.new
+      
+      timestamps(prefix).each do |timestamp|
+        
+        time = Time.parse(timestamp)
+        
+        prefix_snap[time]  = read_complete?(prefix, time)
+        
+      end
+      
+      set_snapshot(prefix, prefix_snap)
+      
+      prefix_snap
+      
+    end
+    
+    ##
+    #Get the latest completed backup for the given prefix. Will return nil if one isn't available  
+    ## 
+    def latest(prefix)
+      snapshots(prefix).each do  |time, complete|
+        if complete
+          return time
+        end
+      end
+      
+      nil
+    end
+    
+    ##
+    #clear the local cached copy of the snapshot
+    #
+    def clear_snapshots(prefix)
+      @snapshots[prefix] = nil
+    end
+    
+    
+    
+    #Returns true if the backup is complete, false otherwise
+    def complete?(prefix, time)
+      value = snapshots(prefix)[time]
+      value.nil? ? false : value
+    end
+    
+    ##
+    # Returns true if the backup exists
+    #
+    def exists?(prefix, time)
+      !snapshots(prefix)[time].nil?
+      #      backups = bucket.files.all(:prefix => timepath(prefix, time))
+      #      
+      #      !backups.nil? && backups.length > 0
+    end
+    
+    
+    ##
+    #Removes all incomplete backups.  Use wisely, will blitz a backup in progress
+    ##
+    def clean(prefix)
+      snapshots(prefix).each do |time, complete|
+        unless complete
+          remove(prefix, time)
+        end
+      end
+    end
+    
+    #
+    #Delete all files from a snapshot.  Will remove the complete file first to avoid other clients using the backup
+    #
+    def remove(prefix, timestamp)
+      complete_marker =  bucket.files.get(complete_path(prefix, timestamp))
+      
+      #Destroy the complete marker.  This prevents other clients from thinking this backup is complete when the s3 bucket is read
+      unless complete_marker.nil?
+        complete_marker.destroy
+      end
+      
+      files = list_files(prefix, timestamp)
+      
+      files.each do |file|
+        file.destroy
+      end
+      
+      get_snapshot(prefix).delete(timestamp)
+      
+    end
+    
+    
+    ##
+    # Return all files that exist in this backup bucket with the given time
+    ##
+    def list_files(prefix, time)
+      bucket.files.all(:prefix => timepath(prefix, time) )
+    end
+    
+    
+    
+    ##
+    #Perform a rolling delete for the given prefix.  Keeps the "newest" daily backup for the given day, and keeps a backup for the day of the week specified by day of week.  day_of_week follows cron style syntax, 0 = sunday and 6 = saturday
+    # 
+    def roll(prefix, num_days, num_weeks, day_of_week)
+      
+      start =  TimeFactory.utc_time
+      
+      start = Time.utc(start.year, start.month, start.day)
+      
+      clean(prefix)
+      
+      merge_days(prefix, start)
+      
+      
+      snaps = snapshots(prefix)
+      
+      #Nothing to do
+      if snaps.nil? ||snaps.empty?
+        return
+      end
+      
+      newest_time =snaps.keys.last
+      
+      #Truncate the oldest daily to 00 hours minutes and seconds based on the "newest" completed backup after the merge
+      oldest_daily = newest_time - SECONDS_DAY*num_days
+      
+      oldest_daily = Time.utc(oldest_daily.year, oldest_daily.month, oldest_daily.day)
+      
+      #Truncate the oldest weekly to 00 hours minutes and seconds
+      oldest_weekly = newest_time - SECONDS_WEEK*num_weeks
+      
+      oldest_weekly = Time.utc(oldest_weekly.year, oldest_weekly.month, oldest_weekly.day)
+      
+      
+      #Now iterate over every day and keep the number of days.  After that only keep the value that's on the number of weeks
+      snaps.each do |time, complete|
+        
+        #We're done, we've fallen into the day range
+        if time >= oldest_daily
+          break
+        end
+        
+        #Is older than the oldest daily,or not the right day of the week so should be deleted
+        if time < oldest_weekly || !same_day(time, day_of_week)
+          remove(prefix, time)
+        end
+        
+      end
+    end
+    
+    
+    private
+    
+    
+    #Returns true if the backup is complete, false otherwise. Downloads the file so can be slow
+    def read_complete?(prefix, time)
+      
+      found_prefixes = bucket.files.all(:prefix => timepath(prefix, time), :delimiter => COMPLETE_EXTENSION).common_prefixes
+      
+      
+      !found_prefixes.nil? && found_prefixes.length > 0
+    end
+    
+    ##
+    # Returnes true if the time occurs on the same day_of_week.  day_of_week follows cron style syntax, 0 = sunday and 6 = saturday
+    #
+    def same_day(time, day_of_week)
+      
+      unless day_of_week > -1 && day_of_week < 7
+        raise "Invalid day of week. Expected 0-6 but received #{day_of_week}"
+      end
+      
+      time.wday == day_of_week
+      
+    end
+    
+    
+    ##
+    #Iterates over all snapshots and removes any duplicates for a day.  Only keep the latest for a day
+    ##
+    def merge_days(prefix, start)
+      #Use "yesterday" as a starting point
+      previous = nil
+      
+      snapshots(prefix).each do |time, complete|
+        #Skip anything that's before the "start" time above
+        if time > start || !complete
+          next  
+        end
+        
+        #2 backups on the same day, keep the oldest
+        if samedate?(previous, time)  
+          if(previous.to_i > time.to_i)
+            remove(prefix, time)
+          else
+            remove(prefix, previous)
+            previous = time
+          end
+          
+          next
+        end
+        
+        previous = time
+        
+      end
+    end
+    
+    #returns true if the first and second time occur in the same date (assumes utc time)
+    def samedate?(first, second)
+   #   puts "first: #{first};#{first.nil? ? "nil": first.yday}, second #{second};#{second.nil? ? "nil": second.yday}"
+      !first.nil? && !second.nil? && first.yday == second.yday
+    end
+    
+    def timestamps(prefix)
+      bucket.files.all(:prefix => prefix_string(prefix), :delimiter => "/").common_prefixes
+    end
+    
+    def prefix_string(prefix)
+      "#{prefix}/"
+    end
+    
+    
+    ##
+    # Get a snapshot by prefix
+    ##
+    def get_snapshot(prefix)
+      @snapshots.nil? ? nil : @snapshots[prefix]
+    end
+    
+    ##
+    #Set the snapshot into the context
+    ##
+    def set_snapshot(prefix, snaphash)
+      if @snapshots.nil?
+        @snapshots = Hash.new
+      end
+      
+      @snapshots[prefix] = snaphash
+    end
+    
+    
+  end
+end

data/lib/s3snapshot/cli.rb

@@ -0,0 +1,155 @@
+require 'thor'
+require 's3snapshot/dir_upload'
+require 's3snapshot/dir_download'
+require 's3snapshot/backup_manager'
+
+module S3snapshot
+  class CLI < Thor
+    
+    
+    desc "backup", "upload a directory as a snapshot backup to s3"
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :directory, :aliases => "-d", :desc => "The directory to upload", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "A prefix to prepend to the path before the timestamp.  Useful in cluster to specifiy a node name, or a node+directory scheme.  Prefix strategies can be mixed in a bucket, they must just be unique." , :type => :string,  :required => true
+    
+    ##
+    #Uploads the directory to the s3 bucket with a prefix
+    def backup
+      directory = options[:directory]
+      puts "You are uploading directory #{directory}"
+      
+      s3upload = DirUpload.new(options[:awsid], options[:awskey],  options[:bucket],   options[:prefix], directory )
+      s3upload.upload
+    end
+    
+    desc "restore", "restore all files from a snapshot to a directory"
+    
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "The prefix to prepend to before searching for snapshots" , :type => :string,  :required => true
+    method_option :time, :aliases => "-t", :desc => "The timestamp to restore" , :type => :string,  :required => true
+    method_option :dest, :aliases => "-d", :desc => "The destination directory for downloaded files" , :type => :string,  :required => true
+    
+    
+    def restore
+      time = Time.parse(options[:time])
+      download = DirDownload.new(options[:awsid], options[:awskey], options[:bucket], options[:prefix], time, options[:dest])
+      download.download
+    end
+    
+    
+    desc "restorelatest", "restore the latest snapshot to a directory"
+    
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "The prefix to prepend to before searching for snapshots" , :type => :string,  :required => true
+    method_option :dest, :aliases => "-d", :desc => "The destination directory for downloaded files" , :type => :string,  :required => true
+    
+    
+    def restorelatest
+      
+      manager = BackupManager.new(options[:awsid], options[:awskey], options[:bucket])
+      latest_complete = manager.latest(options[:prefix])
+      
+      if latest_complete.nil?
+        puts "Cannot find a complete snapshot with prefix #{options[:prefix]}"
+        exit 1
+      end
+            
+      download = DirDownload.new(options[:awsid], options[:awskey], options[:bucket], options[:prefix], latest_complete, options[:dest])
+      download.download
+    end
+    
+    desc "prefixes", "list all prefixes in an s3 bucket"
+    
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    
+    def prefixes
+      manager = BackupManager.new(options[:awsid], options[:awskey], options[:bucket])
+      
+      puts "Found the following prefixes\n\n"
+      
+      manager.prefixes.each do |prefix|
+        puts prefix[0..-2]
+      end
+      
+      puts "\n"
+    end
+    
+    
+    desc "snapshots", "list all snapshots for a prefix in an s3 bucket"
+    
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "The prefix to prepend to before searching for snapshots" , :type => :string,  :required => true
+    
+    
+    def snapshots
+      manager = BackupManager.new(options[:awsid], options[:awskey], options[:bucket])
+      
+      snap_map = manager.snapshots(options[:prefix])
+      
+      puts "Found the following timestamps from prefix #{options[:prefix]}\n\n"
+      
+      snap_map.each do |key, value|
+        result = value ? "complete" : "unknown"
+        
+        puts "Time: #{key.iso8601}, Status: #{result}"
+      end
+      
+      puts "\n"
+      
+    end
+    
+    
+    desc "clean", "Remove all snapshots in the prefix that do not have a complete status.  Use wisely, could remove backups in progress for a prefix causing corruption"
+    
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "The prefix to prepend to before searching for snapshots" , :type => :string,  :required => true
+    
+    
+    def clean
+      manager = BackupManager.new(options[:awsid], options[:awskey], options[:bucket])
+      
+      manager.clean(options[:prefix])
+      
+    end
+    
+    desc "roll", "Analyze all snapshots and keep the latest daily for each day.  Then keep n days specified from the command line, and n weeks for the specified day of week"
+    
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "The prefix to prepend to before searching for snapshots" , :type => :string,  :required => true
+    method_option :numdays, :aliases => "-n", :desc => "The number of days to keep" , :type => :numeric,  :required => true
+    method_option :numweeks, :aliases => "-w", :desc => "The number of weeks to keep" , :type => :numeric,  :required => true
+    method_option :dayofweek, :aliases => "-o", :desc => "The day of week to keep. 0 based like cron 0 = sunday 6 = saturday" , :type => :numeric,  :required => true
+    
+    
+    
+    def roll
+      manager = BackupManager.new(options[:awsid], options[:awskey], options[:bucket])
+      
+      manager.roll(options[:prefix], options[:numdays], options[:numweeks], options[:dayofweek])
+      
+    end
+    
+    
+  end
+end

data/lib/s3snapshot/dir_download.rb

@@ -0,0 +1,81 @@
+require 'fog'
+require 's3snapshot/sync_op'
+require 's3snapshot/backup_manager'
+require 'time'
+require 'fileutils'
+
+module S3snapshot
+  class DirDownload < SyncOp
+    
+    @time = nil
+    @local_dir = nil
+    @prefix = nil
+    
+    
+    def initialize(aws_id, aws_key, bucket_name, prefix, time, local_dir )
+      super(aws_id, aws_key, bucket_name)
+      @prefix = prefix
+      @time = time
+      @local_dir = local_dir
+    end
+    
+    def download
+      
+      
+      #Check if the backup is complete
+      manager = BackupManager.new(@aws_id, @aws_key, @bucket_name)
+      
+      unless manager.exists?(@prefix, @time)
+        $stderr.puts "Backup with prefix '#{@prefix}' and time #{@time.iso8601} does not exist.  Please check the prefix and time"
+        return
+      end
+      
+      unless manager.complete?(@prefix, @time)
+        $stderr.puts "Backup with prefix '#{@prefix}' and time #{@time.iso8601} is not complete.  The backup is either in progress or never finished.  This snapshot is not safe to restore!"
+        return
+      end
+      
+      
+      #Get all files from this backup
+      files = manager.list_files(@prefix, @time)
+      
+      #Make the local directory
+      unless File.directory?(@local_dir)
+        FileUtils.mkdir(@local_dir)
+      end
+      
+      prefix_path = timepath(@prefix, @time)
+      
+      
+      files.each do |file|
+        destination_path = "#{@local_dir}/#{file.key[prefix_path.length+1..-1]}"
+        
+        directory = destination_path[0..-File.basename(destination_path).length-1]
+        
+        #Create the parent directory for the file if it doesn't exist
+        unless File.directory?(directory)
+          FileUtils.mkdir(directory)
+        end
+        
+        puts "downloading '#{file.key}' to '#{destination_path}'"
+        
+        #Open the file in read/write and create it if it doesn't exist, then write the content from s3 into it
+        File.open(destination_path, File::RDWR|File::CREAT){ |local| local.write(file.body)}
+        
+      end
+      
+      
+      puts "Writing complete marker"
+      
+      
+      puts "backup complete!"
+    end
+    
+    
+    private
+    
+    
+    
+    
+  end
+end

data/lib/s3snapshot/dir_upload.rb

@@ -0,0 +1,65 @@
+require 'fog'
+require 's3snapshot/sync_op'
+require 's3snapshot/time_factory'
+require 'time'
+require 'fileutils'
+
+
+module S3snapshot
+  class DirUpload < SyncOp
+    
+    @tmpdir = nil
+    @local_dir = nil
+    @prefix = nil
+    
+    def initialize(aws_id, aws_key, bucket_name, prefix, local_dir )
+      super(aws_id, aws_key, bucket_name)
+      @local_dir = local_dir
+      @prefix = prefix
+
+    end
+    
+    def upload
+      
+      start_time = TimeFactory.utc_time
+      
+      prefix_path = timepath(@prefix, start_time)
+      
+      files = get_local_files
+      
+      files.each do |file|
+        path = "#{prefix_path}/#{file[@local_dir.length+1..-1]}"
+        
+        puts "uploading '#{file}' to '#{@bucket_name}/#{path}'"
+        bucket.files.create(:key =>path, :body => File.read(file))
+      end
+      
+      
+      puts "Writing complete marker"
+      
+       #Upload the complete marker
+      bucket.files.create(:key => complete_path(@prefix, start_time), :body => TimeFactory.utc_time.iso8601)
+      
+      puts "backup complete!"
+    end
+    
+    
+    private
+    
+    ##
+    #Finds all files in a directory and returns them in an array
+    ##
+    def get_local_files
+      files = Array.new
+      Dir.glob( File.join(@local_dir, '**', '*') ).each do |file| 
+        unless File.directory?(file)
+          files << file
+        end
+      end
+      
+      return files
+    end
+    
+  
+  end
+end

data/lib/s3snapshot/sync_op.rb

@@ -0,0 +1,63 @@
+require 'fog'
+
+module S3snapshot
+  class SyncOp
+    
+    #Constant for the file that will be present with the complete timestamp if a directory was successfully backed up
+    COMPLETE_FILE = "s3snapshot"
+    COMPLETE_EXTENSION = "complete_marker"
+    COMPLETE_MARKER = "#{COMPLETE_FILE}.#{COMPLETE_EXTENSION}"
+    
+    @bucket_name
+    @aws_id
+    @aws_key
+    
+    #Our to aws connection
+    @aws
+    
+    #The current bucket
+    @bucket
+    
+    def initialize(aws_id, aws_key, bucket_name)
+      @bucket_name = bucket_name
+      @aws_id = aws_id
+      @aws_key = aws_key
+    end
+    
+    ##
+    # Return the cached aws connection or create a new one
+    ##
+    def aws
+      @aws ||= Fog::Storage.new(:provider => 'AWS', :aws_access_key_id => @aws_id, :aws_secret_access_key => @aws_key)
+    end
+    
+    ##
+    #Get the cached bucket or create the new one
+    ##
+    def bucket
+      #      @bucket ||=  aws.directories.get(@bucket_name)
+      aws.directories.get(@bucket_name)
+    end
+    
+    #Generate the time path.  If a prefix is specified the format is <prefix>/<timestamp> otherwise it is timestamp.  All timestamps are in iso 8601 format and in  the UTC time zone
+    def timepath(prefix, time)
+     "#{prefix}/#{time.utc.iso8601}"
+    end
+    
+    
+    #
+    #The path to the complete file with the given prefix and time
+    #
+    def complete_path(prefix, time)
+      "#{timepath(prefix, time)}/#{COMPLETE_MARKER}"
+    end
+    
+    ##
+    #Constructs a prefix in the format of [prefix]/[iso time]/complete_file
+    def complete_prefix(prefix, time)
+      "#{timepath(prefix, time)}/#{COMPLETE_FILE}"
+    end
+    
+    
+  end
+end

data/lib/s3snapshot/test_loader.rb

@@ -0,0 +1,52 @@
+require 'thor'
+require 's3snapshot/dir_upload'
+require 's3snapshot/time_factory'
+
+
+module S3snapshot
+  class TestLoader < Thor
+    ##
+    #Test data generation
+    ##
+    
+    
+    desc "gendata", "upload a directory as a snapshot backup to s3"
+    
+    method_option :awsid, :aliases => "-i", :desc => "The aws id", :type => :string, :required => true
+    method_option :awskey,:aliases => "-k", :desc => "The aws secret key", :type => :string, :required => true
+    method_option :bucket, :aliases => "-b", :desc => "The aws bucket to use", :type => :string, :required => true
+    method_option :directory, :aliases => "-d", :desc => "The directory to upload", :type => :string, :required => true
+    method_option :prefix, :aliases => "-p", :desc => "A prefix to prepend to the path before the timestamp.  Useful in cluster to specifiy a node name, or a node+directory scheme.  Prefix strategies can be mixed in a bucket, they must just be unique." , :type => :string,  :required => true
+    method_option :hours, :aliases => "-h", :desc => "The number of hours between backups to generate a timestamp for", :type => :numeric
+    method_option :numbackups, :aliases => "-n", :desc=> "The maximum number of iterations to run at 'hours' interval", :type => :numeric
+    
+    ##
+    #Uploads the directory to the s3 bucket with a prefix
+    def gendata
+      directory = options[:directory]
+      puts "You are uploading directory #{directory}"
+      
+      hours = options[:hours]
+      backups = options[:numbackups] 
+      
+      # subtract off the number of hours * number of backups and convert the hours to seconds to set the start time
+      time = Time.now.utc - hours * backups * 3600
+      
+      for i in (1.. backups)
+        
+        TimeFactory.set_time (time)
+        s3upload = DirUpload.new(options[:awsid], options[:awskey],  options[:bucket],   options[:prefix], directory )
+        s3upload.upload
+        
+        
+        
+        time += hours * 3600
+        
+      end
+      
+      
+    end
+  end
+end
+
+S3snapshot::TestLoader.start

data/lib/s3snapshot/time_factory.rb

@@ -0,0 +1,27 @@
+module S3snapshot
+  ##
+  #Simple time factory that will return the set time if set, otherwise it returns the current time in utc
+  ##
+  class TimeFactory
+    @@set_time = nil
+    
+    def self.utc_time
+      if @@set_time.nil?
+        return Time.now.utc
+      end
+      
+      return @@set_time
+    
+    end
+
+    
+    def self.set_time(time)
+      @@set_time = time
+    end
+    
+    def self.unset_time
+      @@set_time = nil
+    end
+    
+  end
+end

data/lib/s3snapshot/version.rb

@@ -0,0 +1,3 @@
+module S3snapshot
+  VERSION = "0.0.2"
+end

data/lib/s3snapshot.rb

@@ -0,0 +1,4 @@
+module S3snapshot
+require 's3snapshot/cli'
+S3snapshot::CLI.start
+end

data/s3snapshot.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "s3snapshot/version"
+
+Gem::Specification.new do |s|
+  s.name        = "s3snapshot"
+  s.version     = S3snapshot::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Todd"]
+  s.email       = ["foo@bar.com"]
+  s.homepage    = ""
+  s.summary     = %q{Uploads to s3}
+  s.description = %q{see summary}
+
+  s.rubyforge_project = "s3snapshot"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency "fog", "~>0.8.1"  
+  s.add_dependency "thor", "~>0.14.6"
+  s.add_dependency "dictionary", "~>1.0.0"
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification 
+name: s3snapshot
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Todd
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-23 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fog
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 61
+        segments: 
+        - 0
+        - 8
+        - 1
+        version: 0.8.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: thor
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 43
+        segments: 
+        - 0
+        - 14
+        - 6
+        version: 0.14.6
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: dictionary
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :runtime
+  version_requirements: *id003
+description: see summary
+email: 
+- foo@bar.com
+executables: 
+- s3snapshot
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .loadpath
+- .project
+- Gemfile
+- Gemfile.lock.bak
+- Rakefile
+- Readme.markdown
+- bin/s3snapshot
+- lib/s3snapshot.rb
+- lib/s3snapshot/backup_manager.rb
+- lib/s3snapshot/cli.rb
+- lib/s3snapshot/dir_download.rb
+- lib/s3snapshot/dir_upload.rb
+- lib/s3snapshot/sync_op.rb
+- lib/s3snapshot/test_loader.rb
+- lib/s3snapshot/time_factory.rb
+- lib/s3snapshot/version.rb
+- s3snapshot.gemspec
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: s3snapshot
+rubygems_version: 1.8.2
+signing_key: 
+specification_version: 3
+summary: Uploads to s3
+test_files: []
+