awshucks 0.0.1

data/lib/awshucks/commands/backups.rb

@@ -0,0 +1,25 @@
+module Awshucks
+    
+  class BackupsCommand < Command
+    self.command = 'backups'
+    self.usage = 'backups'
+    self.description = "Lists the available backups and their configurations"
+    
+    def execute(args, config)
+      
+      puts "configured backups:"
+      puts ""
+      
+      config.backups.each do |backup|
+        
+        puts "  #{backup.name}:"
+        puts "    location: #{backup.location}"
+        puts ""
+        
+      end
+      
+    end
+        
+  end
+  
+end

data/lib/awshucks/commands/help.rb

@@ -0,0 +1,22 @@
+module Awshucks
+  
+  class HelpCommand < Command
+    self.command = 'help'
+    self.usage = 'help [commandname]'
+    self.description = "displays this help message or lists help for the given command"
+    
+    def execute(args, config)
+      if args.empty?
+        puts config.help_message
+      else
+        if cmd = Command.commands[args.first]
+          puts cmd.option_string
+        else
+          $stderr.puts "Could not find command #{args.first}"
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/awshucks/commands/list.rb

@@ -0,0 +1,39 @@
+module Awshucks
+    
+  class ListCommand < Command
+    self.command = 'list'
+    self.usage = 'list [config name]'
+    self.description = "lists the files stored on S3 for all configs or the specified config"
+    
+    def execute(args, config)
+      @config = config
+      list = []
+      if args.empty?
+        list += config.backups
+      else
+        list << config.backup(args.first)
+      end
+      
+      list.each {|b| list(b) }
+      
+    rescue UnknownBackupError
+      $stderr.puts "Unknown backup: #{args.first}"
+    end
+    
+    #######
+    private
+    #######
+    
+    attr_reader :config
+    
+    def list(backup_info)  
+      puts "stored files from #{backup_info.location}:"
+      file_store = FileStore.new(config.connection, config.bucket, backup_info.name)      
+      file_store.each_file do |filename|
+        puts "  #{filename}"
+      end
+    end
+    
+  end
+  
+end

data/lib/awshucks/commands/new_config.rb

@@ -0,0 +1,36 @@
+module Awshucks
+    
+  class NewConfigCommand < Command
+    self.command = 'new_config'
+    self.usage = 'new_config [filename]'
+    self.description = "generates a new default config file. filename defaults to awshucks.yml"
+    
+    # def custom_options(opts)
+    #   opts.on('-f', '--force', "Overwrite the existing config file if it exists") do
+    #     parsed_options.overwrite = true
+    #   end
+    # end
+    
+    def execute(args, config)
+      # option_parser.parse!(args)
+      
+      dest_path = File.expand_path(args.empty? ? 'awshucks.yml' : args.first)
+      if File.exists?(dest_path)# && !parsed_options.overwrite
+        $stderr.puts "cannot create #{dest_path}, file already exists"
+      else
+        puts "creating new config file at #{dest_path}"
+        FileUtils.cp(awshucks_config, dest_path )
+      end
+    end
+    
+    #######
+    private
+    #######
+    
+    def awshucks_config
+      File.join(RESOURCE_DIR, 'awshucks.yml')
+    end
+    
+  end
+  
+end

data/lib/awshucks/commands/reset_metadata_cache.rb

@@ -0,0 +1,38 @@
+module Awshucks
+    
+  class ResetMetadataCacheCommand < Command
+    self.command = 'reset_metadata_cache'
+    self.usage = 'reset_metadata_cache [config name]'
+    self.description = "Resets (deletes) the metadata cache for all backups or the specified backup. " +
+      "The next backup will rely on the stored metadata for each file to restore the metadata during the next run, " +
+      "Be aware that this process will be significantly slower than using the metadata cache to begin with."
+    
+    def execute(args, config)
+      @config = config
+      list = []
+      if args.empty?
+        list += config.backups
+      else
+        list << config.backup(args.first)
+      end
+      
+      list.each {|b| reset(b) }
+      
+    rescue UnknownBackupError
+      $stderr.puts "Unknown backup: #{args.first}"
+    end
+    
+    #######
+    private
+    #######
+    
+    attr_reader :config
+    
+    def reset(backup_info)
+      puts "processing #{backup_info.name}:"
+      FileStore.new(config.connection, config.bucket, backup_info.name).reset_cache
+    end
+    
+  end
+  
+end

data/lib/awshucks/commands/restore.rb

@@ -0,0 +1,54 @@
+module Awshucks
+    
+  class RestoreCommand < Command
+    self.command = 'restore'
+    self.usage = 'restore <config name> [file_or_dir_pattern [target dir]]'
+    self.description = <<-EOD
+Restores from the specified config name. 
+Optionally, provide a file or directory name or pattern to specify what files to restore. This can be a wildcard, or if it's a specific path, use the relative path from the configured backup location. If it's a wildcard, don't forget to wrap it in quotes!
+By default, awshucks will restore files to the location specified in the configuration, but can be directed elsewhere -- provide a target directory to set the destination.
+    EOD
+    
+    def execute(args, config)
+      @config = config
+      
+      if args.empty?
+        $stderr.puts "Please specify a backup configuration to restore from"
+      else
+        backup = config.backup(args[0])
+        pattern = args[1] || '*'
+        target_dir = args[2] || backup.location
+        restore(backup, pattern, target_dir)
+      end
+
+    rescue UnknownBackupError
+      $stderr.puts "Unknown backup: #{args.first}"
+    end
+    
+    #######
+    private
+    #######
+    
+    attr_reader :config
+    
+    def restore(backup_info, pattern, target_dir)  
+      target_dir = File.expand_path(target_dir)
+
+      puts "restoring backup from #{backup_info.location} to #{target_dir}"    
+
+      file_store = FileStore.new(config.connection, config.bucket, backup_info.name)
+      
+      file_store.each_file do |filename|
+        if File.fnmatch(pattern, filename)
+          puts "  restoring #{filename}"
+          file_store.restore(filename, File.join(target_dir, filename)) 
+        else
+          puts "  #{filename} doesn't match '#{pattern}', skipping"
+        end
+      end
+      
+    end
+
+  end
+  
+end

data/lib/awshucks/config.rb

@@ -0,0 +1,83 @@
+module Awshucks
+
+  class RequiredConfigError < Exception
+    def initialize(name)
+      super "Missing required configuration setting: #{name}"
+    end
+  end
+
+  class UnknownBackupError < Exception;
+    def initialize(name)
+      super "Unknown backup configuration name #{name}"
+    end
+  end
+  
+  class Config
+
+    # class << self
+    #   
+    #   def load(filename)
+    #     new(open(filename))
+    #   end
+    #   
+    # end
+    
+    REQUIRED_CONNECTION_KEYS = %w(access_key_id secret_access_key).freeze
+    REQUIRED_BACKUP_CONFIG_KEYS = %w(location).freeze
+    RESERVED_KEYS = (REQUIRED_CONNECTION_KEYS + %w(bucket)).freeze
+    
+    attr_accessor :help_message # piggyback for the global help --> help command
+    
+    def initialize(filename)
+      @filename = filename
+    end
+    
+    # return the connection info
+    def connection
+      returning({:use_ssl => true, :persistent => false}) do |connection|
+        REQUIRED_CONNECTION_KEYS.each do |key|
+          raise RequiredConfigError, key unless config[key]
+          connection[key.intern] = config[key]
+        end
+      end
+    end
+    
+    def bucket
+      returning(config['bucket']) do |bucket|
+        raise RequiredConfigError, 'bucket' unless bucket
+      end
+    end
+    
+    # return the config for the specified backup, with validation
+    def backup(name)
+      raise UnknownBackupError, name if name.in? REQUIRED_CONNECTION_KEYS
+      returning OpenStruct.new(config[name]) do |backup|
+        REQUIRED_BACKUP_CONFIG_KEYS.each do |required|
+          raise RequiredConfigError, required unless backup.send(required)
+        end
+        backup.name = name
+      end
+    end
+    
+    def backups
+      returning [] do |list|
+        (config.keys - RESERVED_KEYS).each do |name|
+          list << backup(name)
+        end
+      end
+    end
+            
+    private
+    
+    attr_reader :filename
+
+    def config
+      @config ||= YAML.load(File.open(File.expand_path(filename)))
+    rescue Errno::ENOENT # file not found
+      $stderr.puts "config file not found: #{filename}"
+      exit(-1)
+    end
+  
+  end
+  
+end

data/lib/awshucks/ext.rb

@@ -0,0 +1,23 @@
+# miscellaneous core extensions
+
+class Object
+  def returning(obj)
+    yield obj
+    obj
+  end
+end
+
+# thanks to http://devel.touset.org/blog/articles/2007/09/02/enumerable-include
+class Object
+  def in?(enum)
+    enum.include? self
+  end
+end
+
+class File
+  class << self
+    def md5sum(filename)
+      Digest::MD5.hexdigest(read(filename))
+    end
+  end
+end

data/lib/awshucks/file_info.rb

@@ -0,0 +1,23 @@
+module Awshucks
+  
+  class FileInfo
+    
+    attr_reader :filename, :full_path, :mtime, :size
+
+    def initialize(filename, full_path, stat)
+      @filename, @full_path= filename, full_path
+      @mtime = stat.mtime.gmtime
+      @size = stat.size
+    end
+    
+    def md5sum
+      @md5sum ||= File.md5sum(full_path)
+    end
+    
+    def data
+      File.open(full_path)
+    end
+    
+  end
+  
+end

data/lib/awshucks/file_store.rb

@@ -0,0 +1,111 @@
+module Awshucks
+  
+  class UnknownFileError < Exception; end
+  
+  class FileStore
+    
+    def initialize(connection_info, bucket_name, prefix)
+      @bucket_name, @prefix = bucket_name, prefix
+      AWS::S3::Base.establish_connection!(connection_info) unless AWS::S3::Base.connected?
+    end
+    
+    def each_file
+      bucket.objects(:prefix => prefix).each { |f| yield remove_prefix(f.key) unless f.key == metadata_key }
+    end
+    
+    def different_from?(file_info)
+      return true unless info = metadata_for(file_info.filename)
+      if info['mtime'] == file_info.mtime.gmtime.to_s
+        false
+      else
+        if info['md5sum'] == file_info.md5sum
+          info['mtime'] = file_info.mtime.gmtime.to_s # update the metadata cache
+          false
+        else
+          true
+        end
+      end
+    end
+    
+    def store(file_info)
+      metadata[file_info.filename] = { 'mtime' => file_info.mtime.to_s, 'md5sum' => file_info.md5sum }
+      s3_metadata = { 'x-amz-meta-mtime' => file_info.mtime.to_s, 'x-amz-meta-md5sum' => file_info.md5sum }
+      AWS::S3::S3Object.store(prefixed_filename(file_info.filename), file_info.data, bucket.name, s3_metadata).success?
+    end
+    
+    # stores the value of the file stored on S3 to the target file
+    def restore(filename, target)
+      raise UnknownFileError, "unknown file #{filename}" unless file = bucket[prefixed_filename(filename)]
+      FileUtils.mkdir_p(File.dirname(target))
+      File.open(target, 'w') do |out|
+        file.value do |data|
+          out.write(data)
+        end
+      end
+      # warn if the file's md5 doesn't match after writing it out
+      $stderr.puts "warning: md5sums don't match" if metadata_for(filename)['md5sum'] != File.md5sum(target)
+    end
+    
+    def save_cache
+      AWS::S3::S3Object.store(metadata_key, metadata.to_yaml, bucket.name)
+    end
+    
+    def reset_cache
+      puts "deleting the metadata cache..."
+      AWS::S3::S3Object.delete(metadata_key, bucket.name)
+      @metadata = {}
+    end
+    
+    private
+    
+    attr_reader :bucket_name, :prefix
+    
+    def bucket
+      @bucket ||= AWS::S3::Bucket.find(bucket_name)
+    rescue AWS::S3::ResponseError => e
+      # this is awkward, but the NoSuchBucket exception itself doesn't get defined until it's raised.
+      raise e unless e.message =~ /bucket does not exist/
+      puts "could not find bucket #{bucket_name}, creating it"
+      retry if AWS::S3::Bucket.create(bucket_name)
+    end
+    
+    def metadata
+      @metadata ||= load_metadata_from_cache
+    end
+    
+    # returns or caches the metadata for a file from the s3 object if it's not already cached
+    def metadata_for(filename)
+      return metadata[filename] if metadata[filename]
+      return nil unless file = find(filename)
+      metadata[filename] = returning({}) do |info|
+        info['mtime'] = file.metadata['mtime'].to_s
+        info['md5sum'] = file.metadata['md5sum']
+      end
+    end
+    
+    def find(filename)
+      search = prefixed_filename(filename)
+      bucket.objects(:prefix => prefix).detect do |file|
+        file.key == search
+      end
+    end
+    
+    def prefixed_filename(filename)
+      File.join(prefix, filename)
+    end
+    
+    def remove_prefix(filename)
+      filename.sub(/\A#{prefix}\//, '')
+    end
+    
+    def load_metadata_from_cache
+      bucket[metadata_key] ? YAML.load(bucket[metadata_key].value) : {}
+    end
+    
+    def metadata_key
+      prefix + '_metadata.yml'
+    end
+    
+  end
+  
+end