cloudsync 0.1.0

data/.gitignore

@@ -0,0 +1,23 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+cloudsync.log
+cloudsync.yml

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Cory Forsyth
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= cloudsync
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Cory Forsyth. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "cloudsync"
+    gem.summary = %Q{Sync files between various clouds or sftp servers.}
+    gem.description = %Q{Sync files between various clouds or sftp servers. Available backends are S3, CloudFiles, and SFTP servers. Can sync, mirror, and prune.}
+    gem.email = "cory.forsyth@gmail.com"
+    gem.homepage = "http://github.com/megaphone/cloudsync"
+    gem.authors = ["Cory Forsyth"]
+    gem.add_dependency "right_aws", ">= 0"
+    gem.add_dependency "cloudfiles", ">= 0"
+    gem.add_dependency "commander", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "cloudsync #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/cloudsync

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require "commander/import"
+require "lib/cloudsync"
+
+program :version, Cloudsync::VERSION
+program :description, "Sync between various backends (S3, Cloudfiles, SFTP)"
+
+def add_sync_options(c)
+  c.option "--from from_backend", String, "From Backend"
+  c.option "--to to_backend", String, "To Backend"
+  c.option "--dry-run", "Dry run?"
+  c.option "--log LOGFILE", String, "Log file"
+  c.option "-a", "Auto mode -- skip command-line confirmations"
+end
+
+def confirm_proceed(msg)
+  exit unless agree(msg)
+end
+
+command :sync do |c|
+  c.syntax = "cloudsync sync --from from_backend --to to_backend [--dry-run]"
+  c.description = "Copies all files on from_backend to to_backend."
+  add_sync_options(c)
+  c.action do |args, options|
+    options.default :dry_run => false
+    
+    from_backend = options.from.to_sym
+    to_backend   = options.to.to_sym
+    
+    sync_manager = Cloudsync::SyncManager.new \
+                      :from      => from_backend,
+                      :to        => to_backend,
+                      :dry_run => options.dry_run,
+                      :log_file  => options.log
+    
+    unless options.a
+      confirm_proceed("Preparing to sync from #{sync_manager.from_backend} to #{sync_manager.to_backend}. Dry-run: #{!!sync_manager.dry_run?}. Ok to proceed?")
+    end
+    
+    sync_manager.sync!
+  end
+end
+
+command :mirror do |c|
+  c.syntax = "cloudsync mirror --from from_backend --to to_backend [--dry-run]"
+  c.description = "Syncs and then prunes all files on from_backend to to_backend."
+  add_sync_options(c)
+  c.action do |args, options|
+    options.default :dry_run => false
+    
+    from_backend = options.from.to_sym
+    to_backend   = options.to.to_sym
+    
+    sync_manager = Cloudsync::SyncManager.new \
+                      :from      => from_backend,
+                      :to        => to_backend,
+                      :dry_run => options.dry_run,
+                      :log_file  => options.log
+
+    unless options.a
+      confirm_proceed("Preparing to mirror from #{sync_manager.from_backend} to #{sync_manager.to_backend}. Dry-run: #{!!sync_manager.dry_run?}. Ok to proceed?")
+    end
+
+    sync_manager.mirror!
+  end
+end
+
+command :prune do |c|
+  c.syntax = "cloudsync prune --from from_backend --to to_backend [--dry-run]"
+  c.description = "Removes all on to_backend that don't exist on from_backend."
+  add_sync_options(c)
+  c.action do |args, options|
+    options.default :dry_run => false
+    
+    from_backend = options.from.to_sym
+    to_backend   = options.to.to_sym
+    
+    sync_manager = Cloudsync::SyncManager.new \
+                      :from      => from_backend,
+                      :to        => to_backend,
+                      :dry_run => options.dry_run,
+                      :log_file  => options.log
+
+    unless options.a
+      confirm_proceed("Preparing to prune from #{sync_manager.from_backend} to #{sync_manager.to_backend}. Dry-run: #{!!sync_manager.dry_run?}. Ok to proceed?")
+    end
+
+    sync_manager.prune!
+  end
+end

data/cloudsync.gemspec

@@ -0,0 +1,70 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{cloudsync}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Cory Forsyth"]
+  s.date = %q{2010-10-12}
+  s.default_executable = %q{cloudsync}
+  s.description = %q{Sync files between various clouds or sftp servers. Available backends are S3, CloudFiles, and SFTP servers. Can sync, mirror, and prune.}
+  s.email = %q{cory.forsyth@gmail.com}
+  s.executables = ["cloudsync"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/cloudsync",
+     "cloudsync.gemspec",
+     "lib/cloudsync.rb",
+     "lib/cloudsync/backend/base.rb",
+     "lib/cloudsync/backend/cloudfiles.rb",
+     "lib/cloudsync/backend/s3.rb",
+     "lib/cloudsync/backend/sftp.rb",
+     "lib/cloudsync/datetime/datetime.rb",
+     "lib/cloudsync/file.rb",
+     "lib/cloudsync/sync_manager.rb",
+     "lib/cloudsync/version.rb",
+     "test/helper.rb",
+     "test/test_cloudsync.rb"
+  ]
+  s.homepage = %q{http://github.com/megaphone/cloudsync}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Sync files between various clouds or sftp servers.}
+  s.test_files = [
+    "test/helper.rb",
+     "test/test_cloudsync.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<right_aws>, [">= 0"])
+      s.add_runtime_dependency(%q<cloudfiles>, [">= 0"])
+      s.add_runtime_dependency(%q<commander>, [">= 0"])
+    else
+      s.add_dependency(%q<right_aws>, [">= 0"])
+      s.add_dependency(%q<cloudfiles>, [">= 0"])
+      s.add_dependency(%q<commander>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<right_aws>, [">= 0"])
+    s.add_dependency(%q<cloudfiles>, [">= 0"])
+    s.add_dependency(%q<commander>, [">= 0"])
+  end
+end
+

data/lib/cloudsync.rb

@@ -0,0 +1,12 @@
+$:.unshift(File.dirname(__FILE__))
+
+require "cloudsync/sync_manager"
+require "cloudsync/version"
+require "cloudsync/file"
+require "cloudsync/backend/base"
+require "cloudsync/backend/cloudfiles"
+require "cloudsync/backend/s3"
+require "cloudsync/backend/sftp"
+
+# monkeypatches
+require "cloudsync/datetime/datetime"

data/lib/cloudsync/backend/base.rb

@@ -0,0 +1,112 @@
+require 'tempfile'
+
+module Cloudsync
+  module Backend
+    class Base
+      attr_accessor :store, :sync_manager, :name, :prefix
+      
+      def initialize(opts = {})
+        @sync_manager     = opts[:sync_manager]
+        @name             = opts[:name]
+        @backend_type     = opts[:backend] || self.class.to_s.split("::").last
+      end
+      
+      def upload_prefix
+        {:bucket => @bucket, :prefix => @prefix}
+      end
+      
+      def upload_prefix_path
+        if @bucket && @prefix
+          "#{@bucket}/#{@prefix}"
+        end
+      end
+
+      # copy
+      def copy(file, to_backend)
+        start_copy = Time.now
+        $LOGGER.info("Copying file #{file} from #{self} to #{to_backend}")
+        tempfile = download(file)
+        if tempfile
+          to_backend.put(file, tempfile.path)
+
+          $LOGGER.debug("Finished copying #{file} from #{self} to #{to_backend} (#{Time.now - start_copy}s)")
+          tempfile.unlink
+        else
+          $LOGGER.info("Failed to download #{file}")
+        end
+      end
+    
+      def to_s
+        "#{@name}[:#{@backend_type}/#{upload_prefix_path}]"
+      end
+      
+      # needs_update?
+      def needs_update?(file, file_list=[])
+        $LOGGER.debug("Checking if #{file} needs update")
+      
+        local_backend_file = find_file_from_list_or_store(file, file_list)
+
+        if local_backend_file.nil?
+          $LOGGER.debug("File doesn't exist at #{self} (#{file})")
+          return true
+        end
+
+        if file.e_tag == local_backend_file.e_tag
+          $LOGGER.debug("Etags match for #{file}")
+          return false
+        else
+          $LOGGER.debug(["Etags don't match for #{file}.",
+                        "#{file.backend}: #{file.e_tag}",
+                        "#{self}: #{local_backend_file.e_tag}"].join(" "))
+          return true
+        end
+      end
+    
+      # download
+      def download(file)
+        raise NotImplementedError
+      end
+    
+      # put
+      def put(file, local_filepath)
+        raise NotImplementedError
+      end
+    
+      # delete
+      def delete(file, delete_bucket_if_empty=true)
+        raise NotImplementedError
+      end
+    
+      # all_files
+      def all_files
+        raise NotImplementedError
+      end
+      
+      def files_to_sync(upload_prefix={})
+        all_files
+      end
+      
+      # find_file_from_list_or_store
+      def find_file_from_list_or_store(file, file_list=[])
+        get_file_from_list(file, file_list) || get_file_from_store(file)
+      end
+    
+      private
+    
+      def dry_run?
+        return false unless @sync_manager
+        @sync_manager.dry_run?
+      end
+
+      # get_file_from_store
+      def get_file_from_store(file)
+        raise NotImplementedError
+      end
+    
+      # get_file_from_list
+      def get_file_from_list(file, file_list)
+        file_list.detect {|f| f.full_name == file.full_name}
+      end
+    end
+  end
+end

data/lib/cloudsync/backend/cloudfiles.rb

@@ -0,0 +1,128 @@
+require "cloudfiles"
+
+module Cloudsync
+  module Backend
+    class CloudFiles < Base
+      def initialize(opts={})
+        @store = ::CloudFiles::Connection.new \
+                  :username => opts[:username],
+                  :api_key  => opts[:password]
+        super
+      end
+    
+      def download(file)
+        start_time = Time.now
+        $LOGGER.info("Downloading file #{file}")
+      
+        tempfile = file.tempfile
+
+        if !dry_run?
+          if obj = get_obj_from_store(file)
+            obj.save_to_filename(tempfile.path)
+            tempfile.close
+          else
+            $LOGGER.error("Error downloading file #{file}")
+            tempfile.unlink and return nil
+          end
+        end
+
+        $LOGGER.debug("Finished downloading file #{file} from #{self} (#{Time.now - start_time})")
+        tempfile
+      end
+    
+      # Put the contents of the path #local_file_path# into
+      # the Cloudsync::File object #file# 
+      def put(file, local_file_path)
+        start_time = Time.now
+        $LOGGER.info("Putting #{file} to #{self} (#{file.full_upload_path}).")
+        return if dry_run?
+      
+        get_or_create_obj_from_store(file).
+          load_from_filename(local_file_path)
+        $LOGGER.debug("Finished putting #{file} to #{self} (#{Time.now - start_time}s)")
+      end
+    
+      def files_to_sync(upload_prefix={})
+        $LOGGER.info("Getting files to sync [#{self}]")
+
+        containers_to_sync(upload_prefix).inject([]) do |files, container|
+          container = get_or_create_container(container)
+          objects_from_container(container, upload_prefix).each do |path, hash|
+            files << Cloudsync::File.from_cf_info(container, path, hash, self.to_s)
+          end
+          files
+        end
+      end
+      
+      def delete(file, delete_container_if_empty=true)
+        $LOGGER.info("Deleting file #{file}")
+        return if dry_run?
+      
+        container = @store.container(file.container)
+        
+        container.delete_object(file.path)
+      
+        if delete_container_if_empty
+          container.refresh
+          if container.empty?
+            $LOGGER.debug("Deleting empty container '#{container.name}'")
+            @store.delete_container(container.name)
+          end
+        end
+      
+      rescue NoSuchContainerException, NoSuchObjectException => e
+        $LOGGER.error("Failed to delete file #{file}")
+      end
+    
+      private
+      
+      def get_or_create_container(container_name)
+        if @store.container_exists?(container_name)
+          container = @store.container(container_name)
+        else
+          container = @store.create_container(container_name)
+        end
+      end
+
+      def containers_to_sync(upload_prefix)
+        upload_prefix[:bucket] ? [upload_prefix[:bucket]] : @store.containers
+      end
+      
+      def objects_from_container(container, upload_prefix)
+        objects = []
+        if upload_prefix[:prefix]
+          container.objects_detail(:path => upload_prefix[:prefix]).collect do |path, hash|
+            if hash[:content_type] == "application/directory"
+              objects += objects_from_container(container, :prefix => path) 
+            else
+              objects << [path, hash]
+            end
+          end
+        else
+          objects = container.objects_detail
+        end
+        objects
+      end
+
+      def get_obj_from_store(file)
+        @store.container(file.bucket).object(file.upload_path)
+      rescue NoSuchContainerException, NoSuchObjectException => e
+        nil
+      end
+    
+      def get_file_from_store(file)
+        Cloudsync::File.from_cf_obj( get_obj_from_store(file), self.to_s )
+      end
+    
+      def get_or_create_obj_from_store(file)
+        container = get_or_create_container(file.container)
+      
+        if container.object_exists?(file.upload_path)
+          container.object(file.upload_path)
+        else
+          container.create_object(file.upload_path, true)
+        end
+      end
+    end
+  end
+end

data/lib/cloudsync/backend/s3.rb

@@ -0,0 +1,117 @@
+require "right_aws"
+
+module Cloudsync
+  module Backend
+    class S3 < Base
+      def initialize(opts={})
+        @store = RightAws::S3.new(opts[:username],
+                                  opts[:password])
+        super
+      end
+    
+      def put(file, local_filepath)
+        start_time = Time.now
+        $LOGGER.info("Putting #{file} to #{self} (#{file.full_upload_path}).")
+        return if dry_run?
+      
+        # Forces creation of the bucket if necessary
+        get_or_create_obj_from_store(file)
+      
+        local_file = ::File.open(local_filepath)
+        @store.interface.put(file.bucket, file.upload_path, local_file)
+        local_file.close
+      
+        $LOGGER.debug("Finished putting #{file} to #{self} (#{Time.now - start_time})")
+      end
+    
+      def download(file)
+        start_time = Time.now
+        $LOGGER.info("Downloading file #{file}")
+      
+        tempfile = file.tempfile
+
+        if !dry_run?
+          @store.interface.get(file.bucket, file.path) do |chunk|
+            tempfile.write chunk
+          end
+        end
+      
+        tempfile.close
+      
+        $LOGGER.debug("Finished downloading file #{file} from #{self} (#{Time.now - start_time})")
+      
+        tempfile
+      rescue RightAws::AwsError => e
+        $LOGGER.error("Caught error: #{e} (#{file})")
+        if e.message =~ /NoSuchKey/
+          tempfile.unlink and return nil
+        else
+          raise
+        end
+      end
+    
+      def delete(file, delete_bucket_if_empty=true)
+        $LOGGER.info("Deleting #{file}")
+        return if dry_run?
+      
+        get_obj_from_store(file).delete
+      
+        if bucket = @store.bucket(file.bucket)
+          bucket.key(file.path).delete
+        
+          if delete_bucket_if_empty && bucket.keys.empty?
+            $LOGGER.debug("Deleting empty bucket '#{bucket.name}'")
+            bucket.delete
+          end
+        end
+      rescue RightAws::AwsError => e
+        $LOGGER.error("Caught error: #{e} trying to delete #{file}")
+      end
+    
+      def files_to_sync(upload_prefix={})
+        $LOGGER.info("Getting files to sync [#{self}]")
+      
+        buckets_to_sync(upload_prefix).inject([]) do |files, bucket|
+          objects_from_bucket(bucket, upload_prefix).collect do |key|
+            files << Cloudsync::File.from_s3_obj(key, self.to_s)
+          end
+          files
+        end
+      end
+
+      private
+    
+      def buckets_to_sync(upload_prefix)
+        if upload_prefix[:bucket]
+          [@store.bucket(upload_prefix[:bucket], true)]
+        else
+          @store.buckets
+        end
+      end
+      
+      def objects_from_bucket(bucket, upload_prefix)
+        if upload_prefix[:prefix]
+          bucket.keys(:prefix => upload_prefix[:prefix])
+        else 
+          bucket.keys
+        end
+      end
+    
+      # Convenience to grab a single file
+      def get_file_from_store(file)
+        Cloudsync::File.from_s3_obj( get_obj_from_store(file), self.to_s )
+      end
+    
+      def get_or_create_obj_from_store(file)
+        @store.bucket(file.bucket, true).key(file.upload_path)
+      end
+    
+      def get_obj_from_store(file)
+        if bucket = @store.bucket(file.bucket)
+          key = bucket.key(file.upload_path)
+          return key if key.exists?
+        end
+      end
+    end
+  end
+end

data/lib/cloudsync/backend/sftp.rb

@@ -0,0 +1,132 @@
+require 'net/ssh'
+require 'net/sftp'
+
+module Cloudsync::Backend
+  class Sftp < Base
+    attr_accessor :host, :username, :password
+    
+    def initialize(options = {})
+      @host                 = options[:host]
+      @base_path            = options[:base_path]
+      @username             = options[:username]
+      @password             = options[:password]
+      prefix_parts = options[:upload_prefix].split("/")
+      
+      @bucket = prefix_parts.shift
+      @prefix = prefix_parts.join("/")
+      
+      super
+    end
+    
+    # download
+    def download(file)
+      $LOGGER.info("Downloading #{file}")
+      tempfile = file.tempfile
+      
+      if !dry_run?
+        Net::SSH.start(@host, @username, :password => @password) do |ssh|
+          ssh.sftp.connect do |sftp|
+            begin
+              sftp.download!(absolute_path(file.path), tempfile)
+            rescue RuntimeError => e
+              if e.message =~ /permission denied/
+                tempfile.close
+                return tempfile
+              else
+                raise
+              end
+            end
+          end
+        end
+      end
+      tempfile.close
+      tempfile
+    end
+    
+    # put
+    def put(file, local_filepath)
+      $LOGGER.info("Putting #{file} to #{self}")
+      return if dry_run?
+      
+      Net::SSH.start(@host, @username, :password => @password) do |ssh|
+        ssh.sftp.connect do |sftp|
+          sftp.upload!(local_filepath, absolute_path(file.path))
+        end
+      end
+    end
+    
+    # delete
+    def delete(file, delete_bucket_if_empty=true)
+      $LOGGER.info("Deleting #{file}")
+      return if dry_run?
+      
+      Net::SSH.start(@host, @username, :password => @password) do |ssh|
+        ssh.sftp.connect do |sftp|
+          sftp.remove!(absolute_path(file.path))
+        end
+      end
+    end
+    
+    def files_to_sync(upload_prefix={})
+      $LOGGER.info("Getting files to sync [#{self}]")
+      files = []
+      Net::SSH.start(@host, @username, :password => @password) do |ssh|
+        ssh.sftp.connect do |sftp|
+          filepaths = sftp.dir.glob(@base_path, "**/**").collect {|entry| entry.name}
+        
+          files = filepaths.collect do |filepath|
+            attrs = sftp.stat!(absolute_path(filepath))
+            next unless attrs.file?
+
+            e_tag = ssh.exec!("md5sum #{absolute_path(filepath)}").split(" ").first
+            Cloudsync::File.new \
+              :bucket        => @bucket,
+              :path          => filepath,
+              :size          => attrs.size,
+              :last_modified => attrs.mtime,
+              :e_tag         => e_tag,
+              :backend       => self.to_s
+          end.compact
+        end
+      end
+      files
+    end
+    
+    def absolute_path(path)
+      @base_path + "/" + path
+    end
+    
+    private
+    
+    # get_file_from_store
+    def get_file_from_store(file)
+      local_filepath = file.path.sub(/^#{@prefix}\/?/,"")
+      
+      $LOGGER.debug("Looking for local filepath: #{local_filepath}")
+      $LOGGER.debug("Abs filepath: #{absolute_path(local_filepath)}")
+      
+      sftp_file = nil
+      Net::SSH.start(@host, @username, :password => @password) do |ssh|
+        ssh.sftp.connect do |sftp|
+          begin
+            attrs = sftp.stat!(absolute_path(local_filepath))
+          rescue Net::SFTP::StatusException => e
+            break if e.message =~ /no such file/
+            raise
+          end
+          break unless attrs.file?
+          
+          e_tag = ssh.exec!("md5sum #{absolute_path(local_filepath)}").split(" ").first
+          sftp_file = Cloudsync::File.new \
+            :bucket        => @bucket,
+            :path          => local_filepath,
+            :size          => attrs.size,
+            :last_modified => attrs.mtime,
+            :e_tag         => e_tag,
+            :backend       => self.to_s
+        end
+      end
+      sftp_file
+    end
+  end 
+end

data/lib/cloudsync/datetime/datetime.rb

@@ -0,0 +1,17 @@
+class DateTime
+  def to_gm_time
+    to_time(new_offset, :gm)
+  end
+
+  def to_local_time
+    to_time(new_offset(DateTime.now.offset-offset), :local)
+  end
+
+  private
+  def to_time(dest, method)
+    #Convert a fraction of a day to a number of microseconds
+    usec = (dest.sec_fraction * 60 * 60 * 24 * (10**6)).to_i
+    Time.send(method, dest.year, dest.month, dest.day, dest.hour, dest.min,
+              dest.sec, usec)
+  end
+end

data/lib/cloudsync/file.rb

@@ -0,0 +1,75 @@
+module Cloudsync
+  class File
+    attr_accessor :bucket, :path, :size, :last_modified, :e_tag, :backend
+    alias_method :container, :bucket
+    alias_method :container=, :bucket=
+    
+    def initialize(options={})
+      @bucket        = options[:bucket]
+      @path          = options[:path] 
+      @size          = options[:size]
+      @last_modified = options[:last_modified]
+      @e_tag         = options[:e_tag]
+      @backend       = options[:backend]
+    end
+    
+    def self.from_s3_obj(obj, backend=nil)
+      return nil if obj.nil?
+      new({
+        :bucket        => obj.bucket.name,
+        :path          => obj.name,
+        :size          => obj.size,
+        :last_modified => obj.last_modified.to_i,
+        :e_tag         => obj.e_tag.gsub('"',''),
+        :backend       => backend})
+    end
+    
+    def self.from_cf_info(container, path, hash, backend)
+      new({ :bucket        => container.name,
+            :path          => path,
+            :size          => hash[:bytes],
+            :last_modified => hash[:last_modified].to_gm_time.to_i,
+            :e_tag         => hash[:hash],
+            :backend       => backend })
+    end
+    
+    def self.from_cf_obj(obj, backend=nil)
+      return nil if obj.nil?
+      new({
+        :bucket        => obj.container.name,
+        :path          => obj.name,
+        :size          => obj.bytes.to_i,
+        :last_modified => obj.last_modified.to_i,
+        :e_tag         => obj.etag,
+        :backend       => backend})
+    end
+    
+    def to_s
+      "#{path}"
+    end
+    
+    def unique_filename
+      [bucket,e_tag,path].join.gsub(/[^a-zA-Z\-_0-9]/,'')
+    end
+    
+    def full_name
+      [bucket,path].join("/")
+    end
+    
+    def upload_path
+      if @prefix
+        @prefix + "/" + @path
+      else
+        @path
+      end
+    end
+    
+    def full_upload_path
+      [bucket, upload_path].join("/")
+    end
+    
+    def tempfile
+      Tempfile.new(unique_filename)
+    end
+  end
+end

data/lib/cloudsync/sync_manager.rb

@@ -0,0 +1,138 @@
+module Cloudsync
+  class SyncManager
+    attr_accessor :from_backend, :to_backend, :dry_run
+    
+    def initialize(opts={})
+      @from_backend     = get_backend opts[:from]
+      @to_backend       = get_backend opts[:to]
+      
+      if @from_backend == @to_backend
+        raise ArgumentError, "The from_backend can't be the same as the to_backend."
+      end
+      
+      @dry_run          = opts[:dry_run]
+
+      log_file = opts[:log_file] || "cloudsync.log"
+      log_file = ::File.expand_path(log_file)
+      $LOGGER  = Logger.new(log_file)
+    end
+  
+    def sync!
+      sync(:sync)
+    end
+    
+    def sync_all!
+      sync(:sync_all)
+    end
+    
+    def mirror!
+      $LOGGER.info("Mirror from #{from_backend} to #{to_backend} started at #{mirror_start = Time.now}. Dry-run? #{!!dry_run?}")
+      sync!
+      prune!
+      $LOGGER.info("Mirror from #{from_backend} to #{to_backend} finished at #{Time.now}. Took #{Time.now - mirror_start}s")
+    end
+    
+    def dry_run?
+      @dry_run
+    end
+    
+    def prune!
+      prune
+    end
+    
+    private
+    
+    def get_backend(backend_name)
+      opts = configs[backend_name].merge(:name => backend_name, :sync_manager => self)
+      
+      case opts[:backend]
+      when :s3
+        Cloudsync::Backend::S3.new(opts)
+      when :cloudfiles
+        Cloudsync::Backend::CloudFiles.new(opts)
+      when :sftp
+        Cloudsync::Backend::Sftp.new(opts)
+      end
+    end
+    
+    def configs
+      @configs ||= begin
+        if ::File.exists?( path = ::File.expand_path("~/.cloudsync.yml") )
+          YAML::load_file(path)
+        elsif ::File.exists?( path = ::File.expand_path("cloudsync.yml") )
+          YAML::load_file(path)
+        else
+          raise "Couldn't find cloudsync.yml file!"
+        end
+      end
+    end
+    
+    def prune
+      file_stats = {:removed => [], :skipped => []}
+      
+      $LOGGER.info("Prune from #{from_backend} to #{to_backend} started at #{prune_start = Time.now}. Dry-run? #{!!dry_run?}")
+      
+      from_backend_files   = [] # from_backend.files_to_sync(to_backend.upload_prefix)
+      to_backend_files     = to_backend.files_to_sync(from_backend.upload_prefix)
+      total_files          = to_backend_files.size
+      last_decile_complete = 0
+      
+      to_backend_files.each_with_index do |file, index|
+        $LOGGER.debug("Checking if file #{file} exists on [#{from_backend}]")
+        if found_file = from_backend.find_file_from_list_or_store(file, from_backend_files)
+          $LOGGER.debug("Keeping file #{file} because it was found on #{from_backend}.")
+          file_stats[:skipped] << file
+        else
+          $LOGGER.debug("Removing #{file} because it doesn't exist on #{from_backend}.")
+          file_stats[:removed] << file
+          
+          to_backend.delete(file)
+        end
+        
+        if decile_complete(index, total_files) != last_decile_complete
+          last_decile_complete = decile_complete(index, total_files)
+          $LOGGER.info("Prune: Completed #{index} files. #{last_decile_complete * 10}% complete")
+        end
+      end
+      
+      $LOGGER.info(["Prune from #{from_backend} to #{to_backend} finished at #{Time.now}, took #{Time.now - prune_start}s.",
+                   "Skipped #{file_stats[:skipped].size} files.",
+                   "Removed #{file_stats[:removed].size} files"].join(" "))
+      file_stats
+    end
+    
+    def sync(mode)
+      file_stats = {:copied => [], :skipped => []}
+      $LOGGER.info("Sync from #{from_backend} to #{to_backend} started at #{sync_start = Time.now}. Mode: #{mode}. Dry-run? #{!!dry_run?}")
+
+      from_backend_files   = from_backend.files_to_sync(to_backend.upload_prefix)
+      to_backend_files     = to_backend.files_to_sync(from_backend.upload_prefix)
+      total_files          = from_backend_files.size
+      last_decile_complete = 0
+      
+      from_backend_files.each_with_index do |file, index|
+        if (mode == :sync_all || to_backend.needs_update?(file, to_backend_files))
+          file_stats[:copied] << file
+          from_backend.copy(file, to_backend)
+        else
+          file_stats[:skipped] << file
+          $LOGGER.debug("Skipping up-to-date file #{file}")
+        end
+        
+        if decile_complete(index, total_files) != last_decile_complete
+          last_decile_complete = decile_complete(index, total_files)
+          $LOGGER.info("Sync from #{from_backend} to #{to_backend}: Completed #{index} files. #{last_decile_complete * 10}% complete")
+        end
+      end
+      
+      $LOGGER.debug(["Sync from #{from_backend} to #{to_backend} finished at #{Time.now}, took #{Time.now - sync_start}s.",
+                     "Copied #{file_stats[:copied].size} files.",
+                     "Skipped #{file_stats[:skipped].size} files."].join(" "))
+      file_stats
+    end
+    
+    def decile_complete(index, total_files)
+      (index * 100 / total_files) / 10
+    end
+  end
+end

data/lib/cloudsync/version.rb

@@ -0,0 +1,3 @@
+module Cloudsync
+  VERSION = "0.1"
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'cloudsync'
+
+class Test::Unit::TestCase
+end

data/test/test_cloudsync.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestCloudsync < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification 
+name: cloudsync
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Cory Forsyth
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-12 00:00:00 -04:00
+default_executable: cloudsync
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: cloudfiles
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: commander
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: Sync files between various clouds or sftp servers. Available backends are S3, CloudFiles, and SFTP servers. Can sync, mirror, and prune.
+email: cory.forsyth@gmail.com
+executables: 
+- cloudsync
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/cloudsync
+- cloudsync.gemspec
+- lib/cloudsync.rb
+- lib/cloudsync/backend/base.rb
+- lib/cloudsync/backend/cloudfiles.rb
+- lib/cloudsync/backend/s3.rb
+- lib/cloudsync/backend/sftp.rb
+- lib/cloudsync/datetime/datetime.rb
+- lib/cloudsync/file.rb
+- lib/cloudsync/sync_manager.rb
+- lib/cloudsync/version.rb
+- test/helper.rb
+- test/test_cloudsync.rb
+has_rdoc: true
+homepage: http://github.com/megaphone/cloudsync
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Sync files between various clouds or sftp servers.
+test_files: 
+- test/helper.rb
+- test/test_cloudsync.rb