cloudsync 1.0.4 → 2.0.0

data/VERSION

@@ -1 +1 @@
-1.0.4
+2.0.0

data/cloudsync.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{cloudsync}
-  s.version = "1.0.4"
+  s.version = "2.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Cory Forsyth"]

data/lib/cloudsync/backend/base.rb

@@ -3,24 +3,16 @@ require 'tempfile'
 module Cloudsync
   module Backend
     class Base
-      attr_accessor :store, :sync_manager, :name, :prefix
+      attr_accessor :store, :sync_manager, :name, :upload_prefix
       
       def initialize(opts = {})
         @sync_manager     = opts[:sync_manager]
         @name             = opts[:name]
         @backend_type     = opts[:backend] || self.class.to_s.split("::").last
+        @download_prefix  = opts[:download_prefix] || ""
+        @upload_prefix    = opts[:upload_prefix] || ""
       end
       
-      def upload_prefix
-        {:bucket => @bucket, :prefix => @prefix}
-      end
-      
-      def upload_prefix_path
-        if @bucket && @prefix
-          "#{@bucket}/#{@prefix}"
-        end
-      end
-
       # copy
       def copy(file, to_backend)
         start_copy = Time.now
@@ -37,7 +29,7 @@ module Cloudsync
       end
     
       def to_s
-        "#{@name}[:#{@backend_type}/#{upload_prefix_path}]"
+        "#{@name}[:#{@backend_type}/#{@upload_prefix}]"
       end
       
       # needs_update?
@@ -92,7 +84,7 @@ module Cloudsync
       end
     
       private
-    
+      
       def dry_run?
         return false unless @sync_manager
         @sync_manager.dry_run?

data/lib/cloudsync/backend/s3.rb

@@ -26,12 +26,12 @@ module Cloudsync
     
       def download(file)
         start_time = Time.now
-        $LOGGER.info("Downloading file #{file}")
+        $LOGGER.info("Downloading file #{file} (#{file.path})")
       
         tempfile = file.tempfile
 
         if !dry_run?
-          @store.interface.get(file.bucket, file.path) do |chunk|
+          @store.interface.get(file.bucket, file.download_path) do |chunk|
             tempfile.write chunk
           end
         end
@@ -57,7 +57,7 @@ module Cloudsync
         get_obj_from_store(file).delete
       
         if bucket = @store.bucket(file.bucket)
-          bucket.key(file.path).delete
+          bucket.key(file.download_path).delete
         
           if delete_bucket_if_empty && bucket.keys.empty?
             $LOGGER.debug("Deleting empty bucket '#{bucket.name}'")
@@ -68,9 +68,9 @@ module Cloudsync
         $LOGGER.error("Caught error: #{e} trying to delete #{file}")
       end
     
-      def files_to_sync(upload_prefix={})
+      def files_to_sync(upload_prefix="")
         $LOGGER.info("Getting files to sync [#{self}]")
-      
+        
         buckets_to_sync(upload_prefix).inject([]) do |files, bucket|
           objects_from_bucket(bucket, upload_prefix).collect do |key|
             files << Cloudsync::File.from_s3_obj(key, self.to_s)
@@ -81,18 +81,23 @@ module Cloudsync
 
       private
     
-      def buckets_to_sync(upload_prefix)
-        if upload_prefix[:bucket]
-          [@store.bucket(upload_prefix[:bucket], true)]
+      def buckets_to_sync(upload_prefix="")
+        bucket_name = upload_prefix.split("/").first
+        if bucket_name
+          [@store.bucket(bucket_name, true)]
         else
           @store.buckets
         end
       end
       
-      def objects_from_bucket(bucket, upload_prefix)
-        if upload_prefix[:prefix]
-          bucket.keys(:prefix => upload_prefix[:prefix])
-        else 
+      def objects_from_bucket(bucket, upload_prefix="")
+        prefix_parts = upload_prefix.split("/")
+        prefix_parts.shift
+        prefix = prefix_parts.join("/")
+        
+        if !prefix.empty?
+          bucket.keys(:prefix => prefix)
+        else
           bucket.keys
         end
       end
@@ -107,6 +112,7 @@ module Cloudsync
       end
     
       def get_obj_from_store(file)
+        $LOGGER.debug("gofs, buck: #{file.bucket}. upload path: #{file.upload_path}")
         if bucket = @store.bucket(file.bucket)
           key = bucket.key(file.upload_path)
           return key if key.exists?

data/lib/cloudsync/backend/sftp.rb

@@ -8,27 +8,22 @@ module Cloudsync::Backend
     
     def initialize(options = {})
       @host                 = options[:host]
-      @base_path            = options[:base_path]
       @username             = options[:username]
       @password             = options[:password]
-      prefix_parts = options[:upload_prefix].split("/")
-      
-      @bucket = prefix_parts.shift
-      @prefix = prefix_parts.join("/")
       
       super
     end
     
     # download
     def download(file)
-      $LOGGER.info("Downloading #{file}")
+      $LOGGER.info("Downloading #{file} from #{file.download_path}")
       tempfile = file.tempfile
       
       if !dry_run?
         Net::SSH.start(@host, @username, :password => @password) do |ssh|
           ssh.sftp.connect do |sftp|
             begin
-              sftp.download!(absolute_path(file.path), tempfile)
+              sftp.download!(file.download_path, tempfile)
             rescue RuntimeError => e
               if e.message =~ /permission denied/
                 tempfile.close
@@ -46,12 +41,12 @@ module Cloudsync::Backend
     
     # put
     def put(file, local_filepath)
-      $LOGGER.info("Putting #{file} to #{self}")
+      $LOGGER.info("Putting #{file} to #{self} (#{file.upload_path})")
       return if dry_run?
       
       Net::SSH.start(@host, @username, :password => @password) do |ssh|
         ssh.sftp.connect do |sftp|
-          sftp.upload!(local_filepath, absolute_path(file.path))
+          sftp.upload!(local_filepath, file.upload_path)
         end
       end
     end
@@ -63,7 +58,7 @@ module Cloudsync::Backend
       
       Net::SSH.start(@host, @username, :password => @password) do |ssh|
         ssh.sftp.connect do |sftp|
-          sftp.remove!(absolute_path(file.path))
+          sftp.remove!(file.download_path)
         end
       end
     end
@@ -73,62 +68,71 @@ module Cloudsync::Backend
       files = []
       Net::SSH.start(@host, @username, :password => @password) do |ssh|
         ssh.sftp.connect do |sftp|
-          filepaths = sftp.dir.glob(@base_path, "**/**").collect {|entry| entry.name}
+          filepaths = sftp.dir.glob(@download_prefix, "**/**").collect {|entry| entry.name}
         
           files = filepaths.collect do |filepath|
-            attrs = sftp.stat!(absolute_path(filepath))
+            attrs = sftp.stat!(local_filepath_from_filepath(filepath))
             next unless attrs.file?
 
             e_tag = ssh.exec!(md5sum_cmd(filepath)).split(" ").first
             Cloudsync::File.new \
-              :bucket        => @bucket,
-              :path          => filepath,
-              :size          => attrs.size,
-              :last_modified => attrs.mtime,
-              :e_tag         => e_tag,
-              :backend       => self.to_s
+              :path            => filepath,
+              :upload_prefix   => @upload_prefix,
+              :download_prefix => @download_prefix,
+              :size            => attrs.size,
+              :last_modified   => attrs.mtime,
+              :e_tag           => e_tag,
+              :backend         => self.to_s,
+              :backend_type    => Cloudsync::Backend::Sftp
           end.compact
         end
       end
       files
     end
     
-    def absolute_path(path)
-      @base_path + "/" + path
-    end
+    # def absolute_path(path)
+    #   @download_prefix + "/" + path
+    # end
     
     private
     
     def md5sum_cmd(filepath)
-      Escape.shell_command(["md5sum","#{absolute_path(filepath)}"])
+      Escape.shell_command(["md5sum","#{local_filepath_from_filepath(filepath)}"])
+    end
+    
+    def local_filepath_from_filepath(filepath)
+      stripped_path = filepath.sub(/^#{@upload_prefix}\/?/,"")
+      if @download_prefix
+        "#{@download_prefix}/#{stripped_path}"
+      else
+        stripped_path
+      end
     end
     
     # get_file_from_store
     def get_file_from_store(file)
-      local_filepath = file.path.sub(/^#{@prefix}\/?/,"")
-      
-      $LOGGER.debug("Looking for local filepath: #{local_filepath}")
-      $LOGGER.debug("Abs filepath: #{absolute_path(local_filepath)}")
-      
+      $LOGGER.debug("Looking for local filepath: #{local_filepath_from_filepath(file.full_download_path)}")
+
       sftp_file = nil
       Net::SSH.start(@host, @username, :password => @password) do |ssh|
         ssh.sftp.connect do |sftp|
           begin
-            attrs = sftp.stat!(absolute_path(local_filepath))
+            attrs = sftp.stat!(local_filepath_from_filepath(file.full_download_path))
           rescue Net::SFTP::StatusException => e
             break if e.message =~ /no such file/
             raise
           end
           break unless attrs.file?
           
-          e_tag = ssh.exec!(md5sum_cmd(filepath)).split(" ").first
           sftp_file = Cloudsync::File.new \
-            :bucket        => @bucket,
-            :path          => local_filepath,
-            :size          => attrs.size,
-            :last_modified => attrs.mtime,
-            :e_tag         => e_tag,
-            :backend       => self.to_s
+            :path            => file.download_path,
+            :upload_prefix   => @upload_prefix,
+            :download_prefix => @download_prefix,
+            :size            => attrs.size,
+            :last_modified   => attrs.mtime,
+            :e_tag           => ssh.exec!(md5sum_cmd(file.download_path)).split(" ").first,
+            :backend         => self.to_s,
+            :backend_type    => Cloudsync::Backend::Sftp
         end
       end
       sftp_file

data/lib/cloudsync/file.rb

@@ -1,51 +1,55 @@
 module Cloudsync
   class File
-    attr_accessor :bucket, :path, :size, :last_modified, :e_tag, :backend
-    alias_method :container, :bucket
-    alias_method :container=, :bucket=
+    attr_accessor :path, :size, :last_modified, :e_tag, :backend
     
-    def initialize(options={})
-      @bucket        = options[:bucket]
-      @path          = options[:path] 
-      @size          = options[:size]
-      @last_modified = options[:last_modified]
-      @e_tag         = options[:e_tag]
-      @backend       = options[:backend]
+    def initialize(options = {})
+      @path                = options[:path] 
+      @size                = options[:size]
+      @last_modified       = options[:last_modified]
+      @e_tag               = options[:e_tag]
+      @backend             = options[:backend]
+      @upload_prefix       = options[:upload_prefix]
+      @download_prefix     = options[:download_prefix]
+      @backend_type        = options[:backend_type]
     end
     
     def self.from_s3_obj(obj, backend=nil)
       return nil if obj.nil?
       new({
-        :bucket        => obj.bucket.name,
+        :upload_prefix => obj.bucket.name,
         :path          => obj.name,
         :size          => obj.size,
         :last_modified => obj.last_modified.to_i,
         :e_tag         => obj.e_tag.gsub('"',''),
-        :backend       => backend})
+        :backend       => backend,
+        :backend_type  => Cloudsync::Backend::S3})
     end
     
     def self.from_cf_info(container, path, hash, backend)
-      new({ :bucket        => container.name,
+      new({ 
+            :upload_prefix => container.name,
             :path          => path,
             :size          => hash[:bytes],
             :last_modified => hash[:last_modified].to_gm_time.to_i,
             :e_tag         => hash[:hash],
-            :backend       => backend })
+            :backend       => backend,
+            :backend_type  => Cloudsync::Backend::CloudFiles })
     end
     
     def self.from_cf_obj(obj, backend=nil)
       return nil if obj.nil?
       new({
-        :bucket        => obj.container.name,
+        :upload_prefix => obj.container.name,
         :path          => obj.name,
         :size          => obj.bytes.to_i,
         :last_modified => obj.last_modified.to_i,
         :e_tag         => obj.etag,
-        :backend       => backend})
+        :backend       => backend,
+        :backend_type  => Cloudsync::Backend::CloudFiles})
     end
     
     def to_s
-      "#{path}"
+      "#{full_upload_path}"
     end
     
     def unique_filename
@@ -56,14 +60,34 @@ module Cloudsync
       [bucket,path].join("/")
     end
     
+    def bucket
+      @bucket ||= begin
+        @upload_prefix.split("/").first
+      end
+    end
+    alias_method :container, :bucket
+    
     def upload_path
-      if @prefix
-        @prefix + "/" + @path
-      else
+      without_bucket_path = @upload_prefix.sub(/^#{bucket}\/?/,"")
+      if without_bucket_path.empty?
         @path
+      else
+        without_bucket_path + "/" + @path
       end
     end
     
+    def download_path
+      @download_prefix ? "#{@download_prefix}/#{@path}" : @path
+    end
+    
+    def upload_path_without_bucket
+      
+    end
+    
+    def full_download_path
+      [bucket, download_path].join("/")
+    end
+    
     def full_upload_path
       [bucket, upload_path].join("/")
     end
@@ -72,4 +96,4 @@ module Cloudsync
       Tempfile.new(unique_filename)
     end
   end
-end
+end

data/lib/cloudsync/sync_manager.rb

@@ -74,14 +74,13 @@ module Cloudsync
       
       $LOGGER.info("Prune from #{from_backend} to #{to_backend} started at #{prune_start = Time.now}. Dry-run? #{!!dry_run?}")
       
-      from_backend_files   = [] # from_backend.files_to_sync(to_backend.upload_prefix)
       to_backend_files     = to_backend.files_to_sync(from_backend.upload_prefix)
       total_files          = to_backend_files.size
       last_decile_complete = 0
       
       to_backend_files.each_with_index do |file, index|
         $LOGGER.debug("Checking if file #{file} exists on [#{from_backend}]")
-        if found_file = from_backend.find_file_from_list_or_store(file, from_backend_files)
+        if found_file = from_backend.find_file_from_list_or_store(file)
           $LOGGER.debug("Keeping file #{file} because it was found on #{from_backend}.")
           file_stats[:skipped] << file
         else
@@ -108,12 +107,11 @@ module Cloudsync
       $LOGGER.info("Sync from #{from_backend} to #{to_backend} started at #{sync_start = Time.now}. Mode: #{mode}. Dry-run? #{!!dry_run?}")
 
       from_backend_files   = from_backend.files_to_sync(to_backend.upload_prefix)
-      to_backend_files     = [] # to_backend.files_to_sync(from_backend.upload_prefix)
       total_files          = from_backend_files.size
       last_decile_complete = 0
       
       from_backend_files.each_with_index do |file, index|
-        if (mode == :sync_all || to_backend.needs_update?(file, to_backend_files))
+        if (mode == :sync_all || to_backend.needs_update?(file))
           file_stats[:copied] << file
           from_backend.copy(file, to_backend)
         else

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cloudsync
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 15
   prerelease: false
   segments: 
-  - 1
+  - 2
   - 0
-  - 4
-  version: 1.0.4
+  - 0
+  version: 2.0.0
 platform: ruby
 authors: 
 - Cory Forsyth