smooth_s3 0.1.1 → 0.2.0

data/CHANGELOG

@@ -0,0 +1,12 @@
+### 0.2.0 ###
+  - Refactoring run
+  - Connection errors and failed uploads are now all properly rescued by appropriate logic.
+  - Uploads will now be tried up to 3 times before moving on to the next file.
+  - Caching service buckets when initializing, saving tons of requests. This speeds up directory sync operations a lot.
+  - Renamed sync_directory(!) to directory_sync(!) for consistency. Old method names are aliased for backward compatibility.
+
+### 0.1.1 ###
+  - Removed dependency to jeweler (oops!)
+
+### 0.1.0 ###
+  - First release

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/lib/smooth_s3/bucket.rb

@@ -2,16 +2,16 @@ module SmoothS3
   class Bucket
     
     def self.exists?(bucket, service)
-      service.buckets.include? bucket
+      service.refresh.buckets.keys.include? bucket
     end
 
     def self.file_exists?(file, bucket, service)
-      b = service.proxy_service.buckets.find_first(bucket)
+      b = service.buckets[bucket]
 
       begin
         b.objects.find_first(file)
         return true
-      rescue S3::Error::NoSuchKey
+      rescue
         return false
       end
     end
@@ -27,14 +27,15 @@ module SmoothS3
       rescue S3::Error::BucketAlreadyExists
         raise SmoothS3::Error, "A bucket named '#{bucket_name}' already exists in the Global S3 Namespace. Please select one of you existing buckets or try a new name."
       end
+
+      Service.new_buckets[service.aws_key] << new_bucket
     end
 
     def self.store_file(file, remote_file, bucket, service, prefix, overwrite)
-      b = service.proxy_service.buckets.find_first(bucket)
+      b = service.refresh.buckets[bucket]
 
       if prefix
-        remote_file = prefix + remote_file  if prefix =~ /\/$/
-        remote_file = prefix + "/" + remote_file unless prefix =~ /\/$/
+        remote_file = prefix =~ /\/$/ ? (prefix + remote_file) : prefix + "/" + remote_file
       end
 
       unless overwrite == true
@@ -46,13 +47,34 @@ module SmoothS3
 
       bo = b.objects.build(remote_file)
       bo.content = open(file)
-      
-      if bo.save
+ 
+      if Bucket.save_with_retries(3, bo, file)
         puts "'#{file}' was uploaded to S3 bucket '#{bucket}' under the name '#{remote_file}'."
       else
-        puts "There was a problem trying to upload '#{file}' to S3"
+        puts "Impossible to upload '#{file}' to S3 (retries were attempted). Please check file permissions."
       end
     end
 
+    private
+
+      def self.save_with_retries(tries, bucket_object, file)
+        done, attempts_left = false, tries
+
+        until done
+          break if attempts_left <= 0
+
+          begin
+            bucket_object.save
+            done = true
+          rescue
+            puts "There was a problem trying to upload '#{file}' to S3. Retrying..."
+          end
+
+          attempts_left -= 1
+        end
+
+        done
+      end
+
   end
 end

data/lib/smooth_s3/service.rb

@@ -1,6 +1,9 @@
 module SmoothS3
   class Service
     attr_reader :aws_key, :aws_secret, :ssl, :proxy_service
+    attr_accessor :buckets
+
+    @@new_buckets = {}
 
     def initialize(opts={})
       @aws_key = opts.delete(:aws_key)
@@ -10,6 +13,30 @@ module SmoothS3
       @proxy_service = S3::Service.new(:access_key_id => @aws_key, :secret_access_key => @aws_secret, :use_ssl => @ssl)
 
       test_connection
+
+      @buckets = gather_buckets
+      @@new_buckets[@aws_key] = []
+    end
+
+    def gather_buckets
+      service_buckets = {}
+      begin
+        self.proxy_service.buckets.each { |b| service_buckets.merge!(b.name => b) }
+      rescue
+        puts "There was an error trying to fetch the service's buckets. Retrying..."
+        sleep 1
+
+        self.gather_buckets
+      end
+
+      service_buckets
+    end
+
+    def refresh
+      new_buckets = @@new_buckets[self.aws_key]
+      new_buckets.each {|nb| self.buckets[nb.name] = nb}
+      
+      return self
     end
 
     def upload(bucket, files, options={})
@@ -20,12 +47,12 @@ module SmoothS3
       Uploader.upload!(self, bucket, files, options)
     end
 
-    def sync_directory(bucket, directory, options={})
-      Uploader.sync_directory(self, bucket, directory, options)
+    def directory_sync(bucket, directory, options={})
+      Uploader.directory_sync(self, bucket, directory, options)
     end
 
-    def sync_directory!(bucket, directory, options={})
-      Uploader.sync_directory!(self, bucket, directory, options)
+    def directory_sync!(bucket, directory, options={})
+      Uploader.directory_sync!(self, bucket, directory, options)
     end
 
     def timestamped_upload(bucket, files, options={})
@@ -44,11 +71,19 @@ module SmoothS3
       Uploader.timestamped_directory_sync!(self, bucket, directory, options)
     end
 
-    # Utility Methods
-    def buckets
-      @proxy_service.buckets.map { |b| b.name }
+    # Make @@new_buckets accessible outside of the class
+    def self.new_buckets
+      @@new_buckets
     end
 
+    def self.new_buckets=(value)
+      @@new_buckets = value
+    end
+
+    # Preserve backwards compatibility
+    alias_method :sync_directory, :directory_sync
+    alias_method :sync_directory!, :directory_sync!
+
     private
 
       def test_connection

data/lib/smooth_s3/uploader.rb

@@ -2,87 +2,61 @@ module SmoothS3
   class Uploader
 
     def self.upload(service, bucket, files, options={})
-      options[:overwrite] = false unless options[:overwrite]
+      [:overwrite, :timestamped].each {|s| options[s] = false unless options[s]}
       Bucket.select(bucket, service)
       
       valid_files = Uploader.validate_files(files)
       valid_files.each do |vf|
-        Bucket.store_file(vf, vf.split("/")[-1], bucket, service, options[:prefix], options[:overwrite])
+        remote_file_name = options[:timestamped] ? (options[:timestamp] + "_" + vf.split("/")[-1]) : vf.split("/")[-1]
+        Bucket.store_file(vf, remote_file_name, bucket, service, options[:prefix], options[:overwrite])
       end
     end
 
     def self.upload!(service, bucket, files, options={})
-      options.merge!(:overwrite => true)
-      Uploader.upload(service, bucket, files, options)
+      Uploader.upload(service, bucket, files, options.merge!(:overwrite => true))
     end
 
-    def self.sync_directory(service, bucket, directory, options={})
-      options[:overwrite] = false unless options[:overwrite]
+    def self.directory_sync(service, bucket, directory, options={})
+      [:overwrite, :timestamped].each {|s| options[s] = false unless options[s]}
       Bucket.select(bucket, service)
 
       valid_files = Uploader.validate_files_in_directory(directory)
       valid_files.each do |vf|
-        Bucket.store_file(vf[0], vf[1], bucket, service, options[:prefix], options[:overwrite])
+        remote_file_name = options[:timestamped] ? (options[:timestamp] + "_" + vf[1]) : vf[1]
+        Bucket.store_file(vf[0], remote_file_name, bucket, service, options[:prefix], options[:overwrite])
       end
     end
 
-    def self.sync_directory!(service, bucket, directory, options={})
-      options.merge!(:overwrite => true)
-      Uploader.sync_directory(service, bucket, directory, options)
+    def self.directory_sync!(service, bucket, directory, options={})
+      Uploader.directory_sync(service, bucket, directory, options.merge!(:overwrite => true))
     end
 
     def self.timestamped_upload(service, bucket, files, options={})
       options[:overwrite] = false unless options[:overwrite]
-      Bucket.select(bucket, service)
-
-      if options[:timestamp_type] == :epoch
-        timestamp = Time.now.strftime("%s")
-      elsif options[:timestamp_type] == :strftime
-        if options[:timestamp_format]
-          timestamp = Time.now.strftime(options[:timestamp_format])
-        else
-          timestamp = Uploader.default_timestamp
-        end
-      else
-        timestamp = Uploader.default_timestamp
-      end
-
-      valid_files = Uploader.validate_files(files)
-      valid_files.each do |vf|
-        Bucket.store_file(vf, timestamp + "_" + vf.split("/")[-1], bucket, service, options[:prefix], options[:overwrite])
-      end
+      
+      timestamp = Uploader.calculate_timestamp(options)
+      Uploader.upload(service, bucket, files, options.merge!(:timestamped => true, :timestamp => timestamp))
     end
 
     def self.timestamped_upload!(service, bucket, files, options={})
-      options.merge!(:overwrite => true)
-      Uploader.timestamped_upload(service, bucket, files, options)
+      Uploader.timestamped_upload(service, bucket, files, options.merge!(:overwrite => true))
     end
 
     def self.timestamped_directory_sync(service, bucket, directory, options={})
       options[:overwrite] = false unless options[:overwrite]
-      Bucket.select(bucket, service)
-
-      if options[:timestamp_type] == :epoch
-        timestamp = Time.now.strftime("%s")
-      elsif options[:timestamp_type] == :strftime
-        if options[:timestamp_format]
-          timestamp = Time.now.strftime(options[:timestamp_format])
-        else
-          timestamp = Uploader.default_timestamp
-        end
-      else
-        timestamp = Uploader.default_timestamp
-      end
 
-      valid_files = Uploader.validate_files_in_directory(directory)
-      valid_files.each do |vf|
-        Bucket.store_file(vf[0], timestamp + "_" + vf[1], bucket, service, options[:prefix], options[:overwrite])
-      end
+      timestamp = Uploader.calculate_timestamp(options)
+      Uploader.directory_sync(service, bucket, directory, options.merge!(:timestamped => true, :timestamp => timestamp))
     end
 
     def self.timestamped_directory_sync!(service, bucket, directory, options={})
-      options.merge!(:overwrite => true)
-      Uploader.timestamped_directory_sync(service, bucket, directory, options)
+      Uploader.timestamped_directory_sync(service, bucket, directory, options.merge!(:overwrite => true))
+    end
+
+    # Preserve backwards compatibility
+    class << self
+      alias_method :sync_directory, :directory_sync
+      alias_method :sync_directory!, :directory_sync!
     end
 
     private
@@ -121,6 +95,18 @@ module SmoothS3
         valid_files
       end
 
+      def self.calculate_timestamp(options)
+        timestamp = nil
+
+        if options[:timestamp_type] == :epoch
+          timestamp = Time.now.strftime("%s")
+        elsif options[:timestamp_type] == :strftime && options[:timestamp_format]
+          timestamp = Time.now.strftime(options[:timestamp_format])
+        end
+
+        timestamp || Uploader.default_timestamp
+      end
+
       def self.default_timestamp
         Time.now.strftime("%Y%m%d%H%M%S")
       end

data/lib/smooth_s3.rb

@@ -11,5 +11,5 @@ require 'smooth_s3/uploader.rb'
 require 'smooth_s3/error.rb'
 
 module SmoothS3
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smooth_s3
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-12-08 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: s3
-  requirement: &2156974900 !ruby/object:Gem::Requirement
+  requirement: &2153519260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,7 @@ dependencies:
         version: 0.3.9
   type: :runtime
   prerelease: false
-  version_requirements: *2156974900
+  version_requirements: *2153519260
 description: A user-friendly superset of the S3 gem geared towards file system backup
   operations. Simplifies standard actions such as basic uploads, for example allowing
   multiple files to be uploaded in one operation and adds new functionality such as
@@ -37,6 +37,7 @@ files:
 - Gemfile.lock
 - README.textile
 - VERSION
+- CHANGELOG
 - lib/smooth_s3.rb
 - lib/smooth_s3/bucket.rb
 - lib/smooth_s3/error.rb