defog 0.3.2 → 0.4.0

data/lib/defog/file.rb

@@ -43,9 +43,14 @@ module Defog
   class File < ::File
 
     def initialize(opts={}, &block) #:nodoc:
-      opts = opts.keyword_args(:handle => :required, :mode => :required, :persist => :optional, :size_hint => :optional)
+      opts = opts.keyword_args(:handle => :required,
+                               :mode => :required,
+                               :persist => :optional,
+                               :synchronize => { :valid => [:async, true, false], :default => true},
+                               :size_hint => :optional)
       @handle = opts.handle
       @persist = opts.persist
+      @synchronize = opts.synchronize
 
       key = @handle.key
       proxy_path = @handle.proxy_path
@@ -75,38 +80,51 @@ module Defog
       super(proxy_path, opts.mode, &block)
     end
 
-    def download_proxy
-      @handle.proxy.fog_wrapper.get_file(@handle.key, @handle.proxy_path, @encoding)
-    end
-
-    def upload_proxy
-      @handle.proxy.fog_wrapper.put_file(@handle.key, @handle.proxy_path, @encoding) 
-    end
-
-
     # Closes the proxy file and synchronizes the cloud storage (if it was
     # opened as writeable) then deletes the proxy file.
     #
-    # Synchronization can be suppressed by passing the option
-    #    :synchronize => false
+    # Synchronization (i.e. upload of a proxy) can be controlled by passing the option
+    #    :synchronize => :async # upload asynchronously in a separate thread
+    #    :synchronize => true   # upload synchronously
+    #    :synchronize => false  # don't upload
     # Synchronization will also be implicitly suppressed if the proxy file
     # was deleted before this call, e.g., via <code>::File.unlink(file.path)</code>.
     #
-    #
     # Whether the proxy file gets deleted vs persisted after the close can
     # be set by passing the option
     #    :persist => true or false
-    # (This will override the setting of <code>:persist</code> passed to Proxy#file)
+    #
+    # The :persist and :synchronize values override the settings passed to
+    # Handle#open, which in turn overrides the settings passed to Proxy.new
     #
     def close(opts={})
-      opts = opts.keyword_args(:persist => @persist, :synchronize => true)
+      opts = opts.keyword_args(:persist => @persist,
+                               :synchronize => { :valid => [true, false, :async], :default => @synchronize })
+      @persist = opts.persist
+      @synchronize = opts.synchronize
       super()
-      proxy_path = @handle.proxy_path
-      if proxy_path.exist?
-        upload_proxy if @upload and opts.synchronize
-        proxy_path.unlink unless opts.persist
+      if @handle.proxy_path.exist?
+        if @upload and @synchronize == :async
+          Thread.new { wrap_proxy }
+        else
+          wrap_proxy
+        end
       end
-      @handle.proxy.release_proxy_path(proxy_path)
+      @handle.proxy.release_proxy_path(@handle.proxy_path)
+    end
+
+    def download_proxy #:nodoc:
+      @handle.proxy.fog_wrapper.get_file(@handle.key, @handle.proxy_path, @encoding)
+    end
+
+    def upload_proxy #:nodoc:
+      @handle.proxy.fog_wrapper.put_file(@handle.key, @handle.proxy_path, @encoding) 
     end
+
+    def wrap_proxy #:nodoc:
+      upload_proxy if @upload and @synchronize
+      @handle.proxy_path.unlink unless @persist
+    end
+
   end
 end

data/lib/defog/fog_wrapper.rb

@@ -29,6 +29,7 @@ module Defog #:nodoc: all
     end
 
     def put_file(key, path, encoding)
+      return if path.exist? and fog_head(key) and Digest::MD5.hexdigest(path.read) == get_md5(key)
       path.open("r#{encoding}") do |file|
         fog_directory.files.create(:key => key, :body => file)
       end
@@ -38,6 +39,8 @@ module Defog #:nodoc: all
       fog_directory.files.head(key)
     end
 
+    private
+
     class Local < FogWrapper
       def provider ; :local ; end
 
@@ -70,7 +73,7 @@ module Defog #:nodoc: all
       def initialize(opts={})
         opts = opts.keyword_args(:aws_access_key_id => :required, :aws_secret_access_key => :required, :region => :optional, :bucket => :required)
         @location = opts.delete(:bucket)
-        @fog_connection = Fog::Storage.new(opts.merge(:provider => provider))
+        @fog_connection = (@@aws_connection_cache||={})[opts] ||= Fog::Storage.new(opts.merge(:provider => provider))
         @fog_connection.directories.create :key => @location unless @fog_connection.directories.map(&:key).include? @location
         @fog_directory = @fog_connection.directories.get(@location)
       end

data/lib/defog/handle.rb

@@ -88,19 +88,27 @@ module Defog
     # Like ::File.open, if called with a block yields the file object to
     # the block and ensures the file will be closed when leaving the block.
     #
-    # Normally upon close the proxy file is synchronized as needed and then deleted.
-    # Pass
-    #    :persist => true
-    # to suppress deleting the file and so maintain the file after closing.   See File#close for more
-    # details.
+    # Normally the proxy file gets deleted upon close (after synchronized
+    # as needed) rather than persisted, although the default behavior can
+    # be controlled by Defog::Proxy.new.  To specify persistence behavior
+    # on a per-file basis, use
+    #    :persist => true-or-false
+    # See File#close for more details.
     #
     # If you are managing your cache size, when opening a proxy for writing
     # you may want to provide a hint as to the expected size of the data:
     #    :size_hint => 500.kilobytes
     # See README for more details.
     #
+    # Normally upon close of a writeable proxy file, the synchronization
+    # happens synchronously and the close will wait, althrough the behavior
+    # can be controlled by Defog::Proxy.new.  To specify synchronization
+    # behavior on a per-file basis, use
+    #    :synchronize => true-or-false-or-async
+    # See File#close for more details.
+    #
     def open(mode, opts={}, &block)
-      opts = opts.keyword_args(:persist => @proxy.persist, :size_hint => :optional)
+      opts = opts.keyword_args(:persist => @proxy.persist, :synchronize => @proxy.synchronize, :size_hint => :optional)
       File.open(opts.merge(:handle => self, :mode => mode), &block)
     end
 

data/lib/defog/proxy.rb

@@ -8,6 +8,7 @@ module Defog
 
     attr_reader :proxy_root
     attr_reader :persist
+    attr_reader :synchronize
     attr_reader :max_cache_size
     attr_reader :fog_wrapper    # :nodoc:
 
@@ -21,7 +22,9 @@ module Defog
     # <code>:local</code> and <code>:AWS</code> are supported.  When using
     # <code>:AWS</code>, an additional option <code>:bucket</code> must be
     # specified; all files proxied by this instance must be in a single
-    # bucket.
+    # bucket.  (It's OK to create multiple Defog::Proxy instances with
+    # the same access info but different buckets; they will internally
+    # share a single Fog::Storage isntance hence AWS connection.)
     #
     # By default, each proxy's root directory is placed in a reasonable
     # safe place, under <code>Rails.root/tmp</code> if Rails is defined
@@ -34,25 +37,41 @@ module Defog
     # to worry about it.  But if you do care, you can specify the option:
     #   :proxy_root => "/root/for/this/proxy/files"
     #
-    # You can turn on persistence of local proxy files by specifying
+    # You can specify that by default local proxy files will be persisted,
+    # by specifying
     #   :persist => true
     # The persistence behavior can be overriden on a per-file basis when
-    # opening a proxy (see Defog::Handle#open)
+    # opening or closing a proxy (see Defog::Handle#open, Defog::File#close)
     #
     # You can enable cache management by specifying a max cache size in
     # bytes, e.g.
     #    :max_cache_size => 3.gigabytes
     # See the README for discussion.  [Number#gigabytes is defined in
     # Rails' ActiveSupport core extensions]
+    # 
+    # Normally synchronization (i.e. upload) of changes to local proxy
+    # files happens synchronously on close; i.e. Defog::File#close waits
+    # until the upload completes.  However, you can control synchronization
+    # by specifying
+    #     :synchronize => :async        # Synchronize in a separate thread, don't wait
+    #     :synchronize => false         # Don't synchronize at all.  Defeats the purpose of Defog
+    #     :synchronize => true          # This is the default behavior
+    # The synchronization behavior can be overridden on a per-file basis
+    # when opening or closing a proxy (see Defog::Handle#open,
+    # Defog::File#close).  Note that this applies only to upload of changes to
+    # proxy files that are opened as writeable; the download of data to
+    # readable proxy files always happens synchronously.
     def initialize(opts={})
       opts = opts.keyword_args(:provider => :required,
                                :proxy_root => :optional,
                                :persist => :optional,
+                               :synchronize => {:valid => [:async, true, false], :default => true},
                                :max_cache_size => :optional,
                                :OTHERS => :optional)
 
       @proxy_root = Pathname.new(opts.delete(:proxy_root)) if opts.proxy_root
       @persist = opts.delete(:persist)
+      @synchronize = opts.delete(:synchronize)
       @max_cache_size = opts.delete(:max_cache_size)
       @reserved_proxy_paths = Set.new
 

data/lib/defog/version.rb

@@ -1,3 +1,3 @@
 module Defog
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/spec/file_spec.rb

@@ -114,6 +114,17 @@ shared_examples "create" do
     expect {remote_body}.should raise_error
   end
 
+  it "should create remote asynchronously if :synchronize => async" do
+    file = @proxy.file(key, @mode)
+    create_proxy("upload me in thread")
+    Thread.should_receive(:new) { |&block|
+      expect {remote_body}.should raise_error
+      block.call
+    }
+    file.close(:synchronize => :async)
+    remote_body.should == "upload me in thread"
+  end
+
 end
 
 shared_examples "update" do
@@ -127,6 +138,18 @@ shared_examples "update" do
     remote_body.should == "upload me"
   end
 
+  it "should overwrite remote asynchronously if :synchronize => :async" do
+    create_remote("overwrite me")
+    file = @proxy.file(key, @mode)
+    create_proxy("upload me")
+    Thread.should_receive(:new) { |&block|
+      remote_body.should == "overwrite me"
+      block.call
+    }
+    file.close(:synchronize => :async)
+    remote_body.should == "upload me"
+  end
+
   it "should not overwrite remote if proxy is deleted" do
     create_remote("keep me")
     @proxy.file(key, @mode) do |file|
@@ -206,8 +229,8 @@ shared_examples "a proxy file" do |proxyargs|
       it_should_behave_like "read" if mode == "r" or mode == "a+"
       it_should_behave_like "write" if mode =~ %r{[wa+]}
       it_should_behave_like "read after write" if mode == "w+"
+      it_should_behave_like "create" if mode =~ %r{w}
       it_should_behave_like "append" if mode =~ %r{a}
-      it_should_behave_like "create" if mode =~ %r{wa}
       it_should_behave_like "update" if mode =~ %r{[wa+]}
       it_should_behave_like "persistence"
     end

data/spec/proxy_spec.rb

@@ -39,6 +39,20 @@ shared_examples "a proxy" do |args|
     end
   end
 
+  context "settings" do
+    it "should set default for :persist => true" do
+      @proxy = Defog::Proxy.new(args.merge(:persist => true))
+      Defog::File.should_receive(:open).with(hash_including :persist => true)
+      @proxy.file(key, "w") do end
+    end
+    it "should set default for :synchronize => :async" do
+      @proxy = Defog::Proxy.new(args.merge(:synchronize => :async))
+      Defog::File.should_receive(:open).with(hash_including :synchronize => :async)
+      @proxy.file(key, "w") do end
+    end
+  end
+
+
   context "iteration" do
 
     before(:each) do
@@ -232,6 +246,25 @@ describe Defog::Proxy do
     it "should use the bucket name as the location" do
       Defog::Proxy.new(args).location.should == args[:bucket]
     end
+
+    it "should share fog connection with same bucket" do
+      proxy1 = Defog::Proxy.new(args)
+      proxy2 = Defog::Proxy.new(args)
+      proxy1.fog_connection.should be_equal proxy2.fog_connection
+    end
+
+    it "should share fog connection with different bucket" do
+      proxy1 = Defog::Proxy.new(args)
+      proxy2 = Defog::Proxy.new(args.merge(:bucket => "other"))
+      proxy1.fog_connection.should be_equal proxy2.fog_connection
+    end
+
+    it "should not share fog connection with different connection args" do
+      proxy1 = Defog::Proxy.new(args)
+      proxy2 = Defog::Proxy.new(args.merge(:aws_access_key_id => "other"))
+      proxy1.fog_connection.should_not be_equal proxy2.fog_connection
+    end
+
   end
 
   it "should raise error on bad provider" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: defog
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-29 00:00:00.000000000 Z
+date: 2012-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &70315023604540 !ruby/object:Gem::Requirement
+  requirement: &70345231086320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70315023604540
+  version_requirements: *70345231086320
 - !ruby/object:Gem::Dependency
   name: hash_keyword_args
-  requirement: &70315023604100 !ruby/object:Gem::Requirement
+  requirement: &70345231085600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70315023604100
+  version_requirements: *70345231085600
 - !ruby/object:Gem::Dependency
   name: fastandand
-  requirement: &70315023603660 !ruby/object:Gem::Requirement
+  requirement: &70345231084580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70315023603660
+  version_requirements: *70345231084580
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70315023603220 !ruby/object:Gem::Requirement
+  requirement: &70345231082860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315023603220
+  version_requirements: *70345231082860
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70315023602800 !ruby/object:Gem::Requirement
+  requirement: &70345231005400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315023602800
+  version_requirements: *70345231005400
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70315023602380 !ruby/object:Gem::Requirement
+  requirement: &70345231001760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315023602380
+  version_requirements: *70345231001760
 - !ruby/object:Gem::Dependency
   name: simplecov-gem-adapter
-  requirement: &70315023601960 !ruby/object:Gem::Requirement
+  requirement: &70345231000720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +87,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315023601960
+  version_requirements: *70345231000720
 description: Wrapper to fog gem, proxying access to cloud files as local files.
 email:
 - ronen@barzel.org