defog 0.2.0 → 0.3.0

data/README.rdoc

@@ -98,6 +98,11 @@ In addition, the handle allows you to look up the path where the local proxy fil
 
     defog.file("key").proxy_path  # => Pathname where proxy file is, was, or will be
 
+You can also iterate through handles of all cloud files, e.g.:
+
+    defog.each { |handle| puts handle.key }
+    defog.each.select { |handle| handle.last_modified < 12.hours.ago }
+
 === Persistence
 
 By default, Defog will delete the local proxy when closing a file.
@@ -170,7 +175,14 @@ If this would not free up enough space (because of open proxies or just
 because the remote is larger than the cache), raises
 Defog::Error::CacheFull and doesn't actually delete anything.
 
-You can also manually delete an individual persisted file, such as via:
+For writeable proxes, of course Defog doesn't know in advance the size of
+the data you will write into proxy file.  As a crude estimate, if the
+remote file already exists, Defog will reserve the same amount of space.
+Instead, you can tell Defog the expected size via:
+
+  defog.file("key", "w", :size_hint => size-in-bytes)
+
+You can also manage the cache manually, by explicitly deleting an individual persisted proxy files, such as via:
 
         defog.file("key").proxy_path.unlink
 

data/lib/defog/file.rb

@@ -33,7 +33,7 @@ module Defog
   class File < ::File
 
     def initialize(opts={}, &block) #:nodoc:
-      opts = opts.keyword_args(:handle => :required, :mode => :required, :persist => :optional)
+      opts = opts.keyword_args(:handle => :required, :mode => :required, :persist => :optional, :size_hint => :optional)
       @handle = opts.handle
       @persist = opts.persist
 
@@ -42,21 +42,28 @@ module Defog
       proxy_path.dirname.mkpath
       case opts.mode
       when "r"
-        create_proxy
+        download = true
+        @upload = false
+        cache_size = @handle.size
       when "w", "w+"
+        download = false
         @upload = true
+        cache_size = opts.size_hint || @handle.size
       when "r+", "a", "a+"
-        create_proxy
+        download = true
         @upload = true
+        cache_size = [opts.size_hint, @handle.size].compact.max
       else
         raise ArgumentError, "Invalid mode #{opts.mode.inspect}"
       end
 
+      @handle.proxy.manage_cache(cache_size, proxy_path)
+      @handle.proxy.reserve_proxy_path(proxy_path)
+      download_proxy if download
       super(proxy_path, opts.mode, &block)
     end
 
-    def create_proxy
-      @handle.proxy.open_proxy_file(@handle)
+    def download_proxy
       @handle.proxy.fog_wrapper.get_file(@handle.key, @handle.proxy_path)
     end
 
@@ -82,11 +89,12 @@ module Defog
     def close(opts={})
       opts = opts.keyword_args(:persist => @persist, :synchronize => true)
       super()
-      if @handle.proxy_path.exist?
+      proxy_path = @handle.proxy_path
+      if proxy_path.exist?
         upload_proxy if @upload and opts.synchronize
-        @handle.proxy_path.unlink unless opts.persist
+        proxy_path.unlink unless opts.persist
       end
-      @handle.proxy.close_proxy_file(@handle)
+      @handle.proxy.release_proxy_path(proxy_path)
     end
   end
 end

data/lib/defog/handle.rb

@@ -93,7 +93,7 @@ module Defog
     # to suppress deleting the file and so maintain the file after closing.   See File#close for more
     # details.
     def open(mode, opts={}, &block)
-      opts = opts.keyword_args(:persist => @proxy.persist)
+      opts = opts.keyword_args(:persist => @proxy.persist, :size_hint => :optional)
       File.open(opts.merge(:handle => self, :mode => mode), &block)
     end
 

data/lib/defog/proxy.rb

@@ -54,7 +54,7 @@ module Defog
       @proxy_root = Pathname.new(opts.delete(:proxy_root)) if opts.proxy_root
       @persist = opts.delete(:persist)
       @max_cache_size = opts.delete(:max_cache_size)
-      @open_proxy_paths = Set.new
+      @reserved_proxy_paths = Set.new
 
       @fog_wrapper = FogWrapper.connect(opts)
 
@@ -111,39 +111,55 @@ module Defog
       end
     end
 
-    def open_proxy_file(handle) #:nodoc:
-      manage_cache(handle) if max_cache_size
-      @open_proxy_paths << handle.proxy_path
+    # Iterate through the cloud storage, yielding a Defog::Handle for each
+    # remote file.
+    #
+    # If no block is given, an enumerator is returned.
+    def each
+      if block_given?
+        @fog_wrapper.fog_directory.files.all.each do |fog_model|
+          yield file(fog_model.key)
+        end
+      else
+        to_enum(:each)
+      end
     end
 
-    def close_proxy_file(handle) #:nodoc:
-      @open_proxy_paths.delete handle.proxy_path
+    ###############################
+    # public-but-internal methods
+    #
+
+    def reserve_proxy_path(proxy_path) #:nodoc:
+      @reserved_proxy_paths << proxy_path
     end
 
-    private
+    def release_proxy_path(proxy_path) #:nodoc:
+      @reserved_proxy_paths.delete proxy_path
+    end
 
-    def manage_cache(handle)
-      remote_size = handle.size
-      proxy_path = handle.proxy_path
+    def manage_cache(want_size, proxy_path) #:nodoc:
+      return if max_cache_size.nil?
+      return if want_size.nil?
+      return if want_size <= 0
 
       # find available space (not counting current proxy)
       available = max_cache_size
       proxy_root.find { |path| available -= path.size if path.file? and path != proxy_path}
-      return if available >= remote_size
+      return if available >= want_size
 
-      space_needed = remote_size - available
+      space_needed = want_size - available
 
       # find all paths in the cache that aren't currently open (not
       # counting current proxy)
       candidates = []
-      proxy_root.find { |path| candidates << path if path.file? and not @open_proxy_paths.include?(path) and path != proxy_path}
+      proxy_root.find { |path| candidates << path if path.file? and not @reserved_proxy_paths.include?(path) and path != proxy_path}
 
       # take candidates in LRU order until that would be enough space
       would_free = 0
       candidates = Set.new(candidates.sort_by(&:atime).take_while{|path| (would_free < space_needed).tap{|condition| would_free += path.size}})
 
       # still not enough...?
-      raise Error::CacheFull, "No room in cache for #{handle.key.inspect}: size=#{remote_size} available=#{available} can_free=#{would_free}, max_cache_size=#{max_cache_size}" if would_free < space_needed
+      raise Error::CacheFull, "No room in cache for #{proxy_path.relative_path_from(proxy_root)}: size=#{want_size} available=#{available} can_free=#{would_free} (max_cache_size=#{max_cache_size})" if would_free < space_needed
 
       # LRU order may have taken more than needed, if last file was a big
       # chunk.  So take another pass, eliminating files that aren't needed.

data/lib/defog/version.rb

@@ -1,3 +1,3 @@
 module Defog
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/proxy_spec.rb

@@ -39,6 +39,31 @@ shared_examples "a proxy" do |args|
     end
   end
 
+  context "iteration" do
+
+    before(:each) do
+      @proxy = Defog::Proxy.new(args)
+      @proxy.fog_directory.files.all.each do |model| model.destroy end
+      create_other_remote("i0", 10)
+      create_other_remote("i1", 10)
+      create_other_remote("i2", 10)
+    end
+
+    it "should iterate through remotes" do
+      seen = []
+      @proxy.each do |handle|
+        seen << handle.key
+      end
+      seen.should =~ [other_key("i0"), other_key("i1"), other_key("i2")]
+    end
+
+    it "should return an enumerator" do
+      @proxy.each.map(&:key).should =~ [other_key("i0"), other_key("i1"), other_key("i2")]
+    end
+
+  end
+
+
   context "proxy root location" do
     it "should default proxy root to tmpdir/defog" do
       proxy = Defog::Proxy.new(args)
@@ -66,6 +91,10 @@ shared_examples "a proxy" do |args|
       @proxy.proxy_root.mkpath
     end
 
+    it "should fail normally when trying to proxy a file that doesn't exist" do
+      expect { @proxy.file("nonesuch", "r") }.should raise_error(Defog::Error::NoCloudFile)
+    end
+
     it "should raise an error trying to proxy a file larger than the cache" do
       create_remote("x" * 101)
       expect { @proxy.file(key, "r") }.should raise_error(Defog::Error::CacheFull)
@@ -92,20 +121,31 @@ shared_examples "a proxy" do |args|
       other_proxy_path("c").should_not be_exist
     end
 
+    it "should delete proxies to make room for hinted size" do
+      create_other_proxy("a", 10)
+      create_other_proxy("b", 30)
+      create_other_proxy("c", 40)
+      expect { @proxy.file(key, "w", :size_hint => 80) do end }.should_not raise_error(Defog::Error::CacheFull)
+      proxy_path.should be_exist
+      other_proxy_path("a").should be_exist
+      other_proxy_path("b").should_not be_exist
+      other_proxy_path("c").should_not be_exist
+    end
+
     it "should not delete proxies that are open" do
       create_other_proxy("a", 20)
       create_other_proxy("b", 20)
       create_other_remote("R", 30)
-      create_other_remote("S", 30)
       create_remote("x" * 30)
-      @proxy.file("R", "r") do
-        @proxy.file("S", "r") do
+      @proxy.file(other_key("R"), "r") do
+        @proxy.file(other_key("S"), "w") do
+          create_other_proxy("S", 30)
           expect { @proxy.file(key, "r") do end }.should_not raise_error(Defog::Error::CacheFull)
           proxy_path.should be_exist
-          other_proxy_path("a").should_not be_exist
-          other_proxy_path("b").should_not be_exist
           other_proxy_path("R").should be_exist
           other_proxy_path("S").should be_exist
+          other_proxy_path("a").should_not be_exist
+          other_proxy_path("b").should_not be_exist
         end
       end
     end
@@ -113,7 +153,7 @@ shared_examples "a proxy" do |args|
     it "should delete proxies that are no longer open" do
       create_other_remote("R", 60)
       create_remote("z" * 60)
-      @proxy.file("R", "r") do end
+      @proxy.file(other_key("R"), "r") do end
       other_proxy_path("R").should be_exist
       expect { @proxy.file(key, "r") do end }.should_not raise_error(Defog::Error::CacheFull)
       proxy_path.should be_exist
@@ -123,11 +163,11 @@ shared_examples "a proxy" do |args|
     it "should not delete proxies if there wouldn't be enough space" do
       create_other_proxy("a", 20)
       create_other_proxy("b", 20)
-      create_other_remote("r", 30)
-      create_other_remote("s", 30)
+      create_other_remote("R", 30)
+      create_other_remote("S", 30)
       create_remote("z" * 50)
-      @proxy.file("r", "r") do
-        @proxy.file("s", "r") do
+      @proxy.file(other_key("R"), "r") do
+        @proxy.file(other_key("S"), "r") do
           expect { @proxy.file(key, "r") do end }.should raise_error(Defog::Error::CacheFull)
           proxy_path.should_not be_exist
           other_proxy_path("a").should be_exist
@@ -138,22 +178,28 @@ shared_examples "a proxy" do |args|
       end
     end
 
-    private
+  end
 
-    def create_other_proxy(otherkey, size)
-      other_proxy_path(otherkey).open("w") do |f|
-        f.write("x" * size)
-      end
-    end
+  private
 
-    def other_proxy_path(otherkey)
-      @proxy.file(otherkey).proxy_path
-    end
+  def other_key(okey)
+    "#{key}-#{okey}"
+  end
 
-    def create_other_remote(otherkey, size)
-      @proxy.fog_directory.files.create(:key => otherkey, :body => "x" * size)
+  def create_other_proxy(okey, size)
+    path = other_proxy_path(okey)
+    path.dirname.mkpath
+    path.open("w") do |f|
+      f.write("x" * size)
     end
+  end
+
+  def other_proxy_path(okey)
+    @proxy.file(other_key(okey)).proxy_path
+  end
 
+  def create_other_remote(okey, size)
+    @proxy.fog_directory.files.create(:key => other_key(okey), :body => "x" * size)
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: defog
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &70234406646360 !ruby/object:Gem::Requirement
+  requirement: &70100481071360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70234406646360
+  version_requirements: *70100481071360
 - !ruby/object:Gem::Dependency
   name: hash_keyword_args
-  requirement: &70234406645920 !ruby/object:Gem::Requirement
+  requirement: &70100481070940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70234406645920
+  version_requirements: *70100481070940
 - !ruby/object:Gem::Dependency
   name: fastandand
-  requirement: &70234406645400 !ruby/object:Gem::Requirement
+  requirement: &70100481070520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70234406645400
+  version_requirements: *70100481070520
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70234406644900 !ruby/object:Gem::Requirement
+  requirement: &70100481070100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70234406644900
+  version_requirements: *70100481070100
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70234406644280 !ruby/object:Gem::Requirement
+  requirement: &70100481069680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70234406644280
+  version_requirements: *70100481069680
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70234406643420 !ruby/object:Gem::Requirement
+  requirement: &70100481069240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70234406643420
+  version_requirements: *70100481069240
 - !ruby/object:Gem::Dependency
   name: simplecov-gem-adapter
-  requirement: &70234406643000 !ruby/object:Gem::Requirement
+  requirement: &70100481068720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +87,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70234406643000
+  version_requirements: *70100481068720
 description: Wrapper to fog gem, proxying access to cloud files as local files.
 email:
 - ronen@barzel.org