disk_store 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7faa15c4c175c701e092ae666603104e0cd3c79d
-  data.tar.gz: 645d9a33a2dbe7f3174ab36227ab3618ed6d147c
+  metadata.gz: 7de985eb8736731f07a776e31aba1a3847cf34d6
+  data.tar.gz: 121f7f38bbd85a30a3250fd99edd7e1c122a9a56
 SHA512:
-  metadata.gz: 22842f3a27bc962003309cad6e0ccb573667984049615167d22d6024d01b85b3eb369f939e0f809b097c7dd783771ea4379d1478b1e8f5c0c3da39a21ea9fd60
-  data.tar.gz: 3264b63300a777e1c60d234b8b723383b95df29a4053f5227a4f9c4bf00e753965bb1d1ce6b2b1e1d33f7c6f607681c06e159428613731be353794d3ebfe0e77
+  metadata.gz: 44ec9112fdb3bcc1edf9c13fde166237acfa32c73d1e5c9d220e3b108ebdee6cf147d063e0abb59c59c3cb80021d08d4b396932f47be7110499d915bc7a34999
+  data.tar.gz: 81b754f6d6fe78c746c7c655c65623376153f799b7aa9452e38b10a21eb6df98f454bac02f6d8a0af5bf2b364283973d1b930a1fc673b13705aff6fe0b25c3f0

data/.gitignore

@@ -1 +1,2 @@
 *.gem
+tmp

data/.travis.yml

@@ -1,5 +1,4 @@
 language: ruby
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    disk_store (0.1.2)
+    disk_store (0.2.0)
       rake
 
 GEM
@@ -12,7 +12,7 @@ GEM
       safe_yaml (~> 0.9.0)
     diff-lcs (1.2.5)
     multi_json (1.8.4)
-    rake (10.2.2)
+    rake (10.3.1)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)

data/README.md

@@ -24,13 +24,36 @@ examples.
 
 DiskStore requires a directory to to store the files.
 
-It takes a single parameter, which is the directory at which to store the cached files.
-If no parameter is specified, it uses the current directory.
+It has two parameters:
+
+* The first is the path to the cache directory. If no directory is given, then the current directory is used.
+* The second is an options hash that controls the Reaper. More info below.
 
 ```ruby
 cache = DiskStore.new("path/to/my/cache/directory")
 ```
 
+#### Reaper
+
+By default, DiskStore does not perform any evictions on files in the cache. Be careful with this, because
+it can cause your disk to fill up if left to its own means.
+
+The reaper configuration includes:
+
+* `cache_size`: The maximum size of the cache before the reaper begins to evict files (in bytes).
+* `reaper_interval`: How often the reaper will check for files to evict (in seconds).
+* `eviction_strategy`: Sets how the reaper will determine which files to evict.
+
+Current available eviction strategies are:
+
+* LRU (least recently used) - deletes the files with the oldest last access time
+
+To configure DiskStore with an LRU eviction strategy:
+
+``` ruby
+cache = DiskStore.new('cache/dir', eviction_strategy: :LRU)
+```
+
 ### Reading
 
 ```ruby

data/disk_store.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Cache fiels the smart way.}
   gem.homepage      = "http://cosmos.layervault.com/cache.html"
   gem.license       = 'MIT'
+  gem.required_ruby_version = ">= 2.0.0"
 
   gem.files         = `git ls-files`.split($/).delete_if { |f| f.include?('examples/') }
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/disk_store/eviction_strategies/lru.rb

@@ -0,0 +1,34 @@
+class DiskStore
+  class Reaper
+    module LRU
+      def files_to_evict
+        # Collect and sort files based on last access time
+        sorted_files = files
+          .map { |file|
+            data = nil
+            File.new(file, 'rb').tap { |fd|
+              data = { path: file, last_fetch: fd.atime, size: fd.size }
+            }.close
+            data
+          }
+          .sort { |a, b| a[:last_fetch] <=> b[:last_fetch] } # Oldest first
+
+        # Determine which files to evict
+        space_to_evict = current_cache_size - maximum_cache_size
+        space_evicted = 0
+        evictions = []
+        while space_evicted < space_to_evict
+          evicted_file = sorted_files.shift
+          space_evicted += evicted_file[:size]
+          evictions << evicted_file
+        end
+
+        evictions
+      end
+
+      def directories_to_evict
+        empty_directories
+      end
+    end
+  end
+end

data/lib/disk_store/reaper.rb

@@ -0,0 +1,106 @@
+Dir[File.join(File.dirname(__FILE__), "eviction_strategies", "*.rb")].each { |f| require f }
+
+class DiskStore
+  class Reaper
+    DEFAULT_OPTS = {
+      cache_size: 1073741824, # 1 gigabyte
+      reaper_interval: 10, # seconds,
+      eviction_strategy: nil
+    }
+
+    @reapers = {}
+
+    # Spawn exactly 1 reaper for each cache path
+    def self.spawn_for(path, opts = {})
+      return @reapers[path] if @reapers.has_key?(path)
+
+      reaper = Reaper.new(path, opts)
+      reaper.spawn!
+
+      @reapers[path] = reaper
+      reaper
+    end
+
+    # Mostly useful for testing purposes
+    def self.kill_all!
+      @reapers.each { |path, reaper| reaper.thread.kill }
+      @reapers = {}
+    end
+
+    attr_reader :path, :thread
+
+    def initialize(path, opts = {})
+      @path = path
+      @options = DEFAULT_OPTS.merge(opts)
+      @thread = nil
+
+      set_eviction_strategy(@options[:eviction_strategy])
+    end
+
+    def set_eviction_strategy(strategy)
+      return if strategy.nil?
+      self.class.send :prepend, DiskStore::Reaper.const_get(strategy)
+    end
+
+    def spawn!
+      @thread = Thread.new do
+        loop do
+          perform_sweep! if needs_eviction?
+          wait_for_next
+        end
+      end
+    end
+
+    def alive?
+      @thread && @thread.alive?
+    end
+
+    def running?
+      @thread && !@thread.stop?
+    end
+
+    private
+
+    def perform_sweep!
+      # Evict and delete selected files
+      files_to_evict.each { |file| FileUtils.rm(file[:path]) }
+      directories_to_evict.each { |dir| Dir.rmdir(dir) }
+    end
+
+    def needs_eviction?
+      current_cache_size > maximum_cache_size
+    end
+
+    def files_to_evict
+      []
+    end
+
+    def directories_to_evict
+      []
+    end
+
+    def wait_for_next
+      sleep @options[:reaper_interval]
+    end
+
+    def files
+      Dir[File.join(path, "**", "*")].select { |f| File.file?(f) }
+    end
+
+    def directories
+      Dir[File.join(path, "**", "*")].select { |f| File.directory?(f) }
+    end
+
+    def empty_directories
+      directories.select { |d| Dir.entries(d).size == 2 }
+    end
+
+    def current_cache_size
+      files.map { |file| File.new(file).size }.inject { |sum, size| sum + size } || 0
+    end
+
+    def maximum_cache_size
+      @options[:cache_size].to_i
+    end
+  end
+end

data/lib/disk_store/version.rb

@@ -1,3 +1,3 @@
 class DiskStore
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/lib/disk_store.rb

@@ -1,13 +1,18 @@
 require 'open-uri'
+require 'disk_store/reaper'
 
 class DiskStore
   DIR_FORMATTER = "%03X"
   FILENAME_MAX_SIZE = 228 # max filename size on file system is 255, minus room for timestamp and random characters appended by Tempfile (used by atomic write)
   EXCLUDED_DIRS = ['.', '..'].freeze
 
-  def initialize(path=nil)
+  attr_reader :reaper
+
+  def initialize(path=nil, opts = {})
     path ||= "."
     @root_path = File.expand_path path
+    @options = opts
+    @reaper = Reaper.spawn_for(@root_path, @options)
   end
 
   def read(key)

data/spec/reaper_spec.rb

@@ -0,0 +1,129 @@
+require 'spec_helper'
+
+describe DiskStore::Reaper do
+  let(:file_contents) { "Hello, Doge" }
+  let(:file) { Tempfile.new("My Temp File.psd") }
+  let(:key) { "doge" }
+
+  before(:each) do
+    file.write file_contents
+    file.flush
+    file.rewind
+  end
+
+  around(:each) do |example|
+    Dir.mktmpdir do |tmpdir|
+      @tmpdir = tmpdir
+      example.run
+    end
+  end
+
+  after(:each) do
+    DiskStore::Reaper.kill_all!
+  end
+
+  it 'is spawned with a new cache' do
+    cache = DiskStore.new(@tmpdir)
+    expect(cache.reaper).to_not be_nil
+    expect(cache.reaper.path).to eq @tmpdir
+    expect(cache.reaper).to be_alive
+  end
+
+  it 'only spawns one reaper per cache dir' do
+    cache1 = DiskStore.new(@tmpdir)
+    cache2 = DiskStore.new(@tmpdir)
+
+    expect(cache1.reaper).to be cache2.reaper
+  end
+
+  it 'allows you to set the maximum cache size' do
+    cache = DiskStore.new(@tmpdir, cache_size: 2000)
+    expect(cache.reaper.send(:maximum_cache_size)).to eq 2000
+  end
+
+  describe 'cache files' do
+    let (:cache) { DiskStore.new(@tmpdir) }
+    let (:reaper) { cache.reaper }
+
+    before(:each) do
+      cache.write key, file
+    end
+
+    it "correctly calculates the cache size" do
+      expect(reaper.send(:current_cache_size)).to eq file_contents.size
+    end
+
+    it "finds all files" do
+      expect(reaper.send(:files).size).to eq 1
+      expect(File.read(reaper.send(:files).first)).to eq file_contents
+    end
+
+    it "finds all directories" do
+      expect(reaper.send(:directories).size).to eq 2
+    end
+  end
+
+  describe 'LRU eviction' do
+    context 'when below the cache size' do
+      let (:cache) { DiskStore.new(@tmpdir, eviction_strategy: :LRU, cache_size: 1073741824) }
+
+      before(:each) do
+        cache.write key, file
+      end
+
+      it "does not require eviction" do
+        expect(cache.reaper.send(:needs_eviction?)).to be_false
+      end
+    end
+
+    context 'when over the cache size' do
+      let(:file2_contents) { "Hi friend" }
+      let(:file2) { Tempfile.new("Friends are magical.psd") }
+      let(:key2) { "friend" }
+
+      let (:cache) { DiskStore.new(@tmpdir, eviction_strategy: :LRU, cache_size: file_contents.size + 1) }
+      let (:reaper) { cache.reaper }
+
+      before(:each) do
+        # Kill the thread so we can test without it getting
+        # in the way
+        reaper.thread.kill
+        cache.write key, file
+
+        file2.write file2_contents
+        file2.flush
+        file2.rewind
+
+        cache.write key2, file2
+      end
+
+      it "requires eviction" do
+        expect(reaper.send(:needs_eviction?)).to be_true
+      end
+
+      it "correctly determines the files to evict" do
+        files = reaper.send(:files_to_evict).map { |f| f[:path] }
+        
+        expect(files.size).to eq 1
+        expect(File.read(files.first)).to eq file_contents
+      end
+
+      it "removes the file during eviction" do
+        reaper.send(:perform_sweep!)
+
+        expect(reaper.send(:files).size).to eq 1
+        expect(File.read(reaper.send(:files).first)).to eq file2_contents
+      end
+
+      it "correctly determines empty directories" do
+        reaper.send(:files).each { |f| FileUtils.rm(f) }
+        expect(reaper.send(:empty_directories).size).to eq 2
+      end
+
+      it "removes empty directories during eviction" do
+        2.times { reaper.send(:perform_sweep!) }
+        expect(reaper.send(:empty_directories).size).to eq 0
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: disk_store
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Kelly Sutton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -95,9 +95,12 @@ files:
 - Rakefile
 - disk_store.gemspec
 - lib/disk_store.rb
+- lib/disk_store/eviction_strategies/lru.rb
+- lib/disk_store/reaper.rb
 - lib/disk_store/version.rb
 - spec/cassettes/DiskStore/web_resources/should_cache_the_result_of_a_web_resource_in_a_file.json
 - spec/disk_store_spec.rb
+- spec/reaper_spec.rb
 - spec/spec_helper.rb
 homepage: http://cosmos.layervault.com/cache.html
 licenses:
@@ -111,7 +114,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
@@ -119,11 +122,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.9
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Cache fiels the smart way.
 test_files:
 - spec/cassettes/DiskStore/web_resources/should_cache_the_result_of_a_web_resource_in_a_file.json
 - spec/disk_store_spec.rb
+- spec/reaper_spec.rb
 - spec/spec_helper.rb