disk_store 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7faa15c4c175c701e092ae666603104e0cd3c79d
4
- data.tar.gz: 645d9a33a2dbe7f3174ab36227ab3618ed6d147c
3
+ metadata.gz: 7de985eb8736731f07a776e31aba1a3847cf34d6
4
+ data.tar.gz: 121f7f38bbd85a30a3250fd99edd7e1c122a9a56
5
5
  SHA512:
6
- metadata.gz: 22842f3a27bc962003309cad6e0ccb573667984049615167d22d6024d01b85b3eb369f939e0f809b097c7dd783771ea4379d1478b1e8f5c0c3da39a21ea9fd60
7
- data.tar.gz: 3264b63300a777e1c60d234b8b723383b95df29a4053f5227a4f9c4bf00e753965bb1d1ce6b2b1e1d33f7c6f607681c06e159428613731be353794d3ebfe0e77
6
+ metadata.gz: 44ec9112fdb3bcc1edf9c13fde166237acfa32c73d1e5c9d220e3b108ebdee6cf147d063e0abb59c59c3cb80021d08d4b396932f47be7110499d915bc7a34999
7
+ data.tar.gz: 81b754f6d6fe78c746c7c655c65623376153f799b7aa9452e38b10a21eb6df98f454bac02f6d8a0af5bf2b364283973d1b930a1fc673b13705aff6fe0b25c3f0
data/.gitignore CHANGED
@@ -1 +1,2 @@
1
1
  *.gem
2
+ tmp
data/.travis.yml CHANGED
@@ -1,5 +1,4 @@
1
1
  language: ruby
2
2
  rvm:
3
- - 1.9.3
4
3
  - 2.0.0
5
4
  - 2.1.0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- disk_store (0.1.2)
4
+ disk_store (0.2.0)
5
5
  rake
6
6
 
7
7
  GEM
@@ -12,7 +12,7 @@ GEM
12
12
  safe_yaml (~> 0.9.0)
13
13
  diff-lcs (1.2.5)
14
14
  multi_json (1.8.4)
15
- rake (10.2.2)
15
+ rake (10.3.1)
16
16
  rspec (2.14.1)
17
17
  rspec-core (~> 2.14.0)
18
18
  rspec-expectations (~> 2.14.0)
data/README.md CHANGED
@@ -24,13 +24,36 @@ examples.
24
24
 
25
25
  DiskStore requires a directory to to store the files.
26
26
 
27
- It takes a single parameter, which is the directory at which to store the cached files.
28
- If no parameter is specified, it uses the current directory.
27
+ It has two parameters:
28
+
29
+ * The first is the path to the cache directory. If no directory is given, then the current directory is used.
30
+ * The second is an options hash that controls the Reaper. More info below.
29
31
 
30
32
  ```ruby
31
33
  cache = DiskStore.new("path/to/my/cache/directory")
32
34
  ```
33
35
 
36
+ #### Reaper
37
+
38
+ By default, DiskStore does not perform any evictions on files in the cache. Be careful with this, because
39
+ it can cause your disk to fill up if left to its own means.
40
+
41
+ The reaper configuration includes:
42
+
43
+ * `cache_size`: The maximum size of the cache before the reaper begins to evict files (in bytes).
44
+ * `reaper_interval`: How often the reaper will check for files to evict (in seconds).
45
+ * `eviction_strategy`: Sets how the reaper will determine which files to evict.
46
+
47
+ Current available eviction strategies are:
48
+
49
+ * LRU (least recently used) - deletes the files with the oldest last access time
50
+
51
+ To configure DiskStore with an LRU eviction strategy:
52
+
53
+ ``` ruby
54
+ cache = DiskStore.new('cache/dir', eviction_strategy: :LRU)
55
+ ```
56
+
34
57
  ### Reading
35
58
 
36
59
  ```ruby
data/disk_store.gemspec CHANGED
@@ -12,6 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.summary = %q{Cache fiels the smart way.}
13
13
  gem.homepage = "http://cosmos.layervault.com/cache.html"
14
14
  gem.license = 'MIT'
15
+ gem.required_ruby_version = ">= 2.0.0"
15
16
 
16
17
  gem.files = `git ls-files`.split($/).delete_if { |f| f.include?('examples/') }
17
18
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -0,0 +1,34 @@
1
+ class DiskStore
2
+ class Reaper
3
+ module LRU
4
+ def files_to_evict
5
+ # Collect and sort files based on last access time
6
+ sorted_files = files
7
+ .map { |file|
8
+ data = nil
9
+ File.new(file, 'rb').tap { |fd|
10
+ data = { path: file, last_fetch: fd.atime, size: fd.size }
11
+ }.close
12
+ data
13
+ }
14
+ .sort { |a, b| a[:last_fetch] <=> b[:last_fetch] } # Oldest first
15
+
16
+ # Determine which files to evict
17
+ space_to_evict = current_cache_size - maximum_cache_size
18
+ space_evicted = 0
19
+ evictions = []
20
+ while space_evicted < space_to_evict
21
+ evicted_file = sorted_files.shift
22
+ space_evicted += evicted_file[:size]
23
+ evictions << evicted_file
24
+ end
25
+
26
+ evictions
27
+ end
28
+
29
+ def directories_to_evict
30
+ empty_directories
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,106 @@
1
+ Dir[File.join(File.dirname(__FILE__), "eviction_strategies", "*.rb")].each { |f| require f }
2
+
3
+ class DiskStore
4
+ class Reaper
5
+ DEFAULT_OPTS = {
6
+ cache_size: 1073741824, # 1 gigabyte
7
+ reaper_interval: 10, # seconds,
8
+ eviction_strategy: nil
9
+ }
10
+
11
+ @reapers = {}
12
+
13
+ # Spawn exactly 1 reaper for each cache path
14
+ def self.spawn_for(path, opts = {})
15
+ return @reapers[path] if @reapers.has_key?(path)
16
+
17
+ reaper = Reaper.new(path, opts)
18
+ reaper.spawn!
19
+
20
+ @reapers[path] = reaper
21
+ reaper
22
+ end
23
+
24
+ # Mostly useful for testing purposes
25
+ def self.kill_all!
26
+ @reapers.each { |path, reaper| reaper.thread.kill }
27
+ @reapers = {}
28
+ end
29
+
30
+ attr_reader :path, :thread
31
+
32
+ def initialize(path, opts = {})
33
+ @path = path
34
+ @options = DEFAULT_OPTS.merge(opts)
35
+ @thread = nil
36
+
37
+ set_eviction_strategy(@options[:eviction_strategy])
38
+ end
39
+
40
+ def set_eviction_strategy(strategy)
41
+ return if strategy.nil?
42
+ self.class.send :prepend, DiskStore::Reaper.const_get(strategy)
43
+ end
44
+
45
+ def spawn!
46
+ @thread = Thread.new do
47
+ loop do
48
+ perform_sweep! if needs_eviction?
49
+ wait_for_next
50
+ end
51
+ end
52
+ end
53
+
54
+ def alive?
55
+ @thread && @thread.alive?
56
+ end
57
+
58
+ def running?
59
+ @thread && !@thread.stop?
60
+ end
61
+
62
+ private
63
+
64
+ def perform_sweep!
65
+ # Evict and delete selected files
66
+ files_to_evict.each { |file| FileUtils.rm(file[:path]) }
67
+ directories_to_evict.each { |dir| Dir.rmdir(dir) }
68
+ end
69
+
70
+ def needs_eviction?
71
+ current_cache_size > maximum_cache_size
72
+ end
73
+
74
+ def files_to_evict
75
+ []
76
+ end
77
+
78
+ def directories_to_evict
79
+ []
80
+ end
81
+
82
+ def wait_for_next
83
+ sleep @options[:reaper_interval]
84
+ end
85
+
86
+ def files
87
+ Dir[File.join(path, "**", "*")].select { |f| File.file?(f) }
88
+ end
89
+
90
+ def directories
91
+ Dir[File.join(path, "**", "*")].select { |f| File.directory?(f) }
92
+ end
93
+
94
+ def empty_directories
95
+ directories.select { |d| Dir.entries(d).size == 2 }
96
+ end
97
+
98
+ def current_cache_size
99
+ files.map { |file| File.new(file).size }.inject { |sum, size| sum + size } || 0
100
+ end
101
+
102
+ def maximum_cache_size
103
+ @options[:cache_size].to_i
104
+ end
105
+ end
106
+ end
@@ -1,3 +1,3 @@
1
1
  class DiskStore
2
- VERSION = "0.1.2"
2
+ VERSION = "0.2.0"
3
3
  end
data/lib/disk_store.rb CHANGED
@@ -1,13 +1,18 @@
1
1
  require 'open-uri'
2
+ require 'disk_store/reaper'
2
3
 
3
4
  class DiskStore
4
5
  DIR_FORMATTER = "%03X"
5
6
  FILENAME_MAX_SIZE = 228 # max filename size on file system is 255, minus room for timestamp and random characters appended by Tempfile (used by atomic write)
6
7
  EXCLUDED_DIRS = ['.', '..'].freeze
7
8
 
8
- def initialize(path=nil)
9
+ attr_reader :reaper
10
+
11
+ def initialize(path=nil, opts = {})
9
12
  path ||= "."
10
13
  @root_path = File.expand_path path
14
+ @options = opts
15
+ @reaper = Reaper.spawn_for(@root_path, @options)
11
16
  end
12
17
 
13
18
  def read(key)
@@ -0,0 +1,129 @@
1
+ require 'spec_helper'
2
+
3
+ describe DiskStore::Reaper do
4
+ let(:file_contents) { "Hello, Doge" }
5
+ let(:file) { Tempfile.new("My Temp File.psd") }
6
+ let(:key) { "doge" }
7
+
8
+ before(:each) do
9
+ file.write file_contents
10
+ file.flush
11
+ file.rewind
12
+ end
13
+
14
+ around(:each) do |example|
15
+ Dir.mktmpdir do |tmpdir|
16
+ @tmpdir = tmpdir
17
+ example.run
18
+ end
19
+ end
20
+
21
+ after(:each) do
22
+ DiskStore::Reaper.kill_all!
23
+ end
24
+
25
+ it 'is spawned with a new cache' do
26
+ cache = DiskStore.new(@tmpdir)
27
+ expect(cache.reaper).to_not be_nil
28
+ expect(cache.reaper.path).to eq @tmpdir
29
+ expect(cache.reaper).to be_alive
30
+ end
31
+
32
+ it 'only spawns one reaper per cache dir' do
33
+ cache1 = DiskStore.new(@tmpdir)
34
+ cache2 = DiskStore.new(@tmpdir)
35
+
36
+ expect(cache1.reaper).to be cache2.reaper
37
+ end
38
+
39
+ it 'allows you to set the maximum cache size' do
40
+ cache = DiskStore.new(@tmpdir, cache_size: 2000)
41
+ expect(cache.reaper.send(:maximum_cache_size)).to eq 2000
42
+ end
43
+
44
+ describe 'cache files' do
45
+ let (:cache) { DiskStore.new(@tmpdir) }
46
+ let (:reaper) { cache.reaper }
47
+
48
+ before(:each) do
49
+ cache.write key, file
50
+ end
51
+
52
+ it "correctly calculates the cache size" do
53
+ expect(reaper.send(:current_cache_size)).to eq file_contents.size
54
+ end
55
+
56
+ it "finds all files" do
57
+ expect(reaper.send(:files).size).to eq 1
58
+ expect(File.read(reaper.send(:files).first)).to eq file_contents
59
+ end
60
+
61
+ it "finds all directories" do
62
+ expect(reaper.send(:directories).size).to eq 2
63
+ end
64
+ end
65
+
66
+ describe 'LRU eviction' do
67
+ context 'when below the cache size' do
68
+ let (:cache) { DiskStore.new(@tmpdir, eviction_strategy: :LRU, cache_size: 1073741824) }
69
+
70
+ before(:each) do
71
+ cache.write key, file
72
+ end
73
+
74
+ it "does not require eviction" do
75
+ expect(cache.reaper.send(:needs_eviction?)).to be_false
76
+ end
77
+ end
78
+
79
+ context 'when over the cache size' do
80
+ let(:file2_contents) { "Hi friend" }
81
+ let(:file2) { Tempfile.new("Friends are magical.psd") }
82
+ let(:key2) { "friend" }
83
+
84
+ let (:cache) { DiskStore.new(@tmpdir, eviction_strategy: :LRU, cache_size: file_contents.size + 1) }
85
+ let (:reaper) { cache.reaper }
86
+
87
+ before(:each) do
88
+ # Kill the thread so we can test without it getting
89
+ # in the way
90
+ reaper.thread.kill
91
+ cache.write key, file
92
+
93
+ file2.write file2_contents
94
+ file2.flush
95
+ file2.rewind
96
+
97
+ cache.write key2, file2
98
+ end
99
+
100
+ it "requires eviction" do
101
+ expect(reaper.send(:needs_eviction?)).to be_true
102
+ end
103
+
104
+ it "correctly determines the files to evict" do
105
+ files = reaper.send(:files_to_evict).map { |f| f[:path] }
106
+
107
+ expect(files.size).to eq 1
108
+ expect(File.read(files.first)).to eq file_contents
109
+ end
110
+
111
+ it "removes the file during eviction" do
112
+ reaper.send(:perform_sweep!)
113
+
114
+ expect(reaper.send(:files).size).to eq 1
115
+ expect(File.read(reaper.send(:files).first)).to eq file2_contents
116
+ end
117
+
118
+ it "correctly determines empty directories" do
119
+ reaper.send(:files).each { |f| FileUtils.rm(f) }
120
+ expect(reaper.send(:empty_directories).size).to eq 2
121
+ end
122
+
123
+ it "removes empty directories during eviction" do
124
+ 2.times { reaper.send(:perform_sweep!) }
125
+ expect(reaper.send(:empty_directories).size).to eq 0
126
+ end
127
+ end
128
+ end
129
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: disk_store
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kelly Sutton
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-15 00:00:00.000000000 Z
11
+ date: 2014-05-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -95,9 +95,12 @@ files:
95
95
  - Rakefile
96
96
  - disk_store.gemspec
97
97
  - lib/disk_store.rb
98
+ - lib/disk_store/eviction_strategies/lru.rb
99
+ - lib/disk_store/reaper.rb
98
100
  - lib/disk_store/version.rb
99
101
  - spec/cassettes/DiskStore/web_resources/should_cache_the_result_of_a_web_resource_in_a_file.json
100
102
  - spec/disk_store_spec.rb
103
+ - spec/reaper_spec.rb
101
104
  - spec/spec_helper.rb
102
105
  homepage: http://cosmos.layervault.com/cache.html
103
106
  licenses:
@@ -111,7 +114,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
111
114
  requirements:
112
115
  - - '>='
113
116
  - !ruby/object:Gem::Version
114
- version: '0'
117
+ version: 2.0.0
115
118
  required_rubygems_version: !ruby/object:Gem::Requirement
116
119
  requirements:
117
120
  - - '>='
@@ -119,11 +122,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
119
122
  version: '0'
120
123
  requirements: []
121
124
  rubyforge_project:
122
- rubygems_version: 2.1.9
125
+ rubygems_version: 2.2.2
123
126
  signing_key:
124
127
  specification_version: 4
125
128
  summary: Cache fiels the smart way.
126
129
  test_files:
127
130
  - spec/cassettes/DiskStore/web_resources/should_cache_the_result_of_a_web_resource_in_a_file.json
128
131
  - spec/disk_store_spec.rb
132
+ - spec/reaper_spec.rb
129
133
  - spec/spec_helper.rb