disk_store 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +0 -1
- data/Gemfile.lock +2 -2
- data/README.md +25 -2
- data/disk_store.gemspec +1 -0
- data/lib/disk_store/eviction_strategies/lru.rb +34 -0
- data/lib/disk_store/reaper.rb +106 -0
- data/lib/disk_store/version.rb +1 -1
- data/lib/disk_store.rb +6 -1
- data/spec/reaper_spec.rb +129 -0
- metadata +8 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7de985eb8736731f07a776e31aba1a3847cf34d6
|
4
|
+
data.tar.gz: 121f7f38bbd85a30a3250fd99edd7e1c122a9a56
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 44ec9112fdb3bcc1edf9c13fde166237acfa32c73d1e5c9d220e3b108ebdee6cf147d063e0abb59c59c3cb80021d08d4b396932f47be7110499d915bc7a34999
|
7
|
+
data.tar.gz: 81b754f6d6fe78c746c7c655c65623376153f799b7aa9452e38b10a21eb6df98f454bac02f6d8a0af5bf2b364283973d1b930a1fc673b13705aff6fe0b25c3f0
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
disk_store (0.
|
4
|
+
disk_store (0.2.0)
|
5
5
|
rake
|
6
6
|
|
7
7
|
GEM
|
@@ -12,7 +12,7 @@ GEM
|
|
12
12
|
safe_yaml (~> 0.9.0)
|
13
13
|
diff-lcs (1.2.5)
|
14
14
|
multi_json (1.8.4)
|
15
|
-
rake (10.
|
15
|
+
rake (10.3.1)
|
16
16
|
rspec (2.14.1)
|
17
17
|
rspec-core (~> 2.14.0)
|
18
18
|
rspec-expectations (~> 2.14.0)
|
data/README.md
CHANGED
@@ -24,13 +24,36 @@ examples.
|
|
24
24
|
|
25
25
|
DiskStore requires a directory to to store the files.
|
26
26
|
|
27
|
-
It
|
28
|
-
|
27
|
+
It has two parameters:
|
28
|
+
|
29
|
+
* The first is the path to the cache directory. If no directory is given, then the current directory is used.
|
30
|
+
* The second is an options hash that controls the Reaper. More info below.
|
29
31
|
|
30
32
|
```ruby
|
31
33
|
cache = DiskStore.new("path/to/my/cache/directory")
|
32
34
|
```
|
33
35
|
|
36
|
+
#### Reaper
|
37
|
+
|
38
|
+
By default, DiskStore does not perform any evictions on files in the cache. Be careful with this, because
|
39
|
+
it can cause your disk to fill up if left to its own means.
|
40
|
+
|
41
|
+
The reaper configuration includes:
|
42
|
+
|
43
|
+
* `cache_size`: The maximum size of the cache before the reaper begins to evict files (in bytes).
|
44
|
+
* `reaper_interval`: How often the reaper will check for files to evict (in seconds).
|
45
|
+
* `eviction_strategy`: Sets how the reaper will determine which files to evict.
|
46
|
+
|
47
|
+
Current available eviction strategies are:
|
48
|
+
|
49
|
+
* LRU (least recently used) - deletes the files with the oldest last access time
|
50
|
+
|
51
|
+
To configure DiskStore with an LRU eviction strategy:
|
52
|
+
|
53
|
+
``` ruby
|
54
|
+
cache = DiskStore.new('cache/dir', eviction_strategy: :LRU)
|
55
|
+
```
|
56
|
+
|
34
57
|
### Reading
|
35
58
|
|
36
59
|
```ruby
|
data/disk_store.gemspec
CHANGED
@@ -12,6 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.summary = %q{Cache fiels the smart way.}
|
13
13
|
gem.homepage = "http://cosmos.layervault.com/cache.html"
|
14
14
|
gem.license = 'MIT'
|
15
|
+
gem.required_ruby_version = ">= 2.0.0"
|
15
16
|
|
16
17
|
gem.files = `git ls-files`.split($/).delete_if { |f| f.include?('examples/') }
|
17
18
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class DiskStore
|
2
|
+
class Reaper
|
3
|
+
module LRU
|
4
|
+
def files_to_evict
|
5
|
+
# Collect and sort files based on last access time
|
6
|
+
sorted_files = files
|
7
|
+
.map { |file|
|
8
|
+
data = nil
|
9
|
+
File.new(file, 'rb').tap { |fd|
|
10
|
+
data = { path: file, last_fetch: fd.atime, size: fd.size }
|
11
|
+
}.close
|
12
|
+
data
|
13
|
+
}
|
14
|
+
.sort { |a, b| a[:last_fetch] <=> b[:last_fetch] } # Oldest first
|
15
|
+
|
16
|
+
# Determine which files to evict
|
17
|
+
space_to_evict = current_cache_size - maximum_cache_size
|
18
|
+
space_evicted = 0
|
19
|
+
evictions = []
|
20
|
+
while space_evicted < space_to_evict
|
21
|
+
evicted_file = sorted_files.shift
|
22
|
+
space_evicted += evicted_file[:size]
|
23
|
+
evictions << evicted_file
|
24
|
+
end
|
25
|
+
|
26
|
+
evictions
|
27
|
+
end
|
28
|
+
|
29
|
+
def directories_to_evict
|
30
|
+
empty_directories
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
Dir[File.join(File.dirname(__FILE__), "eviction_strategies", "*.rb")].each { |f| require f }
|
2
|
+
|
3
|
+
class DiskStore
|
4
|
+
class Reaper
|
5
|
+
DEFAULT_OPTS = {
|
6
|
+
cache_size: 1073741824, # 1 gigabyte
|
7
|
+
reaper_interval: 10, # seconds,
|
8
|
+
eviction_strategy: nil
|
9
|
+
}
|
10
|
+
|
11
|
+
@reapers = {}
|
12
|
+
|
13
|
+
# Spawn exactly 1 reaper for each cache path
|
14
|
+
def self.spawn_for(path, opts = {})
|
15
|
+
return @reapers[path] if @reapers.has_key?(path)
|
16
|
+
|
17
|
+
reaper = Reaper.new(path, opts)
|
18
|
+
reaper.spawn!
|
19
|
+
|
20
|
+
@reapers[path] = reaper
|
21
|
+
reaper
|
22
|
+
end
|
23
|
+
|
24
|
+
# Mostly useful for testing purposes
|
25
|
+
def self.kill_all!
|
26
|
+
@reapers.each { |path, reaper| reaper.thread.kill }
|
27
|
+
@reapers = {}
|
28
|
+
end
|
29
|
+
|
30
|
+
attr_reader :path, :thread
|
31
|
+
|
32
|
+
def initialize(path, opts = {})
|
33
|
+
@path = path
|
34
|
+
@options = DEFAULT_OPTS.merge(opts)
|
35
|
+
@thread = nil
|
36
|
+
|
37
|
+
set_eviction_strategy(@options[:eviction_strategy])
|
38
|
+
end
|
39
|
+
|
40
|
+
def set_eviction_strategy(strategy)
|
41
|
+
return if strategy.nil?
|
42
|
+
self.class.send :prepend, DiskStore::Reaper.const_get(strategy)
|
43
|
+
end
|
44
|
+
|
45
|
+
def spawn!
|
46
|
+
@thread = Thread.new do
|
47
|
+
loop do
|
48
|
+
perform_sweep! if needs_eviction?
|
49
|
+
wait_for_next
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def alive?
|
55
|
+
@thread && @thread.alive?
|
56
|
+
end
|
57
|
+
|
58
|
+
def running?
|
59
|
+
@thread && !@thread.stop?
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def perform_sweep!
|
65
|
+
# Evict and delete selected files
|
66
|
+
files_to_evict.each { |file| FileUtils.rm(file[:path]) }
|
67
|
+
directories_to_evict.each { |dir| Dir.rmdir(dir) }
|
68
|
+
end
|
69
|
+
|
70
|
+
def needs_eviction?
|
71
|
+
current_cache_size > maximum_cache_size
|
72
|
+
end
|
73
|
+
|
74
|
+
def files_to_evict
|
75
|
+
[]
|
76
|
+
end
|
77
|
+
|
78
|
+
def directories_to_evict
|
79
|
+
[]
|
80
|
+
end
|
81
|
+
|
82
|
+
def wait_for_next
|
83
|
+
sleep @options[:reaper_interval]
|
84
|
+
end
|
85
|
+
|
86
|
+
def files
|
87
|
+
Dir[File.join(path, "**", "*")].select { |f| File.file?(f) }
|
88
|
+
end
|
89
|
+
|
90
|
+
def directories
|
91
|
+
Dir[File.join(path, "**", "*")].select { |f| File.directory?(f) }
|
92
|
+
end
|
93
|
+
|
94
|
+
def empty_directories
|
95
|
+
directories.select { |d| Dir.entries(d).size == 2 }
|
96
|
+
end
|
97
|
+
|
98
|
+
def current_cache_size
|
99
|
+
files.map { |file| File.new(file).size }.inject { |sum, size| sum + size } || 0
|
100
|
+
end
|
101
|
+
|
102
|
+
def maximum_cache_size
|
103
|
+
@options[:cache_size].to_i
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
data/lib/disk_store/version.rb
CHANGED
data/lib/disk_store.rb
CHANGED
@@ -1,13 +1,18 @@
|
|
1
1
|
require 'open-uri'
|
2
|
+
require 'disk_store/reaper'
|
2
3
|
|
3
4
|
class DiskStore
|
4
5
|
DIR_FORMATTER = "%03X"
|
5
6
|
FILENAME_MAX_SIZE = 228 # max filename size on file system is 255, minus room for timestamp and random characters appended by Tempfile (used by atomic write)
|
6
7
|
EXCLUDED_DIRS = ['.', '..'].freeze
|
7
8
|
|
8
|
-
|
9
|
+
attr_reader :reaper
|
10
|
+
|
11
|
+
def initialize(path=nil, opts = {})
|
9
12
|
path ||= "."
|
10
13
|
@root_path = File.expand_path path
|
14
|
+
@options = opts
|
15
|
+
@reaper = Reaper.spawn_for(@root_path, @options)
|
11
16
|
end
|
12
17
|
|
13
18
|
def read(key)
|
data/spec/reaper_spec.rb
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe DiskStore::Reaper do
|
4
|
+
let(:file_contents) { "Hello, Doge" }
|
5
|
+
let(:file) { Tempfile.new("My Temp File.psd") }
|
6
|
+
let(:key) { "doge" }
|
7
|
+
|
8
|
+
before(:each) do
|
9
|
+
file.write file_contents
|
10
|
+
file.flush
|
11
|
+
file.rewind
|
12
|
+
end
|
13
|
+
|
14
|
+
around(:each) do |example|
|
15
|
+
Dir.mktmpdir do |tmpdir|
|
16
|
+
@tmpdir = tmpdir
|
17
|
+
example.run
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
after(:each) do
|
22
|
+
DiskStore::Reaper.kill_all!
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'is spawned with a new cache' do
|
26
|
+
cache = DiskStore.new(@tmpdir)
|
27
|
+
expect(cache.reaper).to_not be_nil
|
28
|
+
expect(cache.reaper.path).to eq @tmpdir
|
29
|
+
expect(cache.reaper).to be_alive
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'only spawns one reaper per cache dir' do
|
33
|
+
cache1 = DiskStore.new(@tmpdir)
|
34
|
+
cache2 = DiskStore.new(@tmpdir)
|
35
|
+
|
36
|
+
expect(cache1.reaper).to be cache2.reaper
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'allows you to set the maximum cache size' do
|
40
|
+
cache = DiskStore.new(@tmpdir, cache_size: 2000)
|
41
|
+
expect(cache.reaper.send(:maximum_cache_size)).to eq 2000
|
42
|
+
end
|
43
|
+
|
44
|
+
describe 'cache files' do
|
45
|
+
let (:cache) { DiskStore.new(@tmpdir) }
|
46
|
+
let (:reaper) { cache.reaper }
|
47
|
+
|
48
|
+
before(:each) do
|
49
|
+
cache.write key, file
|
50
|
+
end
|
51
|
+
|
52
|
+
it "correctly calculates the cache size" do
|
53
|
+
expect(reaper.send(:current_cache_size)).to eq file_contents.size
|
54
|
+
end
|
55
|
+
|
56
|
+
it "finds all files" do
|
57
|
+
expect(reaper.send(:files).size).to eq 1
|
58
|
+
expect(File.read(reaper.send(:files).first)).to eq file_contents
|
59
|
+
end
|
60
|
+
|
61
|
+
it "finds all directories" do
|
62
|
+
expect(reaper.send(:directories).size).to eq 2
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe 'LRU eviction' do
|
67
|
+
context 'when below the cache size' do
|
68
|
+
let (:cache) { DiskStore.new(@tmpdir, eviction_strategy: :LRU, cache_size: 1073741824) }
|
69
|
+
|
70
|
+
before(:each) do
|
71
|
+
cache.write key, file
|
72
|
+
end
|
73
|
+
|
74
|
+
it "does not require eviction" do
|
75
|
+
expect(cache.reaper.send(:needs_eviction?)).to be_false
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
context 'when over the cache size' do
|
80
|
+
let(:file2_contents) { "Hi friend" }
|
81
|
+
let(:file2) { Tempfile.new("Friends are magical.psd") }
|
82
|
+
let(:key2) { "friend" }
|
83
|
+
|
84
|
+
let (:cache) { DiskStore.new(@tmpdir, eviction_strategy: :LRU, cache_size: file_contents.size + 1) }
|
85
|
+
let (:reaper) { cache.reaper }
|
86
|
+
|
87
|
+
before(:each) do
|
88
|
+
# Kill the thread so we can test without it getting
|
89
|
+
# in the way
|
90
|
+
reaper.thread.kill
|
91
|
+
cache.write key, file
|
92
|
+
|
93
|
+
file2.write file2_contents
|
94
|
+
file2.flush
|
95
|
+
file2.rewind
|
96
|
+
|
97
|
+
cache.write key2, file2
|
98
|
+
end
|
99
|
+
|
100
|
+
it "requires eviction" do
|
101
|
+
expect(reaper.send(:needs_eviction?)).to be_true
|
102
|
+
end
|
103
|
+
|
104
|
+
it "correctly determines the files to evict" do
|
105
|
+
files = reaper.send(:files_to_evict).map { |f| f[:path] }
|
106
|
+
|
107
|
+
expect(files.size).to eq 1
|
108
|
+
expect(File.read(files.first)).to eq file_contents
|
109
|
+
end
|
110
|
+
|
111
|
+
it "removes the file during eviction" do
|
112
|
+
reaper.send(:perform_sweep!)
|
113
|
+
|
114
|
+
expect(reaper.send(:files).size).to eq 1
|
115
|
+
expect(File.read(reaper.send(:files).first)).to eq file2_contents
|
116
|
+
end
|
117
|
+
|
118
|
+
it "correctly determines empty directories" do
|
119
|
+
reaper.send(:files).each { |f| FileUtils.rm(f) }
|
120
|
+
expect(reaper.send(:empty_directories).size).to eq 2
|
121
|
+
end
|
122
|
+
|
123
|
+
it "removes empty directories during eviction" do
|
124
|
+
2.times { reaper.send(:perform_sweep!) }
|
125
|
+
expect(reaper.send(:empty_directories).size).to eq 0
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: disk_store
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kelly Sutton
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -95,9 +95,12 @@ files:
|
|
95
95
|
- Rakefile
|
96
96
|
- disk_store.gemspec
|
97
97
|
- lib/disk_store.rb
|
98
|
+
- lib/disk_store/eviction_strategies/lru.rb
|
99
|
+
- lib/disk_store/reaper.rb
|
98
100
|
- lib/disk_store/version.rb
|
99
101
|
- spec/cassettes/DiskStore/web_resources/should_cache_the_result_of_a_web_resource_in_a_file.json
|
100
102
|
- spec/disk_store_spec.rb
|
103
|
+
- spec/reaper_spec.rb
|
101
104
|
- spec/spec_helper.rb
|
102
105
|
homepage: http://cosmos.layervault.com/cache.html
|
103
106
|
licenses:
|
@@ -111,7 +114,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
111
114
|
requirements:
|
112
115
|
- - '>='
|
113
116
|
- !ruby/object:Gem::Version
|
114
|
-
version:
|
117
|
+
version: 2.0.0
|
115
118
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
116
119
|
requirements:
|
117
120
|
- - '>='
|
@@ -119,11 +122,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
119
122
|
version: '0'
|
120
123
|
requirements: []
|
121
124
|
rubyforge_project:
|
122
|
-
rubygems_version: 2.
|
125
|
+
rubygems_version: 2.2.2
|
123
126
|
signing_key:
|
124
127
|
specification_version: 4
|
125
128
|
summary: Cache fiels the smart way.
|
126
129
|
test_files:
|
127
130
|
- spec/cassettes/DiskStore/web_resources/should_cache_the_result_of_a_web_resource_in_a_file.json
|
128
131
|
- spec/disk_store_spec.rb
|
132
|
+
- spec/reaper_spec.rb
|
129
133
|
- spec/spec_helper.rb
|