hammerspace 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.chef/cookbooks/hammerspace-development/attributes/default.rb +7 -0
- data/.chef/cookbooks/hammerspace-development/attributes/essential.rb +6 -0
- data/.chef/cookbooks/hammerspace-development/attributes/sparkey.rb +7 -0
- data/.chef/cookbooks/hammerspace-development/recipes/default.rb +32 -0
- data/.chef/cookbooks/hammerspace-development/recipes/essential.rb +9 -0
- data/.chef/cookbooks/hammerspace-development/recipes/ruby.rb +21 -0
- data/.chef/cookbooks/hammerspace-development/recipes/sparkey.rb +56 -0
- data/.chef/cookbooks/hammerspace-development/templates/default/.bash_profile.erb +2 -0
- data/.chef/roles/hammerspace-development.rb +6 -0
- data/.gitignore +8 -0
- data/CHANGELOG.md +22 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +522 -0
- data/Vagrantfile +30 -0
- data/hammerspace.gemspec +21 -0
- data/lib/hammerspace.rb +12 -0
- data/lib/hammerspace/backend.rb +106 -0
- data/lib/hammerspace/backend/sparkey.rb +319 -0
- data/lib/hammerspace/hash.rb +62 -0
- data/lib/hammerspace/hash_methods.rb +234 -0
- data/lib/hammerspace/version.rb +3 -0
- data/script/write_concurrency_test.rb +36 -0
- data/spec/features/hash_spec.rb +1487 -0
- data/spec/lib/hammerspace/backend/sparkey_spec.rb +191 -0
- data/spec/lib/hammerspace/hash_spec.rb +143 -0
- data/spec/lib/hammerspace_spec.rb +27 -0
- data/spec/spec_helper.rb +25 -0
- data/spec/support/sparkey_directory_helper.rb +26 -0
- data/spec/support/write_concurrency_test.rb +38 -0
- metadata +96 -0
@@ -0,0 +1,191 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Hammerspace::Backend::Sparkey do
|
4
|
+
|
5
|
+
let(:path) { HAMMERSPACE_ROOT }
|
6
|
+
let(:options) { {} }
|
7
|
+
|
8
|
+
before do
|
9
|
+
FileUtils.rm_rf(path, :secure => true)
|
10
|
+
end
|
11
|
+
|
12
|
+
after(:all) do
|
13
|
+
FileUtils.rm_rf(path, :secure => true)
|
14
|
+
end
|
15
|
+
|
16
|
+
it "creates path on set" do
|
17
|
+
hash = Hammerspace.new(path, options)
|
18
|
+
hash['foo'] = 'bar'
|
19
|
+
hash.close
|
20
|
+
|
21
|
+
Dir.exist?(path).should be_true
|
22
|
+
end
|
23
|
+
|
24
|
+
it "bulks writes" do
|
25
|
+
Gnista::Hash.should_receive(:write).once.and_call_original
|
26
|
+
|
27
|
+
hash = Hammerspace.new(path, options)
|
28
|
+
hash['foo'] = 'bar'
|
29
|
+
hash['foo'] = 'newvalue'
|
30
|
+
hash.close
|
31
|
+
end
|
32
|
+
|
33
|
+
it "handles high write concurrency and cleans up" do
|
34
|
+
run_write_concurrency_test(path, options)
|
35
|
+
|
36
|
+
# Also, at the end of the test, there should be one directory and one symlink.
|
37
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
38
|
+
SparkeyDirectoryHelper.has_current_symlink?(path).should be_true
|
39
|
+
SparkeyDirectoryHelper.has_unknown_files?(path).should be_false
|
40
|
+
end
|
41
|
+
|
42
|
+
describe "#check_fs" do
|
43
|
+
|
44
|
+
it "should call check methods" do
|
45
|
+
Hammerspace::Backend::Sparkey.any_instance.should_receive(:flock_works?).once.and_call_original
|
46
|
+
Hammerspace::Backend::Sparkey.any_instance.should_receive(:dir_cleanup_works?).once.and_call_original
|
47
|
+
|
48
|
+
Hammerspace.new(path, options)
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
describe "#flock_works?" do
|
54
|
+
|
55
|
+
it "should check flock and return true" do
|
56
|
+
Hammerspace.new(path, options).backend.flock_works?.should be_true
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
describe "#dir_cleanup_works?" do
|
62
|
+
|
63
|
+
it "should check directory cleanup and return true" do
|
64
|
+
Hammerspace.new(path, options).backend.dir_cleanup_works?.should be_true
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
describe "#clear" do
|
70
|
+
|
71
|
+
it "removes all keys and values and cleans up" do
|
72
|
+
hash = Hammerspace.new(path, options)
|
73
|
+
hash['foo'] = 'bar'
|
74
|
+
hash.close
|
75
|
+
|
76
|
+
hash = Hammerspace.new(path, options)
|
77
|
+
hash.clear
|
78
|
+
hash['foo'].should be_nil
|
79
|
+
hash.size.should == 0
|
80
|
+
hash.close
|
81
|
+
|
82
|
+
SparkeyDirectoryHelper.directory_count(path).should == 0
|
83
|
+
SparkeyDirectoryHelper.has_current_symlink?(path).should be_false
|
84
|
+
end
|
85
|
+
|
86
|
+
it "removes unflushed keys and values and cleans up" do
|
87
|
+
hash = Hammerspace.new(path, options)
|
88
|
+
hash['foo'] = 'bar'
|
89
|
+
hash.clear
|
90
|
+
hash['foo'].should be_nil
|
91
|
+
hash.size.should == 0
|
92
|
+
hash.close
|
93
|
+
|
94
|
+
SparkeyDirectoryHelper.directory_count(path).should == 0
|
95
|
+
SparkeyDirectoryHelper.has_current_symlink?(path).should be_false
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
|
100
|
+
describe "#close" do
|
101
|
+
|
102
|
+
it "removes empty directories" do
|
103
|
+
writer1 = Hammerspace.new(path, options)
|
104
|
+
writer1['foo'] = 'bar'
|
105
|
+
writer1.close
|
106
|
+
|
107
|
+
reader = Hammerspace.new(path, options)
|
108
|
+
reader['foo'].should == 'bar'
|
109
|
+
|
110
|
+
writer2 = Hammerspace.new(path, options)
|
111
|
+
writer2['foo'] = 'bar'
|
112
|
+
writer2.close
|
113
|
+
|
114
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
115
|
+
|
116
|
+
reader.close
|
117
|
+
|
118
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
|
123
|
+
describe "#each" do
|
124
|
+
|
125
|
+
it "removes empty directories after iteration with block" do
|
126
|
+
writer1 = Hammerspace.new(path, options)
|
127
|
+
writer1['foo'] = 'bar'
|
128
|
+
writer1.close
|
129
|
+
|
130
|
+
reader = Hammerspace.new(path, options)
|
131
|
+
reader.each do |key,value|
|
132
|
+
writer2 = Hammerspace.new(path, options)
|
133
|
+
writer2['foo'] = 'bar'
|
134
|
+
writer2.close
|
135
|
+
|
136
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
137
|
+
end
|
138
|
+
|
139
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
140
|
+
|
141
|
+
reader.close
|
142
|
+
end
|
143
|
+
|
144
|
+
it "removes empty directories after iteration with enumerator" do
|
145
|
+
writer1 = Hammerspace.new(path, options)
|
146
|
+
writer1['foo'] = 'bar'
|
147
|
+
writer1.close
|
148
|
+
|
149
|
+
reader = Hammerspace.new(path, options)
|
150
|
+
reader.each.map do |key,value|
|
151
|
+
writer2 = Hammerspace.new(path, options)
|
152
|
+
writer2['foo'] = 'bar'
|
153
|
+
writer2.close
|
154
|
+
|
155
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
156
|
+
end
|
157
|
+
|
158
|
+
SparkeyDirectoryHelper.directory_count(path).should == 1
|
159
|
+
|
160
|
+
reader.close
|
161
|
+
end
|
162
|
+
|
163
|
+
end
|
164
|
+
|
165
|
+
describe "#include?" do
|
166
|
+
|
167
|
+
it "calls has_key?" do
|
168
|
+
Gnista::Hash.any_instance.should_receive(:include?).once.and_call_original
|
169
|
+
|
170
|
+
hash = Hammerspace.new(path, options)
|
171
|
+
hash['foo'] = 'bar'
|
172
|
+
hash.include?('foo').should be_true
|
173
|
+
hash.close
|
174
|
+
end
|
175
|
+
|
176
|
+
end
|
177
|
+
|
178
|
+
describe "#member?" do
|
179
|
+
|
180
|
+
it "calls has_key?" do
|
181
|
+
Gnista::Hash.any_instance.should_receive(:include?).once.and_call_original
|
182
|
+
|
183
|
+
hash = Hammerspace.new(path, options)
|
184
|
+
hash['foo'] = 'bar'
|
185
|
+
hash.member?('foo').should be_true
|
186
|
+
hash.close
|
187
|
+
end
|
188
|
+
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Hammerspace::Hash do
|
4
|
+
|
5
|
+
let(:path) { HAMMERSPACE_ROOT }
|
6
|
+
let(:options) { {} }
|
7
|
+
|
8
|
+
before do
|
9
|
+
FileUtils.rm_rf(path, :secure => true)
|
10
|
+
end
|
11
|
+
|
12
|
+
after(:all) do
|
13
|
+
FileUtils.rm_rf(path, :secure => true)
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "#initialize" do
|
17
|
+
|
18
|
+
it "creates the backend" do
|
19
|
+
hash = Hammerspace::Hash.new(path, options)
|
20
|
+
hash.backend.should be_a_kind_of(Hammerspace::Backend::Base)
|
21
|
+
end
|
22
|
+
|
23
|
+
it "takes a third argument and sets default" do
|
24
|
+
hash = Hammerspace::Hash.new(path, options, 'default')
|
25
|
+
hash.default.should == 'default'
|
26
|
+
end
|
27
|
+
|
28
|
+
it "takes a block and sets default_proc" do
|
29
|
+
hash = Hammerspace::Hash.new(path, options) { |h,k| k }
|
30
|
+
hash.default_proc.should be_an_instance_of(Proc)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "raises ArgumentError if both third argument and block are passed" do
|
34
|
+
expect {
|
35
|
+
Hammerspace::Hash.new(path, options, 'default') { |h,k| k }
|
36
|
+
}.to raise_error(ArgumentError)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "raises ArgumentError if a fourth argument is passed" do
|
40
|
+
expect {
|
41
|
+
Hammerspace::Hash.new(path, options, 'default', 'bogus')
|
42
|
+
}.to raise_error(ArgumentError)
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
describe "#default=" do
|
48
|
+
|
49
|
+
it "sets default" do
|
50
|
+
hash = Hammerspace::Hash.new(path, options)
|
51
|
+
hash.default = 'bar'
|
52
|
+
hash.default.should == 'bar'
|
53
|
+
end
|
54
|
+
|
55
|
+
it "unsets default_proc" do
|
56
|
+
hash = Hammerspace::Hash.new(path, options)
|
57
|
+
hash.default_proc = lambda { |h,k| k }
|
58
|
+
hash.default = 'bar'
|
59
|
+
hash.default_proc.should be_nil
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
|
64
|
+
describe "#default_proc=" do
|
65
|
+
|
66
|
+
it "sets default_proc" do
|
67
|
+
p = lambda { |h,k| k }
|
68
|
+
hash = Hammerspace::Hash.new(path, options)
|
69
|
+
hash.default_proc = p
|
70
|
+
hash.default_proc.should == p
|
71
|
+
end
|
72
|
+
|
73
|
+
it "unsets default" do
|
74
|
+
hash = Hammerspace::Hash.new(path, options)
|
75
|
+
hash.default = 'bar'
|
76
|
+
hash.default_proc = p
|
77
|
+
hash.default('foo').should be_nil
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
|
82
|
+
describe "#default" do
|
83
|
+
|
84
|
+
context "with default set" do
|
85
|
+
|
86
|
+
context "with an argument" do
|
87
|
+
|
88
|
+
it "returns default value" do
|
89
|
+
hash = Hammerspace::Hash.new(path, options)
|
90
|
+
hash.default = 'bar'
|
91
|
+
hash.default('foo').should == 'bar'
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
|
96
|
+
context "without an argument" do
|
97
|
+
|
98
|
+
it "returns default value" do
|
99
|
+
hash = Hammerspace::Hash.new(path, options)
|
100
|
+
hash.default = 'bar'
|
101
|
+
hash.default('foo').should == 'bar'
|
102
|
+
end
|
103
|
+
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
|
108
|
+
context "with default_proc set" do
|
109
|
+
|
110
|
+
context "with an argument" do
|
111
|
+
|
112
|
+
it "evaluates proc" do
|
113
|
+
hash = Hammerspace::Hash.new(path, options) do |h,k|
|
114
|
+
h.should == hash
|
115
|
+
k.reverse
|
116
|
+
end
|
117
|
+
hash.default('foo').should == 'oof'
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
121
|
+
|
122
|
+
context "without an argument" do
|
123
|
+
|
124
|
+
it "returns nil" do
|
125
|
+
hash = Hammerspace::Hash.new(path, options) { |h,k| k }
|
126
|
+
hash.default.should be_nil
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
|
135
|
+
it "supports enumerable" do
|
136
|
+
hash = Hammerspace::Hash.new(path, options)
|
137
|
+
hash['a'] = 'A'
|
138
|
+
hash['b'] = 'B'
|
139
|
+
result = hash.map { |key,value| key + value }
|
140
|
+
result.should == ['aA', 'bB']
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Hammerspace do
|
4
|
+
|
5
|
+
let(:path) { HAMMERSPACE_ROOT }
|
6
|
+
let(:options) { {} }
|
7
|
+
|
8
|
+
describe "#initialize" do
|
9
|
+
|
10
|
+
it "returns a Hammerspace::Hash object" do
|
11
|
+
hash = Hammerspace.new(path, options)
|
12
|
+
hash.should be_an_instance_of(Hammerspace::Hash)
|
13
|
+
end
|
14
|
+
|
15
|
+
it "takes a third argument and sets default" do
|
16
|
+
hash = Hammerspace.new(path, options, 'default')
|
17
|
+
hash.default.should == 'default'
|
18
|
+
end
|
19
|
+
|
20
|
+
it "takes a block and sets default_proc" do
|
21
|
+
hash = Hammerspace::Hash.new(path, options) { |h,k| k }
|
22
|
+
hash.default_proc.should be_an_instance_of(Proc)
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'simplecov'
|
2
|
+
SimpleCov.start do
|
3
|
+
add_filter 'spec'
|
4
|
+
end
|
5
|
+
|
6
|
+
SimpleCov.use_merging false
|
7
|
+
|
8
|
+
# from https://gist.github.com/clicube/5017378
|
9
|
+
pid = Process.pid
|
10
|
+
SimpleCov.at_exit do
|
11
|
+
SimpleCov.result.format! if Process.pid == pid
|
12
|
+
end
|
13
|
+
|
14
|
+
require 'hammerspace'
|
15
|
+
|
16
|
+
require 'support/sparkey_directory_helper'
|
17
|
+
require 'support/write_concurrency_test'
|
18
|
+
|
19
|
+
RSpec.configure do |config|
|
20
|
+
config.color_enabled = true
|
21
|
+
config.include(WriteConcurrencyTest)
|
22
|
+
end
|
23
|
+
|
24
|
+
HAMMERSPACE_ROOT = ENV['HAMMERSPACE_ROOT'] || 'tmp'
|
25
|
+
warn "Temporary hammerspace files will be written to #{HAMMERSPACE_ROOT}"
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module SparkeyDirectoryHelper
|
2
|
+
|
3
|
+
def self.directory_count(path)
|
4
|
+
dirs = 0
|
5
|
+
Dir.glob(File.join(path, '*')) do |f|
|
6
|
+
dirs += 1 if File.directory?(f) && !File.symlink?(f)
|
7
|
+
end
|
8
|
+
dirs
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.has_current_symlink?(path)
|
12
|
+
File.symlink?(File.join(path, 'current'))
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.has_unknown_files?(path)
|
16
|
+
unknown_files = false
|
17
|
+
Dir.glob(File.join(path, '*')) do |file|
|
18
|
+
next if File.directory?(file)
|
19
|
+
next if File.basename(file) == 'current' && File.symlink?(file)
|
20
|
+
next if File.basename(file) == 'hammerspace.lock' && File.file?(file)
|
21
|
+
unknown_files = true
|
22
|
+
end
|
23
|
+
unknown_files
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module WriteConcurrencyTest
|
2
|
+
|
3
|
+
# Initialize n hashes (of joyful nonsense), fork one process for each. Have
|
4
|
+
# them madly write their hash, and flush (repeat many, many times). While
|
5
|
+
# this is happening, read from the hammerspace. It should contain one of the
|
6
|
+
# n original hashes. Though which one I shall never tell.
|
7
|
+
def run_write_concurrency_test(path, options, concurrency = 10, iterations = 10, size = 10)
|
8
|
+
pids = []
|
9
|
+
|
10
|
+
concurrency.times do |id|
|
11
|
+
pids << fork do
|
12
|
+
iterations.times do
|
13
|
+
hash = Hammerspace.new(path, options)
|
14
|
+
size.times { |i| hash[i.to_s] = id.to_s }
|
15
|
+
hash.close
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# Wait for first hash to be written, otherwise our hash.size expectations will fail.
|
21
|
+
sleep(0.5)
|
22
|
+
|
23
|
+
iterations.times do
|
24
|
+
hash = Hammerspace.new(path, options)
|
25
|
+
hash_size = hash.size
|
26
|
+
raise "hash.size == #{hash_size}, expected #{size}" unless hash_size == size
|
27
|
+
size.times do |i|
|
28
|
+
unless hash[i.to_s] == hash['0']
|
29
|
+
raise "hash[#{i.to_s}] == #{hash[i.to_s]}, expected #{hash['0']}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
hash.close
|
33
|
+
end
|
34
|
+
|
35
|
+
pids.each { |pid| Process.wait(pid) }
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|