hammerspace-fork 0.1.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.chef/cookbooks/hammerspace-development/attributes/default.rb +7 -0
- data/.chef/cookbooks/hammerspace-development/attributes/essential.rb +6 -0
- data/.chef/cookbooks/hammerspace-development/attributes/sparkey.rb +7 -0
- data/.chef/cookbooks/hammerspace-development/recipes/default.rb +32 -0
- data/.chef/cookbooks/hammerspace-development/recipes/essential.rb +9 -0
- data/.chef/cookbooks/hammerspace-development/recipes/ruby.rb +21 -0
- data/.chef/cookbooks/hammerspace-development/recipes/sparkey.rb +56 -0
- data/.chef/cookbooks/hammerspace-development/templates/default/.bash_profile.erb +2 -0
- data/.chef/roles/hammerspace-development.rb +6 -0
- data/.gitignore +8 -0
- data/CHANGELOG.md +30 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +523 -0
- data/Vagrantfile +30 -0
- data/hammerspace-fork.gemspec +21 -0
- data/lib/hammerspace.rb +12 -0
- data/lib/hammerspace/backend.rb +106 -0
- data/lib/hammerspace/backend/sparkey.rb +319 -0
- data/lib/hammerspace/hash.rb +62 -0
- data/lib/hammerspace/hash_methods.rb +234 -0
- data/lib/hammerspace/version.rb +3 -0
- data/script/write_concurrency_test.rb +36 -0
- data/spec/features/hash_spec.rb +1487 -0
- data/spec/lib/hammerspace/backend/sparkey_spec.rb +191 -0
- data/spec/lib/hammerspace/hash_spec.rb +143 -0
- data/spec/lib/hammerspace_spec.rb +27 -0
- data/spec/spec_helper.rb +25 -0
- data/spec/support/sparkey_directory_helper.rb +26 -0
- data/spec/support/write_concurrency_test.rb +38 -0
- metadata +96 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5a9b6b90f1ae4489a0e5b2a7673a5717b455f80e
|
4
|
+
data.tar.gz: efa0776dff153dff8f4adffbb3f1123fbc8b1d1f
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 363aca651c54fa6de62c6e7e3b6db799510b34472a35ebd2cde4a98864b0d4dfa12c1777dc871e6c3151aa9066bebcfcbadb96d6ebcb26342ebaded59a5b73a6
|
7
|
+
data.tar.gz: 504af3c3e04f05b3b4507f41a588d88a5c8a918bf789eca2601f946542fe64f177776703b3de0f71ffcaa04b2ba8dd5e12f17c3d90444a6a2360be2f178b95c5
|
@@ -0,0 +1,7 @@
|
|
1
|
+
default.hammerspace_development.user = 'vagrant'
|
2
|
+
default.hammerspace_development.hammerspace.home = '/home/vagrant/hammerspace'
|
3
|
+
default.hammerspace_development.hammerspace.gem_home = '/home/vagrant/.gems'
|
4
|
+
default.hammerspace_development.hammerspace.root = '/var/lib/hammerspace'
|
5
|
+
|
6
|
+
default.hammerspace_development.ruby.version = '1.9.1'
|
7
|
+
default.hammerspace_development.sparkey.version = 'daa9941221584d89da68803303854ef7e3a8f68d'
|
@@ -0,0 +1,7 @@
|
|
1
|
+
default.hammerspace_development.sparkey.home = "/home/#{default.hammerspace_development.user}/sparkey"
|
2
|
+
default.hammerspace_development.sparkey.source_file = "https://github.com/spotify/sparkey/archive/#{node.hammerspace_development.sparkey.version}.tar.gz"
|
3
|
+
default.hammerspace_development.sparkey.local_dir = File.join(default.hammerspace_development.sparkey.home, "sparkey-#{node.hammerspace_development.sparkey.version}")
|
4
|
+
|
5
|
+
default.hammerspace_development.sparkey.packages = [
|
6
|
+
'libsnappy-dev',
|
7
|
+
]
|
@@ -0,0 +1,32 @@
|
|
1
|
+
include_recipe "hammerspace-development::essential"
|
2
|
+
include_recipe "hammerspace-development::sparkey"
|
3
|
+
include_recipe "hammerspace-development::ruby"
|
4
|
+
|
5
|
+
template "/home/#{node.hammerspace_development.user}/.bash_profile" do
|
6
|
+
owner node.hammerspace_development.user
|
7
|
+
group node.hammerspace_development.user
|
8
|
+
mode '0755'
|
9
|
+
end
|
10
|
+
|
11
|
+
directory node.hammerspace_development.hammerspace.gem_home do
|
12
|
+
owner node.hammerspace_development.user
|
13
|
+
group node.hammerspace_development.user
|
14
|
+
mode '0755'
|
15
|
+
recursive true
|
16
|
+
action :create
|
17
|
+
end
|
18
|
+
|
19
|
+
execute "hammerspace-bundle-install" do
|
20
|
+
cwd node.hammerspace_development.hammerspace.home
|
21
|
+
user node.hammerspace_development.user
|
22
|
+
group node.hammerspace_development.user
|
23
|
+
command "bundle install --path #{node.hammerspace_development.hammerspace.gem_home}"
|
24
|
+
end
|
25
|
+
|
26
|
+
directory node.hammerspace_development.hammerspace.root do
|
27
|
+
owner node.hammerspace_development.user
|
28
|
+
group node.hammerspace_development.user
|
29
|
+
mode '0755'
|
30
|
+
recursive true
|
31
|
+
action :create
|
32
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
package "ruby#{node.hammerspace_development.ruby.version}" do
|
2
|
+
action :upgrade
|
3
|
+
options "--force-yes"
|
4
|
+
end
|
5
|
+
|
6
|
+
package "ruby#{node.hammerspace_development.ruby.version}-dev" do
|
7
|
+
action :upgrade
|
8
|
+
options "--force-yes"
|
9
|
+
end
|
10
|
+
|
11
|
+
# other common packages needed by ruby gems
|
12
|
+
["libxslt-dev", "libxml2-dev"].each do |p|
|
13
|
+
package p do
|
14
|
+
action :upgrade
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
gem_package 'bundler' do
|
19
|
+
action :install
|
20
|
+
end
|
21
|
+
|
@@ -0,0 +1,56 @@
|
|
1
|
+
sparkey_local_file = File.join(
|
2
|
+
node.hammerspace_development.sparkey.home,
|
3
|
+
File.basename(node.hammerspace_development.sparkey.source_file))
|
4
|
+
|
5
|
+
node.hammerspace_development.sparkey.packages.each do |p|
|
6
|
+
package p do
|
7
|
+
action :upgrade
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
directory node.hammerspace_development.sparkey.home do
|
12
|
+
owner node.hammerspace_development.user
|
13
|
+
group node.hammerspace_development.user
|
14
|
+
mode '0755'
|
15
|
+
recursive true
|
16
|
+
action :create
|
17
|
+
end
|
18
|
+
|
19
|
+
remote_file sparkey_local_file do
|
20
|
+
source node.hammerspace_development.sparkey.source_file
|
21
|
+
owner node.hammerspace_development.user
|
22
|
+
group node.hammerspace_development.user
|
23
|
+
mode "644"
|
24
|
+
|
25
|
+
action :create_if_missing
|
26
|
+
notifies :run, "execute[extract-sparkey-#{node.hammerspace_development.sparkey.version}]", :immediately
|
27
|
+
end
|
28
|
+
|
29
|
+
execute "extract-sparkey-#{node.hammerspace_development.sparkey.version}" do
|
30
|
+
cwd node.hammerspace_development.sparkey.home
|
31
|
+
user node.hammerspace_development.user
|
32
|
+
command "tar -xvzf #{sparkey_local_file}"
|
33
|
+
|
34
|
+
action :nothing
|
35
|
+
notifies :run, "bash[build-sparkey-#{node.hammerspace_development.sparkey.version}]", :immediately
|
36
|
+
end
|
37
|
+
|
38
|
+
bash "build-sparkey-#{node.hammerspace_development.sparkey.version}" do
|
39
|
+
cwd node.hammerspace_development.sparkey.local_dir
|
40
|
+
user node.hammerspace_development.user
|
41
|
+
code <<-EOS
|
42
|
+
autoreconf --install
|
43
|
+
./configure
|
44
|
+
make
|
45
|
+
EOS
|
46
|
+
|
47
|
+
action :nothing
|
48
|
+
notifies :run, "execute[install-sparkey-#{node.hammerspace_development.sparkey.version}]", :immediately
|
49
|
+
end
|
50
|
+
|
51
|
+
execute "install-sparkey-#{node.hammerspace_development.sparkey.version}" do
|
52
|
+
cwd node.hammerspace_development.sparkey.local_dir
|
53
|
+
command "make install && sudo ldconfig"
|
54
|
+
|
55
|
+
action :nothing
|
56
|
+
end
|
data/.gitignore
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# v0.1.4
|
2
|
+
* Upgrade to gnista 0.0.5.
|
3
|
+
* Remove work around for gnista bug.
|
4
|
+
|
5
|
+
# v0.1.3
|
6
|
+
* Work around gnista bug that causes ruby crashes on OS X.
|
7
|
+
* Upgrade to sparkey 0.2.0 in vagrant.
|
8
|
+
|
9
|
+
# v0.1.2
|
10
|
+
* Support vagrant for local development.
|
11
|
+
* Remove dependency on colored gem.
|
12
|
+
* Add MIT license.
|
13
|
+
* Documentation updates.
|
14
|
+
|
15
|
+
# v0.1.1
|
16
|
+
* Expose the uid of the directory that the current reader is reading from.
|
17
|
+
* Documentation updates.
|
18
|
+
|
19
|
+
# v0.1.0
|
20
|
+
* Change semantics of block passed to constructor, now used to specify default_proc.
|
21
|
+
* Add support for most Ruby Hash methods.
|
22
|
+
* Major internal refactor, new HashMethods module allows new backends to be written more easily.
|
23
|
+
* Add documentation.
|
24
|
+
|
25
|
+
# v0.0.2
|
26
|
+
* Add support for multiple writers with last-write-wins semantics.
|
27
|
+
* Implement `clear` method.
|
28
|
+
|
29
|
+
# v0.0.1
|
30
|
+
* Initial release.
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Airbnb, Inc.
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,523 @@
|
|
1
|
+
Hammerspace
|
2
|
+
===========
|
3
|
+
|
4
|
+
Hash-like interface to persistent, concurrent, off-heap storage
|
5
|
+
|
6
|
+
**Notice:** This is a forked release of hammerspace, which has been released to RubyGems as `hammerspace-fork`. It is [upstream hammerspace](https://github.com/airbnb/hammerspace) plus [my PR to correct the gnista dependency](https://github.com/airbnb/hammerspace/pull/8)
|
7
|
+
|
8
|
+
## What is Hammerspace?
|
9
|
+
|
10
|
+
_[Hammerspace](http://en.wikipedia.org/wiki/Hammerspace) ... is a
|
11
|
+
fan-envisioned extradimensional, instantly accessible storage area in fiction,
|
12
|
+
which is used to explain how animated, comic, and game characters can produce
|
13
|
+
objects out of thin air._
|
14
|
+
|
15
|
+
This gem provides persistent, concurrently-accessible off-heap storage of
|
16
|
+
strings with a familiar hash-like interface. It is optimized for bulk writes
|
17
|
+
and random reads.
|
18
|
+
|
19
|
+
|
20
|
+
## Motivation
|
21
|
+
|
22
|
+
Applications often use data that never changes or changes very infrequently. In
|
23
|
+
many cases, some latency is acceptable when accessing this data. For example, a
|
24
|
+
user's profile may be loaded from a web service, a database, or an external
|
25
|
+
shared cache like memcache. In other cases, latency is much more sensitive. For
|
26
|
+
example, translations may be used many times and incurring even a ~2ms delay to
|
27
|
+
access them from an external cache would be prohibitively slow.
|
28
|
+
|
29
|
+
To work around the performance issue, this type of data is often loaded into
|
30
|
+
the application at startup. Unfortunately, this means the data is stored on the
|
31
|
+
heap, where the garbage collector must scan over the objects on every run (at
|
32
|
+
least in the case of Ruby MRI). Further, for application servers that utilize
|
33
|
+
multiple processes, each process has its own copy of the data which is an
|
34
|
+
inefficient use of memory.
|
35
|
+
|
36
|
+
Hammerspace solves these problems by moving the data off the heap onto disk.
|
37
|
+
Leveraging libraries and data structures optimized for bulk writes and random
|
38
|
+
reads allows an acceptable level of performance to be maintained. Because the
|
39
|
+
data is persistent, it does not need to be reloaded from an external cache or
|
40
|
+
service on application startup unless the data has changed.
|
41
|
+
|
42
|
+
Unfortunately, these low-level libraries don't always support concurrent
|
43
|
+
writers. Hammerspace adds concurrency control to allow multiple processes to
|
44
|
+
update and read from a single shared copy of the data safely. Finally,
|
45
|
+
hammerspace's interface is designed to mimic Ruby's `Hash` to make integrating
|
46
|
+
with existing applications simple and straightforward. Different low-level
|
47
|
+
libraries can be used by implementing a new backend that uses the library.
|
48
|
+
(Currently, only [Sparkey](https://github.com/spotify/sparkey) is supported.)
|
49
|
+
Backends only need to implement a small set of methods (`[]`, `[]=`, `close`,
|
50
|
+
`delete`, `each`, `uid`), but can override the default implementation of other
|
51
|
+
methods if the underlying library supports more efficient implementations.
|
52
|
+
|
53
|
+
## Installation
|
54
|
+
|
55
|
+
### Requirements
|
56
|
+
|
57
|
+
* [Gnista](https://github.com/emnl/gnista), Ruby bindings for Sparkey
|
58
|
+
* [Sparkey](https://github.com/spotify/sparkey), constant key/value storage library
|
59
|
+
* [Snappy](https://code.google.com/p/snappy/), compression/decompression library (unused, but required to compile Sparkey)
|
60
|
+
* A filesystem that supports `flock(2)` and unlinking files/directories with outstanding file descriptors (ext3/4 will do just fine)
|
61
|
+
|
62
|
+
|
63
|
+
### Installation
|
64
|
+
|
65
|
+
Add the following line to your Gemfile:
|
66
|
+
|
67
|
+
gem 'hammerspace'
|
68
|
+
|
69
|
+
Then run:
|
70
|
+
|
71
|
+
bundle
|
72
|
+
|
73
|
+
### Vagrant
|
74
|
+
|
75
|
+
To make development easier, the source tree contains a Vagrantfile and a small
|
76
|
+
cookbook to install all the prerequisites. The vagrant environment also serves
|
77
|
+
as a consistent environment to run the test suite.
|
78
|
+
|
79
|
+
To use it, make sure you have vagrant installed, then:
|
80
|
+
|
81
|
+
vagrant up
|
82
|
+
vagrant ssh
|
83
|
+
bundle exec rspec
|
84
|
+
|
85
|
+
|
86
|
+
## Usage
|
87
|
+
|
88
|
+
### Getting Started
|
89
|
+
|
90
|
+
For the most part, hammerspace acts like a Ruby hash. But since it's a hash
|
91
|
+
that persists on disk, you have to tell it where to store the files. The
|
92
|
+
enclosing directory and any parent directories are created if they don't
|
93
|
+
already exist.
|
94
|
+
|
95
|
+
```ruby
|
96
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
97
|
+
|
98
|
+
h["cartoons"] = "mallets"
|
99
|
+
h["games"] = "inventory"
|
100
|
+
h["rubyists"] = "data"
|
101
|
+
|
102
|
+
h.size #=> 3
|
103
|
+
h["cartoons"] #=> "mallets"
|
104
|
+
|
105
|
+
h.map { |k,v| "#{k.capitalize} use hammerspace to store #{v}." }
|
106
|
+
|
107
|
+
h.close
|
108
|
+
```
|
109
|
+
|
110
|
+
You should call `close` on the hammerspace object when you're done with it.
|
111
|
+
This flushes any pending writes to disk and closes any open file handles.
|
112
|
+
|
113
|
+
|
114
|
+
### Options
|
115
|
+
|
116
|
+
The constructor takes a hash of options as an optional second argument.
|
117
|
+
Currently the only option supported is `:backend` which specifies which backend
|
118
|
+
class to use. Since there is only one backend supported at this time, there is
|
119
|
+
currently no reason to pass this argument.
|
120
|
+
|
121
|
+
```ruby
|
122
|
+
h = Hammerspace.new("/tmp/hammerspace", {:backend => Hammerspace::Backend::Sparkey})
|
123
|
+
```
|
124
|
+
|
125
|
+
|
126
|
+
### Default Values
|
127
|
+
|
128
|
+
The constructor takes a default value as an optional third argument. This
|
129
|
+
functions the same as Ruby's `Hash`, except with `Hash` it is the first
|
130
|
+
argument.
|
131
|
+
|
132
|
+
```ruby
|
133
|
+
h = Hammerspace.new("/tmp/hammerspace", {}, "default")
|
134
|
+
h["foo"] = "bar"
|
135
|
+
h["foo"] #=> "bar"
|
136
|
+
h["new"] #=> "default"
|
137
|
+
h.close
|
138
|
+
```
|
139
|
+
|
140
|
+
The constructor also takes a block to specify a default Proc, which works the
|
141
|
+
same way as Ruby's `Hash`. As with `Hash`, it is the block's responsibility to
|
142
|
+
store the value in the hash if required.
|
143
|
+
|
144
|
+
```ruby
|
145
|
+
h = Hammerspace.new("/tmp/hammerspace") { |hash, key| hash[key] = "#{key} (default)" }
|
146
|
+
h["new"] #=> "new (default)"
|
147
|
+
h.has_key?("new") #=> true
|
148
|
+
h.close
|
149
|
+
```
|
150
|
+
|
151
|
+
|
152
|
+
### Supported Data Types
|
153
|
+
|
154
|
+
Only string keys and values are supported.
|
155
|
+
|
156
|
+
```ruby
|
157
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
158
|
+
h[1] = "foo" #=> TypeError
|
159
|
+
h["fixnum"] = 8 #=> TypeError
|
160
|
+
h["nil"] = nil #=> TypeError
|
161
|
+
h.close
|
162
|
+
```
|
163
|
+
|
164
|
+
Ruby hashes store references to objects, but hammerspace stores raw bytes. A
|
165
|
+
new Ruby `String` object is created from those bytes when a key is accessed.
|
166
|
+
|
167
|
+
```ruby
|
168
|
+
value = "bar"
|
169
|
+
|
170
|
+
hash = {"foo" => value}
|
171
|
+
hash["foo"] == value #=> true
|
172
|
+
hash["foo"].equal?(value) #=> true
|
173
|
+
|
174
|
+
hammerspace = Hammerspace.new("/tmp/hammerspace")
|
175
|
+
hammerspace["foo"] = value
|
176
|
+
hammerspace["foo"] == value #=> true
|
177
|
+
hammerspace["foo"].equal?(value) #=> false
|
178
|
+
hammerspace.close
|
179
|
+
```
|
180
|
+
|
181
|
+
Since every access results in a new `String` object, mutating values doesn't
|
182
|
+
work unless you create an explicit reference to the string.
|
183
|
+
|
184
|
+
```ruby
|
185
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
186
|
+
h["foo"] = "bar"
|
187
|
+
|
188
|
+
# This doesn't work like Ruby's Hash because every access creates a new object
|
189
|
+
h["foo"].upcase!
|
190
|
+
h["foo"] #=> "bar"
|
191
|
+
|
192
|
+
# An explicit reference is required
|
193
|
+
value = h["foo"]
|
194
|
+
value.upcase!
|
195
|
+
value #=> "BAR"
|
196
|
+
|
197
|
+
# Another access, another a new object
|
198
|
+
h["foo"] #=> "bar"
|
199
|
+
|
200
|
+
h.close
|
201
|
+
```
|
202
|
+
|
203
|
+
This also imples that strings "lose" their encoding when retrieved from
|
204
|
+
hammerspace.
|
205
|
+
|
206
|
+
```ruby
|
207
|
+
value = "bar"
|
208
|
+
value.encoding #=> #<Encoding:UTF-8>
|
209
|
+
|
210
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
211
|
+
h["foo"] = value
|
212
|
+
h["foo"].encoding #=> #<Encoding:ASCII-8BIT>
|
213
|
+
h.close
|
214
|
+
```
|
215
|
+
|
216
|
+
If you require strings in UTF-8, make sure strings are encoded as UTF-8 when
|
217
|
+
storing the key, then force the encoding to be UTF-8 when accessing the key.
|
218
|
+
|
219
|
+
```ruby
|
220
|
+
h[key] = value.encode('utf-8')
|
221
|
+
value = h[key].force_encoding('utf-8')
|
222
|
+
```
|
223
|
+
|
224
|
+
|
225
|
+
### Persistence
|
226
|
+
|
227
|
+
Hammerspace objects are backed by files on disk, so even a new object may
|
228
|
+
already have data in it.
|
229
|
+
|
230
|
+
```ruby
|
231
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
232
|
+
h["foo"] = "bar"
|
233
|
+
h.close
|
234
|
+
|
235
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
236
|
+
h["foo"] #=> "bar"
|
237
|
+
h.close
|
238
|
+
```
|
239
|
+
|
240
|
+
Calling `clear` deletes the data files on disk. The parent directory is not
|
241
|
+
removed, nor is it guaranteed to be empty. Some files containing metadata may
|
242
|
+
still be present, e.g., lock files.
|
243
|
+
|
244
|
+
|
245
|
+
### Concurrency
|
246
|
+
|
247
|
+
Multiple concurrent readers are supported. Readers are isolated from writers,
|
248
|
+
i.e., reads are consistent to the time that the reader was opened. Note that
|
249
|
+
the reader opens its files lazily on first read, not when the hammerspace
|
250
|
+
object is created.
|
251
|
+
|
252
|
+
```ruby
|
253
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
254
|
+
h["foo"] = "bar"
|
255
|
+
h.close
|
256
|
+
|
257
|
+
reader1 = Hammerspace.new("/tmp/hammerspace")
|
258
|
+
reader1["foo"] #=> "bar"
|
259
|
+
|
260
|
+
writer = Hammerspace.new("/tmp/hammerspace")
|
261
|
+
writer["foo"] = "updated"
|
262
|
+
writer.close
|
263
|
+
|
264
|
+
# Still "bar" because reader1 opened its files before the write
|
265
|
+
reader1["foo"] #=> "bar"
|
266
|
+
|
267
|
+
# Updated key is visible because reader2 opened its files after the write
|
268
|
+
reader2 = Hammerspace.new("/tmp/hammerspace")
|
269
|
+
reader2["foo"] #=> "updated"
|
270
|
+
reader2.close
|
271
|
+
|
272
|
+
reader1.close
|
273
|
+
```
|
274
|
+
|
275
|
+
A new hammerspace object does not necessarily need to be created. Calling
|
276
|
+
`close` will close the files, then the reader will open them lazily again on
|
277
|
+
the next read.
|
278
|
+
|
279
|
+
```ruby
|
280
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
281
|
+
h["foo"] = "bar"
|
282
|
+
h.close
|
283
|
+
|
284
|
+
reader = Hammerspace.new("/tmp/hammerspace")
|
285
|
+
reader["foo"] #=> "bar"
|
286
|
+
|
287
|
+
writer = Hammerspace.new("/tmp/hammerspace")
|
288
|
+
writer["foo"] = "updated"
|
289
|
+
writer.close
|
290
|
+
|
291
|
+
reader["foo"] #=> "bar"
|
292
|
+
|
293
|
+
# Close files now, re-open lazily on next read
|
294
|
+
reader.close
|
295
|
+
|
296
|
+
reader["foo"] #=> "updated"
|
297
|
+
reader.close
|
298
|
+
```
|
299
|
+
|
300
|
+
If no hammerspace files exist on disk yet, the reader will fail to open the
|
301
|
+
files. It will try again on next read.
|
302
|
+
|
303
|
+
```ruby
|
304
|
+
reader = Hammerspace.new("/tmp/hammerspace")
|
305
|
+
reader.has_key?("foo") #=> false
|
306
|
+
|
307
|
+
writer = Hammerspace.new("/tmp/hammerspace")
|
308
|
+
writer["foo"] = "bar"
|
309
|
+
writer.close
|
310
|
+
|
311
|
+
# Files are opened here
|
312
|
+
reader.has_key?("foo") #=> true
|
313
|
+
reader.close
|
314
|
+
```
|
315
|
+
|
316
|
+
You can call `uid` to get a unique id that identifies the version of the files
|
317
|
+
being read. `uid` will be `nil` if no hammerspace files exist on disk yet.
|
318
|
+
|
319
|
+
```ruby
|
320
|
+
reader = Hammerspace.new("/tmp/hammerspace")
|
321
|
+
reader.uid #=> nil
|
322
|
+
|
323
|
+
writer = Hammerspace.new("/tmp/hammerspace")
|
324
|
+
writer["foo"] = "bar"
|
325
|
+
writer.close
|
326
|
+
|
327
|
+
reader.close
|
328
|
+
reader.uid #=> "24913_53943df0-e784-4873-ade6-d1cccc848a70"
|
329
|
+
|
330
|
+
# The uid changes on every write, even if the content is the same, i.e., it's
|
331
|
+
# an identifier, not a checksum
|
332
|
+
writer["foo"] = "bar"
|
333
|
+
writer.close
|
334
|
+
|
335
|
+
reader.close
|
336
|
+
reader.uid #=> "24913_9371024e-8c80-477b-8558-7c292bfcbfc1"
|
337
|
+
|
338
|
+
reader.close
|
339
|
+
```
|
340
|
+
|
341
|
+
Multiple concurrent writers are also supported. When a writer flushes its
|
342
|
+
changes it will overwrite any previous versions of the hammerspace.
|
343
|
+
|
344
|
+
In practice, this works because hammerspace is designed to hold data that is
|
345
|
+
bulk-loaded from some authoritative external source. Rather than block writers
|
346
|
+
to enforce consistency, it is simpler to allow writers to concurrently attempt
|
347
|
+
to load the data. The last writer to finish loading the data and flush its
|
348
|
+
writes will have its data persisted.
|
349
|
+
|
350
|
+
```ruby
|
351
|
+
writer1 = Hammerspace.new("/tmp/hammerspace")
|
352
|
+
writer1["color"] = "red"
|
353
|
+
|
354
|
+
# Can start while writer1 is still open
|
355
|
+
writer2 = Hammerspace.new("/tmp/hammerspace")
|
356
|
+
writer2["color"] = "blue"
|
357
|
+
writer2["fruit"] = "banana"
|
358
|
+
writer2.close
|
359
|
+
|
360
|
+
# Reads at this point see writer2's data
|
361
|
+
reader1 = Hammerspace.new("/tmp/hammerspace")
|
362
|
+
reader1["color"] #=> "blue"
|
363
|
+
reader1["fruit"] #=> "banana"
|
364
|
+
reader1.close
|
365
|
+
|
366
|
+
# Replaces writer2's data
|
367
|
+
writer1.close
|
368
|
+
|
369
|
+
# Reads at this point see writer1's data; note that "fruit" key is absent
|
370
|
+
reader2 = Hammerspace.new("/tmp/hammerspace")
|
371
|
+
reader2["color"] #=> "red"
|
372
|
+
reader2["fruit"] #=> nil
|
373
|
+
reader2.close
|
374
|
+
```
|
375
|
+
|
376
|
+
|
377
|
+
### Flushing Writes
|
378
|
+
|
379
|
+
Flushing a write incurs some overhead to build the on-disk hash structures that
|
380
|
+
allows fast lookup later. To avoid the overhead of rebuilding the hash after
|
381
|
+
every write, most write operations do not implicitly flush. Writes can be
|
382
|
+
flushed explicitly by calling `close`.
|
383
|
+
|
384
|
+
Delaying flushing of writes has the side effect of allowing "transactions" --
|
385
|
+
all unflushed writes are private to the hammerspace object doing the writing.
|
386
|
+
|
387
|
+
One exception is the `clear` method which deletes the files on disk. If a
|
388
|
+
reader attempts to open the files immediately after they are deleted, it will
|
389
|
+
perceive the hammerspace to be empty.
|
390
|
+
|
391
|
+
```ruby
|
392
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
393
|
+
h["yesterday"] = "foo"
|
394
|
+
h["today"] = "bar"
|
395
|
+
h.close
|
396
|
+
|
397
|
+
reader1 = Hammerspace.new("/tmp/hammerspace")
|
398
|
+
reader1.keys #=> ["yesterday", "today"]
|
399
|
+
reader1.close
|
400
|
+
|
401
|
+
# Writer wants to remove everything except "today"
|
402
|
+
writer = Hammerspace.new("/tmp/hammerspace")
|
403
|
+
writer.clear
|
404
|
+
|
405
|
+
# Effect of clear is immediately visible to readers
|
406
|
+
reader2 = Hammerspace.new("/tmp/hammerspace")
|
407
|
+
reader2.keys #=> []
|
408
|
+
reader2.close
|
409
|
+
|
410
|
+
writer["today"] = "bar"
|
411
|
+
writer.close
|
412
|
+
|
413
|
+
reader3 = Hammerspace.new("/tmp/hammerspace")
|
414
|
+
reader3.keys #=> ["today"]
|
415
|
+
reader3.close
|
416
|
+
```
|
417
|
+
|
418
|
+
If you want to replace the existing data with new data without flushing in
|
419
|
+
between (i.e., in a "transaction"), use `replace` instead.
|
420
|
+
|
421
|
+
```ruby
|
422
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
423
|
+
h["yesterday"] = "foo"
|
424
|
+
h["today"] = "bar"
|
425
|
+
h.close
|
426
|
+
|
427
|
+
reader1 = Hammerspace.new("/tmp/hammerspace")
|
428
|
+
reader1.keys #=> ["yesterday", "today"]
|
429
|
+
reader1.close
|
430
|
+
|
431
|
+
# Writer wants to remove everything except "today"
|
432
|
+
writer = Hammerspace.new("/tmp/hammerspace")
|
433
|
+
writer.replace({"today" => "bar"})
|
434
|
+
|
435
|
+
# Old keys still present because writer has not flushed yet
|
436
|
+
reader2 = Hammerspace.new("/tmp/hammerspace")
|
437
|
+
reader2.keys #=> ["yesterday", "today"]
|
438
|
+
reader2.close
|
439
|
+
|
440
|
+
writer.close
|
441
|
+
|
442
|
+
reader3 = Hammerspace.new("/tmp/hammerspace")
|
443
|
+
reader3.keys #=> ["today"]
|
444
|
+
reader3.close
|
445
|
+
```
|
446
|
+
|
447
|
+
|
448
|
+
### Interleaving Reads and Writes
|
449
|
+
|
450
|
+
To ensure writes are available to subsequent reads, every read operation
|
451
|
+
implicitly flushes any previous writes.
|
452
|
+
|
453
|
+
```ruby
|
454
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
455
|
+
h["foo"] = "bar"
|
456
|
+
|
457
|
+
# Implicitly flushes write (builds on-disk hash for fast lookup), then opens
|
458
|
+
# newly written on-disk hash for reading
|
459
|
+
h["foo"] #=> "bar"
|
460
|
+
|
461
|
+
h.close
|
462
|
+
```
|
463
|
+
|
464
|
+
While batch reads or writes are relatively fast, interleaved reads and writes
|
465
|
+
are slow because the hash is rebuilt very often.
|
466
|
+
|
467
|
+
```ruby
|
468
|
+
# One flush, fast
|
469
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
470
|
+
h["a"] = "100"
|
471
|
+
h["b"] = "200"
|
472
|
+
h["c"] = "300"
|
473
|
+
h["a"] #=> "100"
|
474
|
+
h["b"] #=> "200"
|
475
|
+
h["c"] #=> "300"
|
476
|
+
h.close
|
477
|
+
|
478
|
+
# Three flushes, slow
|
479
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
480
|
+
h["a"] = "100"
|
481
|
+
h["a"] #=> "100"
|
482
|
+
h["b"] = "200"
|
483
|
+
h["b"] #=> "200"
|
484
|
+
h["c"] = "300"
|
485
|
+
h["c"] #=> "300"
|
486
|
+
h.close
|
487
|
+
```
|
488
|
+
|
489
|
+
To avoid this overhead, and to ensure consistency during iteration, the `each`
|
490
|
+
method opens its own private reader for the duration of the iteration. This is
|
491
|
+
also true for any method that uses `each`, including all methods provided by
|
492
|
+
`Enumerable`.
|
493
|
+
|
494
|
+
```ruby
|
495
|
+
h = Hammerspace.new("/tmp/hammerspace")
|
496
|
+
h["a"] = "100"
|
497
|
+
h["b"] = "200"
|
498
|
+
h["c"] = "300"
|
499
|
+
|
500
|
+
# Flushes the above writes, then opens a private reader for the each call
|
501
|
+
h.each do |key, value|
|
502
|
+
# Writes are done in bulk without flushing in between
|
503
|
+
h[key] = value[0]
|
504
|
+
end
|
505
|
+
|
506
|
+
# Flushes the above writes, then opens the reader
|
507
|
+
h.to_hash #=> {"a"=>"1", "b"=>"2", "c"=>"3"}
|
508
|
+
|
509
|
+
h.close
|
510
|
+
```
|
511
|
+
|
512
|
+
|
513
|
+
### Unsupported Methods
|
514
|
+
|
515
|
+
Besides the incompatibilities with Ruby's `Hash` discussed above, there are
|
516
|
+
some `Hash` methods that are not supported.
|
517
|
+
|
518
|
+
* Methods that return a copy of the hash: `invert`, `merge`, `reject`, `select`
|
519
|
+
* `rehash` is not needed, since hammerspace only supports string keys, and keys are effectively `dup`d
|
520
|
+
* `delete` does not return the value deleted, and it does not support block usage
|
521
|
+
* `hash` and `to_s` are not overriden, so the behavior is that of `Object#hash` and `Object#to_s`
|
522
|
+
* `compare_by_identity`, `compare_by_identity?`
|
523
|
+
* `pretty_print`, `pretty_print_cycle`
|