bud 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +33 -16
- data/bin/budplot +42 -65
- data/bin/budtimelines +235 -0
- data/bin/budvis +24 -122
- data/bin/rebl +1 -0
- data/docs/README.md +21 -10
- data/docs/bfs.md +4 -6
- data/docs/c.html +251 -0
- data/docs/cheat.md +45 -30
- data/docs/deploy.md +26 -26
- data/docs/getstarted.md +6 -4
- data/docs/visualizations.md +43 -31
- data/examples/chat/chat.rb +4 -9
- data/examples/chat/chat_server.rb +1 -8
- data/examples/deploy/deploy_ip_port +1 -0
- data/examples/deploy/keys.rb +5 -0
- data/examples/deploy/tokenring-ec2.rb +9 -9
- data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
- data/examples/deploy/tokenring-thread.rb +15 -0
- data/examples/deploy/tokenring.rb +25 -17
- data/lib/bud/aggs.rb +87 -25
- data/lib/bud/bud_meta.rb +48 -31
- data/lib/bud/bust/bust.rb +16 -15
- data/lib/bud/collections.rb +207 -232
- data/lib/bud/depanalysis.rb +1 -0
- data/lib/bud/deploy/countatomicdelivery.rb +8 -20
- data/lib/bud/deploy/deployer.rb +16 -16
- data/lib/bud/deploy/ec2deploy.rb +34 -35
- data/lib/bud/deploy/forkdeploy.rb +90 -0
- data/lib/bud/deploy/threaddeploy.rb +38 -0
- data/lib/bud/graphs.rb +103 -199
- data/lib/bud/joins.rb +190 -41
- data/lib/bud/monkeypatch.rb +84 -0
- data/lib/bud/rebl.rb +8 -1
- data/lib/bud/rewrite.rb +152 -49
- data/lib/bud/server.rb +1 -0
- data/lib/bud/state.rb +24 -10
- data/lib/bud/storage/dbm.rb +170 -0
- data/lib/bud/storage/tokyocabinet.rb +5 -1
- data/lib/bud/stratify.rb +6 -7
- data/lib/bud/viz.rb +31 -17
- data/lib/bud/viz_util.rb +204 -0
- data/lib/bud.rb +271 -244
- data/lib/bud.rb.orig +806 -0
- metadata +43 -22
- data/docs/bfs.raw +0 -251
- data/docs/diffs +0 -181
- data/examples/basics/out +0 -1103
- data/examples/basics/out.new +0 -856
- data/lib/bud/deploy/localdeploy.rb +0 -53
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 4
|
10
|
+
version: 0.0.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Peter Alvaro
|
@@ -18,7 +18,7 @@ autorequire:
|
|
18
18
|
bindir: bin
|
19
19
|
cert_chain: []
|
20
20
|
|
21
|
-
date: 2011-
|
21
|
+
date: 2011-06-01 00:00:00 -07:00
|
22
22
|
default_executable: rebl
|
23
23
|
dependencies:
|
24
24
|
- !ruby/object:Gem::Dependency
|
@@ -64,7 +64,7 @@ dependencies:
|
|
64
64
|
type: :runtime
|
65
65
|
version_requirements: *id003
|
66
66
|
- !ruby/object:Gem::Dependency
|
67
|
-
name:
|
67
|
+
name: getopt
|
68
68
|
prerelease: false
|
69
69
|
requirement: &id004 !ruby/object:Gem::Requirement
|
70
70
|
none: false
|
@@ -78,7 +78,7 @@ dependencies:
|
|
78
78
|
type: :runtime
|
79
79
|
version_requirements: *id004
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
|
-
name:
|
81
|
+
name: i18n
|
82
82
|
prerelease: false
|
83
83
|
requirement: &id005 !ruby/object:Gem::Requirement
|
84
84
|
none: false
|
@@ -92,7 +92,7 @@ dependencies:
|
|
92
92
|
type: :runtime
|
93
93
|
version_requirements: *id005
|
94
94
|
- !ruby/object:Gem::Dependency
|
95
|
-
name:
|
95
|
+
name: json
|
96
96
|
prerelease: false
|
97
97
|
requirement: &id006 !ruby/object:Gem::Requirement
|
98
98
|
none: false
|
@@ -106,7 +106,7 @@ dependencies:
|
|
106
106
|
type: :runtime
|
107
107
|
version_requirements: *id006
|
108
108
|
- !ruby/object:Gem::Dependency
|
109
|
-
name:
|
109
|
+
name: msgpack
|
110
110
|
prerelease: false
|
111
111
|
requirement: &id007 !ruby/object:Gem::Requirement
|
112
112
|
none: false
|
@@ -120,7 +120,7 @@ dependencies:
|
|
120
120
|
type: :runtime
|
121
121
|
version_requirements: *id007
|
122
122
|
- !ruby/object:Gem::Dependency
|
123
|
-
name:
|
123
|
+
name: nestful
|
124
124
|
prerelease: false
|
125
125
|
requirement: &id008 !ruby/object:Gem::Requirement
|
126
126
|
none: false
|
@@ -134,7 +134,7 @@ dependencies:
|
|
134
134
|
type: :runtime
|
135
135
|
version_requirements: *id008
|
136
136
|
- !ruby/object:Gem::Dependency
|
137
|
-
name:
|
137
|
+
name: ParseTree
|
138
138
|
prerelease: false
|
139
139
|
requirement: &id009 !ruby/object:Gem::Requirement
|
140
140
|
none: false
|
@@ -148,7 +148,7 @@ dependencies:
|
|
148
148
|
type: :runtime
|
149
149
|
version_requirements: *id009
|
150
150
|
- !ruby/object:Gem::Dependency
|
151
|
-
name:
|
151
|
+
name: ruby-graphviz
|
152
152
|
prerelease: false
|
153
153
|
requirement: &id010 !ruby/object:Gem::Requirement
|
154
154
|
none: false
|
@@ -162,7 +162,7 @@ dependencies:
|
|
162
162
|
type: :runtime
|
163
163
|
version_requirements: *id010
|
164
164
|
- !ruby/object:Gem::Dependency
|
165
|
-
name:
|
165
|
+
name: ruby2ruby
|
166
166
|
prerelease: false
|
167
167
|
requirement: &id011 !ruby/object:Gem::Requirement
|
168
168
|
none: false
|
@@ -176,7 +176,7 @@ dependencies:
|
|
176
176
|
type: :runtime
|
177
177
|
version_requirements: *id011
|
178
178
|
- !ruby/object:Gem::Dependency
|
179
|
-
name:
|
179
|
+
name: sexp_path
|
180
180
|
prerelease: false
|
181
181
|
requirement: &id012 !ruby/object:Gem::Requirement
|
182
182
|
none: false
|
@@ -190,7 +190,7 @@ dependencies:
|
|
190
190
|
type: :runtime
|
191
191
|
version_requirements: *id012
|
192
192
|
- !ruby/object:Gem::Dependency
|
193
|
-
name:
|
193
|
+
name: superators
|
194
194
|
prerelease: false
|
195
195
|
requirement: &id013 !ruby/object:Gem::Requirement
|
196
196
|
none: false
|
@@ -204,7 +204,7 @@ dependencies:
|
|
204
204
|
type: :runtime
|
205
205
|
version_requirements: *id013
|
206
206
|
- !ruby/object:Gem::Dependency
|
207
|
-
name:
|
207
|
+
name: syntax
|
208
208
|
prerelease: false
|
209
209
|
requirement: &id014 !ruby/object:Gem::Requirement
|
210
210
|
none: false
|
@@ -217,6 +217,20 @@ dependencies:
|
|
217
217
|
version: "0"
|
218
218
|
type: :runtime
|
219
219
|
version_requirements: *id014
|
220
|
+
- !ruby/object:Gem::Dependency
|
221
|
+
name: uuid
|
222
|
+
prerelease: false
|
223
|
+
requirement: &id015 !ruby/object:Gem::Requirement
|
224
|
+
none: false
|
225
|
+
requirements:
|
226
|
+
- - ">="
|
227
|
+
- !ruby/object:Gem::Version
|
228
|
+
hash: 3
|
229
|
+
segments:
|
230
|
+
- 0
|
231
|
+
version: "0"
|
232
|
+
type: :runtime
|
233
|
+
version_requirements: *id015
|
220
234
|
description: A prototype of the Bloom distributed programming language, as a Ruby DSL.
|
221
235
|
email:
|
222
236
|
- bloomdevs@gmail.com
|
@@ -224,6 +238,7 @@ executables:
|
|
224
238
|
- rebl
|
225
239
|
- budplot
|
226
240
|
- budvis
|
241
|
+
- budtimelines
|
227
242
|
extensions: []
|
228
243
|
|
229
244
|
extra_rdoc_files: []
|
@@ -239,31 +254,36 @@ files:
|
|
239
254
|
- lib/bud/deploy/countatomicdelivery.rb
|
240
255
|
- lib/bud/deploy/deployer.rb
|
241
256
|
- lib/bud/deploy/ec2deploy.rb
|
242
|
-
- lib/bud/deploy/
|
257
|
+
- lib/bud/deploy/forkdeploy.rb
|
258
|
+
- lib/bud/deploy/threaddeploy.rb
|
243
259
|
- lib/bud/errors.rb
|
244
260
|
- lib/bud/graphs.rb
|
245
261
|
- lib/bud/joins.rb
|
262
|
+
- lib/bud/monkeypatch.rb
|
246
263
|
- lib/bud/rebl.rb
|
247
264
|
- lib/bud/rewrite.rb
|
248
265
|
- lib/bud/rtrace.rb
|
249
266
|
- lib/bud/server.rb
|
250
267
|
- lib/bud/state.rb
|
268
|
+
- lib/bud/storage/dbm.rb
|
251
269
|
- lib/bud/storage/tokyocabinet.rb
|
252
270
|
- lib/bud/storage/zookeeper.rb
|
253
271
|
- lib/bud/stratify.rb
|
254
272
|
- lib/bud/viz.rb
|
273
|
+
- lib/bud/viz_util.rb
|
255
274
|
- lib/bud.rb
|
275
|
+
- lib/bud.rb.orig
|
256
276
|
- bin/budplot
|
277
|
+
- bin/budtimelines
|
257
278
|
- bin/budvis
|
258
279
|
- bin/rebl
|
259
280
|
- docs/bfs.md
|
260
|
-
- docs/bfs.raw
|
261
281
|
- docs/bfs_arch.png
|
262
282
|
- docs/bloom-loop.png
|
263
283
|
- docs/bust.md
|
284
|
+
- docs/c.html
|
264
285
|
- docs/cheat.md
|
265
286
|
- docs/deploy.md
|
266
|
-
- docs/diffs
|
267
287
|
- docs/getstarted.md
|
268
288
|
- docs/intro.md
|
269
289
|
- docs/modules.md
|
@@ -273,8 +293,6 @@ files:
|
|
273
293
|
- docs/ruby_hooks.md
|
274
294
|
- docs/visualizations.md
|
275
295
|
- examples/basics/hello.rb
|
276
|
-
- examples/basics/out
|
277
|
-
- examples/basics/out.new
|
278
296
|
- examples/basics/paths.rb
|
279
297
|
- examples/bust/bustclient-example.rb
|
280
298
|
- examples/bust/bustinspector.html
|
@@ -284,8 +302,11 @@ files:
|
|
284
302
|
- examples/chat/chat_protocol.rb
|
285
303
|
- examples/chat/chat_server.rb
|
286
304
|
- examples/chat/README.md
|
305
|
+
- examples/deploy/deploy_ip_port
|
306
|
+
- examples/deploy/keys.rb
|
287
307
|
- examples/deploy/tokenring-ec2.rb
|
288
|
-
- examples/deploy/tokenring-
|
308
|
+
- examples/deploy/tokenring-fork.rb
|
309
|
+
- examples/deploy/tokenring-thread.rb
|
289
310
|
- examples/deploy/tokenring.rb
|
290
311
|
- examples/README
|
291
312
|
- README
|
@@ -302,7 +323,7 @@ require_paths:
|
|
302
323
|
required_ruby_version: !ruby/object:Gem::Requirement
|
303
324
|
none: false
|
304
325
|
requirements:
|
305
|
-
- -
|
326
|
+
- - ~>
|
306
327
|
- !ruby/object:Gem::Version
|
307
328
|
hash: 57
|
308
329
|
segments:
|
data/docs/bfs.raw
DELETED
@@ -1,251 +0,0 @@
|
|
1
|
-
# BFS: A distributed file system in Bloom
|
2
|
-
|
3
|
-
In this document we'll use what we've learned to build a piece of systems software using Bloom. The libraries that ship with Bud provide many of the building blocks we'll need to create a distributed,
|
4
|
-
``chunked'' file system in the style of the Google File System (GFS):
|
5
|
-
|
6
|
-
* a [key-value store](https://github.com/bloom-lang/bud-sandbox/blob/master/kvs/kvs.rb) (KVS)
|
7
|
-
* [nonce generation](https://github.com/bloom-lang/bud-sandbox/blob/master/ordering/nonce.rb)
|
8
|
-
* a [heartbeat protocol](https://github.com/bloom-lang/bud-sandbox/blob/master/heartbeat/heartbeat.rb)
|
9
|
-
|
10
|
-
## High-level architecture
|
11
|
-
|
12
|
-

|
13
|
-
|
14
|
-
BFS implements a chunked, distributed file system (mostly) in the Bloom
|
15
|
-
language. BFS is architecturally based on [BOOM-FS](http://db.cs.berkeley.edu/papers/eurosys10-boom.pdf), which is itself based on
|
16
|
-
the Google File System (GFS). As in GFS, a single master node manages
|
17
|
-
file system metadata, while data blocks are replicated and stored on a large
|
18
|
-
number of storage nodes. Writing or reading data involves a multi-step
|
19
|
-
protocol in which clients interact with the master, retrieving metadata and
|
20
|
-
possibly changing state, then interact with storage nodes to read or write
|
21
|
-
chunks. Background jobs running on the master will contact storage nodes to
|
22
|
-
orchestrate chunk migrations, during which storage nodes communicate with
|
23
|
-
other storage nodes. As in BOOM-FS, the communication protocols and the data
|
24
|
-
channel used for bulk data transfer between clients and datanodes and between
|
25
|
-
datanodes is written outside Bloom (in Ruby).
|
26
|
-
|
27
|
-
## [Basic File System](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/fs_master.rb)
|
28
|
-
|
29
|
-
Before we worry about any of the details of distribution, we need to implement the basic file system metadata operations: _create_, _remove_, _mkdir_ and _ls_.
|
30
|
-
There are many choices for how to implement these operations, and it makes sense to keep them separate from the (largely orthogonal) distributed file system logic.
|
31
|
-
That way, it will be possible later to choose a different implementation of the metadata operations without impacting the rest of the system.
|
32
|
-
Another benefit of modularizing the metadata logic is that it can be independently tested and debugged. We want to get the core of the file system
|
33
|
-
working correctly before we even send a whisper over the network, let alone add any complex features.
|
34
|
-
|
35
|
-
### Protocol
|
36
|
-
|
37
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|12-20
|
38
|
-
|
39
|
-
We create an input interface for each of the operations, and a single output interface for the return for any operation: given a request id, __status__ is a boolean
|
40
|
-
indicating whether the request succeeded, and __data__ may contain return values (e.g., _fsls_ should return an array containing the array contents).
|
41
|
-
|
42
|
-
### Implementation
|
43
|
-
|
44
|
-
We already have a library that provides an updateable flat namespace: the key-value store. We can easily implement the tree structure of a file system over a key-value store
|
45
|
-
in the following way:
|
46
|
-
|
47
|
-
1. keys are paths
|
48
|
-
2. directories have arrays containing child entries (base names)
|
49
|
-
3. files values are their contents
|
50
|
-
|
51
|
-
<!--- (**JMH**: I find it a bit confusing how you toggle from the discussion above to this naive file-storage design here. Can you warn us a bit more clearly that this is a starting point focused on metadata, with (3) being a strawman for data storage that is intended to be overriden later?)
|
52
|
-
--->
|
53
|
-
Note that (3) is a strawman: it will cease to apply when we implement chunked storage later. It is tempting, however, to support (3) so that the resulting program is a working
|
54
|
-
standalone file system.
|
55
|
-
|
56
|
-
We begin our implementation of a KVS-backed metadata system in the following way:
|
57
|
-
|
58
|
-
|
59
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|33-36
|
60
|
-
|
61
|
-
If we wanted to replicate the master node's metadata we could consider mixing in a replicated KVS implementation instead of __BasicKVS__ -- but more on that later.
|
62
|
-
|
63
|
-
### Directory Listing
|
64
|
-
|
65
|
-
The directory listing operation is implemented by a simple block of Bloom statements:
|
66
|
-
|
67
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|51-57
|
68
|
-
|
69
|
-
If we get a __fsls__ request, probe the key-value store for the requested by projecting _reqid_, _path_ from the __fsls__ tuple into __kvget__. If the given path
|
70
|
-
is a key, __kvget_response__ will contain a tuple with the same _reqid_, and the join on the second line will succeed. In this case, we insert the value
|
71
|
-
associated with that key into __fsret__. Otherwise, the third rule will fire, inserting a failure tuple into __fsret__.
|
72
|
-
|
73
|
-
|
74
|
-
### Mutation
|
75
|
-
|
76
|
-
The logic for file and directory creation and deletion follow a similar logic with regard to the parent directory:
|
77
|
-
|
78
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|61-71
|
79
|
-
|
80
|
-
Unlike a directory listing, however, these operations change the state of the file system. In general, any state change will involve
|
81
|
-
carrying out two mutating operations to the key-value store atomically:
|
82
|
-
|
83
|
-
1. update the value (child array) associated with the parent directory entry
|
84
|
-
2. update the key-value pair associated with the object in question (a file or directory being created or destroyed).
|
85
|
-
|
86
|
-
The following Bloom code carries this out:
|
87
|
-
|
88
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|73-73
|
89
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|80-108
|
90
|
-
|
91
|
-
|
92
|
-
<!--- (**JMH**: This next sounds awkward. You *do* take care: by using <= and understanding the atomicity of timesteps in Bloom. I think what you mean to say is that Bloom's atomic timestep model makes this easy compared to ... something.)
|
93
|
-
Note that we need not take any particular care to ensure that the two inserts into __kvput__ occur together atomically. Because both statements use the synchronous
|
94
|
-
-->
|
95
|
-
Note that because both inserts into the __kvput__ collection use the synchronous operator (`<=`), we know that they will occur together in the same fixpoint computation or not at all.
|
96
|
-
Therefore we need not be concerned with explicitly sequencing the operations (e.g., ensuring that the directory entries is created _after_ the file entry) to deal with concurrency:
|
97
|
-
there can be no visible state of the database in which only one of the operations has succeeded.
|
98
|
-
|
99
|
-
If the request is a deletion, we need some additional logic to enforce the constraint that only an empty directory may be removed:
|
100
|
-
|
101
|
-
|
102
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|74-78
|
103
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|110-115
|
104
|
-
|
105
|
-
|
106
|
-
Recall that when we created KVSFS we mixed in __TimestepNonce__, one of the nonce libraries. While we were able to use the _reqid_ field from the input operation as a unique identifier
|
107
|
-
for one of our KVS operations, we need a fresh, unique request id for the second KVS operation in the atomic pair described above. By joining __nonce__, we get
|
108
|
-
an identifier that is unique to this timestep.
|
109
|
-
|
110
|
-
|
111
|
-
## [File Chunking](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/chunking.rb)
|
112
|
-
|
113
|
-
Now that we have a module providing a basic file system, we can extend it to support chunked storage of file contents. The metadata master will contain, in addition to the KVS
|
114
|
-
structure for directory information, a relation mapping a set of chunk identifiers to each file
|
115
|
-
|
116
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|26-26
|
117
|
-
|
118
|
-
and relations associating a chunk with a set of datanodes that host a replica of the chunk.
|
119
|
-
|
120
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|12-12
|
121
|
-
|
122
|
-
(**JMH**: ambiguous reference ahead "these latter")
|
123
|
-
The latter (defined in __HBMaster__) is soft-state, kept up to data by heartbeat messages from datanodes (described in the next section).
|
124
|
-
|
125
|
-
To support chunked storage, we add a few metadata operations to those already defined by FSProtocol:
|
126
|
-
|
127
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|6-16
|
128
|
-
|
129
|
-
* __fschunklist__ returns the set of chunks belonging to a given file.
|
130
|
-
* __fschunklocations__ returns the set of datanodes in possession of a given chunk.
|
131
|
-
* __fsaddchunk__ returns a new chunkid for appending to an existing file, guaranteed to be higher than any existing chunkids for that file, and a list of candidate datanodes that can store a replica of the new chunk.
|
132
|
-
|
133
|
-
We continue to use __fsret__ for return values.
|
134
|
-
|
135
|
-
### Lookups
|
136
|
-
|
137
|
-
Lines 34-44 are a similar pattern to what we saw in the basic FS: whenever we get a __fschunklist__ or __fsaddchunk__ request, we must first ensure that the given file
|
138
|
-
exists, and error out if not. If it does, and the operation was __fschunklist__, we join the metadata relation __chunk__ and return the set of chunks owned
|
139
|
-
by the given (existent) file:
|
140
|
-
|
141
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|47-49
|
142
|
-
|
143
|
-
### Add chunk
|
144
|
-
|
145
|
-
If it was a __fsaddchunk__ request, we need to generate a unique id for a new chunk and return a list of target datanodes. We reuse __TimestepNonce__ to do the former, and join a relation
|
146
|
-
called __available__ that is exported by __HBMaster__ (described in the next section) for the latter:
|
147
|
-
|
148
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|69-76
|
149
|
-
|
150
|
-
Finally, it was a __fschunklocations__ request, we have another possible error scenario, because the nodes associated with chunks are a part of our soft state. Even if the file
|
151
|
-
exists, it may not be the case that we have fresh information in our cache about what datanodes own a replica of the given chunk:
|
152
|
-
|
153
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|54-58
|
154
|
-
|
155
|
-
Otherwise, __chunk_cache__ has information about the given chunk, which we may return to the client:
|
156
|
-
|
157
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|61-64
|
158
|
-
|
159
|
-
|
160
|
-
## Datanodes and Heartbeats
|
161
|
-
|
162
|
-
### [Datanode](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/datanode.rb)
|
163
|
-
|
164
|
-
A datanode runs both Bud code (to support the heartbeat and control protocols) and pure Ruby (to support the data transfer protocol). A datanode's main job is keeping the master
|
165
|
-
aware of it existence and its state, and participating when necessary in data pipelines to read or write chunk data to and from its local storage.
|
166
|
-
|
167
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|11-15
|
168
|
-
|
169
|
-
By mixing in HeartbeatAgent, the datanode includes the machinery necessary to regularly send status messages to the master. __HeartbeatAgent__ provides an input interface
|
170
|
-
called __payload__ that allows an agent to optionally include additional information in heartbeat messages: in our case, we wish to include state deltas which ensure that
|
171
|
-
the master has an accurate view of the set of chunks owned by the datanode.
|
172
|
-
|
173
|
-
When a datanode is constructed, it takes a port at which the embedded data protocol server will listen, and starts the server in the background:
|
174
|
-
|
175
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|61-62
|
176
|
-
|
177
|
-
At regular intervals, a datanode polls its local chunk directory (which is independently written to by the data protocol):
|
178
|
-
|
179
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|26-31
|
180
|
-
|
181
|
-
We update the payload that we send to the master if our recent poll found files that we don't believe the master knows about:
|
182
|
-
|
183
|
-
|
184
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|33-40
|
185
|
-
|
186
|
-
Our view of what the master ``knows'' about reflects our local cache of acknowledgement messages from the master. This logic is defined in __HBMaster__.
|
187
|
-
|
188
|
-
### [Heartbeat master logic](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/hb_master.rb)
|
189
|
-
|
190
|
-
On the master side of heartbeats, we always send an ack when we get a heartbeat, so that the datanode doesn't need to keep resending its
|
191
|
-
payload of local chunks:
|
192
|
-
|
193
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|30-32
|
194
|
-
|
195
|
-
At the same time, we use the Ruby _flatmap_ method to flatten the array of chunks in the heartbeat payload into a set of tuples, which we
|
196
|
-
associate with the heartbeating datanode and the time of receipt in __chunk_cache__:
|
197
|
-
|
198
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|22-28
|
199
|
-
|
200
|
-
We periodically garbage-collect this cached, removing entries for datanodes from whom we have not received a heartbeat in a configurable amount of time.
|
201
|
-
__last_heartbeat__ is an output interface provided by the __HeartbeatAgent__ module, and contains the most recent, non-stale heartbeat contents:
|
202
|
-
|
203
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|34-36
|
204
|
-
|
205
|
-
|
206
|
-
## [BFS Client](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/bfs_client.rb)
|
207
|
-
|
208
|
-
One of the most complicated parts of the basic GFS design is the client component. To minimize load on the centralized master, we take it off the critical
|
209
|
-
path of file transfers. The client therefore needs to pick up this work.
|
210
|
-
|
211
|
-
We won't spend too much time on the details of the client code, as it is nearly all _plain old Ruby_. The basic idea is:
|
212
|
-
|
213
|
-
1. Pure metadata operations
|
214
|
-
* _mkdir_, _create_, _ls_, _rm_
|
215
|
-
* Send the request to the master and inform the caller of the status.
|
216
|
-
* If _ls_, return the directory listing to the caller.
|
217
|
-
2. Append
|
218
|
-
* Send a __fsaddchunk__ request to the master, which should return a new chunkid and a list of datanodes.
|
219
|
-
* Read a chunk worth of data from the input stream.
|
220
|
-
* Connect to the first datanode in the list. Send a header containing the chunkid and the remaining datanodes.
|
221
|
-
* Stream the file contents. The target datanode will then ``play client'' and continue the pipeline to the next datanode, and so on.
|
222
|
-
2. Read
|
223
|
-
* Send a __getchunks__ request to the master for the given file. It should return the list of chunks owned by the file.
|
224
|
-
* For each chunk,
|
225
|
-
* Send a __fschunklocations__ request to the master, which should return a list of datanodes in possession of the chunk (returning a list allows the client to perform retries without more communication with the master, should some of the datanodes fail).
|
226
|
-
* Connect to a datanode from the list and stream the chunk to a local buffer.
|
227
|
-
* As chunks become available, stream them to the caller.
|
228
|
-
|
229
|
-
|
230
|
-
## [Data transfer protocol](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/data_protocol.rb)
|
231
|
-
|
232
|
-
The data transfer protocol comprises a set of support functions for the bulk data transfer protocol whose use is described in the previous section.
|
233
|
-
Because it is _plain old Ruby_ it is not as interesting as the other modules. It provides:
|
234
|
-
|
235
|
-
* The TCP server code that runs at each datanode, which parses headers and writes stream data to the local FS (these files are later detected by the directory poll).
|
236
|
-
* Client API calls to connect to datanodes and stream data. Datanodes also use this protocol to pipeline chunks to downstream datanodes.
|
237
|
-
* Master API code invoked by a background process to replicate chunks from datanodes to other datanodes, when the replication factor for a chunk is too low.
|
238
|
-
|
239
|
-
## [Master background process](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/background.rb)
|
240
|
-
|
241
|
-
So far, we have implemented the BFS master as a strictly reactive system: when clients make requests, it queries and possibly updates local state.
|
242
|
-
To maintain the durability requirement that `REP_FACTOR` copies of every chunk are stored on distinct nodes, the master must be an active system
|
243
|
-
that maintains a near-consistent view of global state, and takes steps to correct violated requirements.
|
244
|
-
|
245
|
-
__chunk_cache__ is the master's view of datanode state, maintained as described by collecting and pruning heartbeat messages.
|
246
|
-
|
247
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/background.rb|24-27
|
248
|
-
|
249
|
-
After defining some helper aggregates (__chunk_cnts_chunk__ or replica count by chunk, and __chunk_cnt_host__ or datanode fill factor),
|
250
|
-
|
251
|
-
==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/background.rb|29-36
|
data/docs/diffs
DELETED
@@ -1,181 +0,0 @@
|
|
1
|
-
24c24
|
2
|
-
< channel used for communication (**JMH**: "bulk data transfer"?) between clients and datanodes and between
|
3
|
-
---
|
4
|
-
> channel used for communication between clients and datanodes and between
|
5
|
-
35,43c35
|
6
|
-
< module FSProtocol
|
7
|
-
< state do
|
8
|
-
< interface input, :fsls, [:reqid, :path]
|
9
|
-
< interface input, :fscreate, [] => [:reqid, :name, :path, :data]
|
10
|
-
< interface input, :fsmkdir, [] => [:reqid, :name, :path]
|
11
|
-
< interface input, :fsrm, [] => [:reqid, :name, :path]
|
12
|
-
< interface output, :fsret, [:reqid, :status, :data]
|
13
|
-
< end
|
14
|
-
< end
|
15
|
-
---
|
16
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|12-20
|
17
|
-
57,58d48
|
18
|
-
< (**JMH**: I find it a bit confusing how you toggle from the discussion above to this naive file-storage design here. Can you warn us a bit more clearly that this is a starting point focused on metadata, with a strawman for data storage to be overriden later?)
|
19
|
-
<
|
20
|
-
62,65c52
|
21
|
-
< module KVSFS
|
22
|
-
< include FSProtocol
|
23
|
-
< include BasicKVS
|
24
|
-
< include TimestepNonce
|
25
|
-
---
|
26
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|33-36
|
27
|
-
67c54
|
28
|
-
< If we wanted to replicate the metadata master (**JMH**: "the master node's metadata"?), we could consider mixing in a replicated KVS implementation instead of __BasicKVS__ -- but more on that later.
|
29
|
-
---
|
30
|
-
> If we wanted to replicate the metadata master, we could consider mixing in a replicated KVS implementation instead of __BasicKVS__ -- but more on that later.
|
31
|
-
71c58
|
32
|
-
< The directory listing operation is implemented by a simple block of Bloom statements:
|
33
|
-
---
|
34
|
-
> The directory listing operation is very simple:
|
35
|
-
73,81c60
|
36
|
-
< bloom :elles do
|
37
|
-
< kvget <= fsls.map{ |l| [l.reqid, l.path] }
|
38
|
-
< fsret <= join([kvget_response, fsls], [kvget_response.reqid, fsls.reqid]).map{ |r, i| [r.reqid, true, r.value] }
|
39
|
-
< fsret <= fsls.map do |l|
|
40
|
-
< unless kvget_response.map{ |r| r.reqid}.include? l.reqid
|
41
|
-
< [l.reqid, false, nil]
|
42
|
-
< end
|
43
|
-
< end
|
44
|
-
< end
|
45
|
-
---
|
46
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|51-59
|
47
|
-
96d74
|
48
|
-
< (**JMH**: Transition: "The following Bloom code carries this out for ...")
|
49
|
-
98,134c76
|
50
|
-
< dir_exists = join [check_parent_exists, kvget_response, nonce], [check_parent_exists.reqid, kvget_response.reqid]
|
51
|
-
<
|
52
|
-
< check_is_empty <= join([fsrm, nonce]).map{|m, n| [n.ident, m.reqid, terminate_with_slash(m.path) + m.name] }
|
53
|
-
< kvget <= check_is_empty.map{|c| [c.reqid, c.name] }
|
54
|
-
< can_remove <= join([kvget_response, check_is_empty], [kvget_response.reqid, check_is_empty.reqid]).map do |r, c|
|
55
|
-
< [c.reqid, c.orig_reqid, c.name] if r.value.length == 0
|
56
|
-
< end
|
57
|
-
<
|
58
|
-
< fsret <= dir_exists.map do |c, r, n|
|
59
|
-
< if c.mtype == :rm
|
60
|
-
< unless can_remove.map{|can| can.orig_reqid}.include? c.reqid
|
61
|
-
< [c.reqid, false, "directory #{} not empty"]
|
62
|
-
< end
|
63
|
-
< end
|
64
|
-
< end
|
65
|
-
<
|
66
|
-
< # update dir entry
|
67
|
-
< # note that it is unnecessary to ensure that a file is created before its corresponding
|
68
|
-
< # directory entry, as both inserts into :kvput below will co-occur in the same timestep.
|
69
|
-
< kvput <= dir_exists.map do |c, r, n|
|
70
|
-
< if c.mtype == :rm
|
71
|
-
< if can_remove.map{|can| can.orig_reqid}.include? c.reqid
|
72
|
-
< [ip_port, c.path, n.ident, r.value.clone.reject{|item| item == c.name}]
|
73
|
-
< end
|
74
|
-
< else
|
75
|
-
< [ip_port, c.path, n.ident, r.value.clone.push(c.name)]
|
76
|
-
< end
|
77
|
-
< end
|
78
|
-
<
|
79
|
-
< kvput <= dir_exists.map do |c, r, n|
|
80
|
-
< case c.mtype
|
81
|
-
< when :mkdir
|
82
|
-
< [ip_port, terminate_with_slash(c.path) + c.name, c.reqid, []]
|
83
|
-
< when :create
|
84
|
-
< [ip_port, terminate_with_slash(c.path) + c.name, c.reqid, "LEAF"]
|
85
|
-
< end
|
86
|
-
< end
|
87
|
-
---
|
88
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|74-110
|
89
|
-
136d77
|
90
|
-
< (**JMH**: This next sounds awkward. You *do* take care: by using <= and understanding the atomicity of timesteps in Bloom. I think what you mean to say is that Bloom's atomic timestep model makes this easy compared to ... something.)
|
91
|
-
150c91
|
92
|
-
< table :chunk, [:chunkid, :file, :siz]
|
93
|
-
---
|
94
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|26-26
|
95
|
-
154c95
|
96
|
-
< table :chunk_cache, [:node, :chunkid, :time]
|
97
|
-
---
|
98
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|12-12
|
99
|
-
156d96
|
100
|
-
< (**JMH**: ambiguous reference ahead "these latter")
|
101
|
-
161,171c101
|
102
|
-
< module ChunkedFSProtocol
|
103
|
-
< include FSProtocol
|
104
|
-
<
|
105
|
-
< state do
|
106
|
-
< interface :input, :fschunklist, [:reqid, :file]
|
107
|
-
< interface :input, :fschunklocations, [:reqid, :chunkid]
|
108
|
-
< interface :input, :fsaddchunk, [:reqid, :file]
|
109
|
-
< # note that no output interface is defined.
|
110
|
-
< # we use :fsret (defined in FSProtocol) for output.
|
111
|
-
< end
|
112
|
-
< end
|
113
|
-
---
|
114
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|6-16
|
115
|
-
185,187c115
|
116
|
-
< chunk_buffer <= join([fschunklist, kvget_response, chunk], [fschunklist.reqid, kvget_response.reqid], [fschunklist.file, chunk.file]).map{ |l, r, c| [l.reqid, c.chunkid] }
|
117
|
-
< chunk_buffer2 <= chunk_buffer.group([chunk_buffer.reqid], accum(chunk_buffer.chunkid))
|
118
|
-
< fsret <= chunk_buffer2.map{ |c| [c.reqid, true, c.chunklist] }
|
119
|
-
---
|
120
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|47-49
|
121
|
-
191d118
|
122
|
-
< (**JMH**: Ambiguous ref "If it")
|
123
|
-
195,202c122
|
124
|
-
< minted_chunk = join([kvget_response, fsaddchunk, available, nonce], [kvget_response.reqid, fsaddchunk.reqid])
|
125
|
-
< chunk <= minted_chunk.map{ |r, a, v, n| [n.ident, a.file, 0] }
|
126
|
-
< fsret <= minted_chunk.map{ |r, a, v, n| [r.reqid, true, [n.ident, v.pref_list.slice(0, (REP_FACTOR + 2))]] }
|
127
|
-
< fsret <= join([kvget_response, fsaddchunk], [kvget_response.reqid, fsaddchunk.reqid]).map do |r, a|
|
128
|
-
< if available.empty? or available.first.pref_list.length < REP_FACTOR
|
129
|
-
< [r.reqid, false, "datanode set cannot satisfy REP_FACTOR = #{REP_FACTOR} with [#{available.first.nil? ? "NIL" : available.first.pref_list.inspect}]"]
|
130
|
-
< end
|
131
|
-
< end
|
132
|
-
---
|
133
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|69-76
|
134
|
-
204d123
|
135
|
-
< (**JMH**: Ambiguous ref "If it")
|
136
|
-
208,212c127
|
137
|
-
< fsret <= fschunklocations.map do |l|
|
138
|
-
< unless chunk_cache.map{|c| c.chunkid}.include? l.chunkid
|
139
|
-
< [l.reqid, false, "no datanodes found for #{l.chunkid} in cc, now #{chunk_cache.length}"]
|
140
|
-
< end
|
141
|
-
< end
|
142
|
-
---
|
143
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|54-58
|
144
|
-
216,219c131
|
145
|
-
< chunkjoin = join [fschunklocations, chunk_cache], [fschunklocations.chunkid, chunk_cache.chunkid]
|
146
|
-
< host_buffer <= chunkjoin.map{|l, c| [l.reqid, c.node] }
|
147
|
-
< host_buffer2 <= host_buffer.group([host_buffer.reqid], accum(host_buffer.host))
|
148
|
-
< fsret <= host_buffer2.map{|c| [c.reqid, true, c.hostlist] }
|
149
|
-
---
|
150
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|61-64
|
151
|
-
229,233c141
|
152
|
-
< module BFSDatanode
|
153
|
-
< include HeartbeatAgent
|
154
|
-
< include StaticMembership
|
155
|
-
< include TimestepNonce
|
156
|
-
< include BFSHBProtocol
|
157
|
-
---
|
158
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|9-13
|
159
|
-
241c149
|
160
|
-
< @dp_server = DataProtocolServer.new(dataport)
|
161
|
-
---
|
162
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|53-53
|
163
|
-
245,250c153
|
164
|
-
< dir_contents <= hb_timer.flat_map do |t|
|
165
|
-
< dir = Dir.new("#{DATADIR}/#{@data_port}")
|
166
|
-
< files = dir.to_a.map{|d| d.to_i unless d =~ /^\./}.uniq!
|
167
|
-
< dir.close
|
168
|
-
< files.map {|f| [f, Time.parse(t.val).to_f]}
|
169
|
-
< end
|
170
|
-
---
|
171
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|24-29
|
172
|
-
255,259c158
|
173
|
-
< to_payload <= join([dir_contents, nonce]).map do |c, n|
|
174
|
-
< unless server_knows.map{|s| s.file}.include? c.file
|
175
|
-
< [n.ident, c.file, c.time]
|
176
|
-
< end
|
177
|
-
< end
|
178
|
-
---
|
179
|
-
> ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|31-35
|
180
|
-
275d173
|
181
|
-
< ### I am autogenerated. Please do not edit me.
|