bud 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/README +33 -16
  2. data/bin/budplot +42 -65
  3. data/bin/budtimelines +235 -0
  4. data/bin/budvis +24 -122
  5. data/bin/rebl +1 -0
  6. data/docs/README.md +21 -10
  7. data/docs/bfs.md +4 -6
  8. data/docs/c.html +251 -0
  9. data/docs/cheat.md +45 -30
  10. data/docs/deploy.md +26 -26
  11. data/docs/getstarted.md +6 -4
  12. data/docs/visualizations.md +43 -31
  13. data/examples/chat/chat.rb +4 -9
  14. data/examples/chat/chat_server.rb +1 -8
  15. data/examples/deploy/deploy_ip_port +1 -0
  16. data/examples/deploy/keys.rb +5 -0
  17. data/examples/deploy/tokenring-ec2.rb +9 -9
  18. data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
  19. data/examples/deploy/tokenring-thread.rb +15 -0
  20. data/examples/deploy/tokenring.rb +25 -17
  21. data/lib/bud/aggs.rb +87 -25
  22. data/lib/bud/bud_meta.rb +48 -31
  23. data/lib/bud/bust/bust.rb +16 -15
  24. data/lib/bud/collections.rb +207 -232
  25. data/lib/bud/depanalysis.rb +1 -0
  26. data/lib/bud/deploy/countatomicdelivery.rb +8 -20
  27. data/lib/bud/deploy/deployer.rb +16 -16
  28. data/lib/bud/deploy/ec2deploy.rb +34 -35
  29. data/lib/bud/deploy/forkdeploy.rb +90 -0
  30. data/lib/bud/deploy/threaddeploy.rb +38 -0
  31. data/lib/bud/graphs.rb +103 -199
  32. data/lib/bud/joins.rb +190 -41
  33. data/lib/bud/monkeypatch.rb +84 -0
  34. data/lib/bud/rebl.rb +8 -1
  35. data/lib/bud/rewrite.rb +152 -49
  36. data/lib/bud/server.rb +1 -0
  37. data/lib/bud/state.rb +24 -10
  38. data/lib/bud/storage/dbm.rb +170 -0
  39. data/lib/bud/storage/tokyocabinet.rb +5 -1
  40. data/lib/bud/stratify.rb +6 -7
  41. data/lib/bud/viz.rb +31 -17
  42. data/lib/bud/viz_util.rb +204 -0
  43. data/lib/bud.rb +271 -244
  44. data/lib/bud.rb.orig +806 -0
  45. metadata +43 -22
  46. data/docs/bfs.raw +0 -251
  47. data/docs/diffs +0 -181
  48. data/examples/basics/out +0 -1103
  49. data/examples/basics/out.new +0 -856
  50. data/lib/bud/deploy/localdeploy.rb +0 -53
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bud
3
3
  version: !ruby/object:Gem::Version
4
- hash: 25
4
+ hash: 23
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 0
9
- - 3
10
- version: 0.0.3
9
+ - 4
10
+ version: 0.0.4
11
11
  platform: ruby
12
12
  authors:
13
13
  - Peter Alvaro
@@ -18,7 +18,7 @@ autorequire:
18
18
  bindir: bin
19
19
  cert_chain: []
20
20
 
21
- date: 2011-04-07 00:00:00 -07:00
21
+ date: 2011-06-01 00:00:00 -07:00
22
22
  default_executable: rebl
23
23
  dependencies:
24
24
  - !ruby/object:Gem::Dependency
@@ -64,7 +64,7 @@ dependencies:
64
64
  type: :runtime
65
65
  version_requirements: *id003
66
66
  - !ruby/object:Gem::Dependency
67
- name: i18n
67
+ name: getopt
68
68
  prerelease: false
69
69
  requirement: &id004 !ruby/object:Gem::Requirement
70
70
  none: false
@@ -78,7 +78,7 @@ dependencies:
78
78
  type: :runtime
79
79
  version_requirements: *id004
80
80
  - !ruby/object:Gem::Dependency
81
- name: msgpack
81
+ name: i18n
82
82
  prerelease: false
83
83
  requirement: &id005 !ruby/object:Gem::Requirement
84
84
  none: false
@@ -92,7 +92,7 @@ dependencies:
92
92
  type: :runtime
93
93
  version_requirements: *id005
94
94
  - !ruby/object:Gem::Dependency
95
- name: nestful
95
+ name: json
96
96
  prerelease: false
97
97
  requirement: &id006 !ruby/object:Gem::Requirement
98
98
  none: false
@@ -106,7 +106,7 @@ dependencies:
106
106
  type: :runtime
107
107
  version_requirements: *id006
108
108
  - !ruby/object:Gem::Dependency
109
- name: ParseTree
109
+ name: msgpack
110
110
  prerelease: false
111
111
  requirement: &id007 !ruby/object:Gem::Requirement
112
112
  none: false
@@ -120,7 +120,7 @@ dependencies:
120
120
  type: :runtime
121
121
  version_requirements: *id007
122
122
  - !ruby/object:Gem::Dependency
123
- name: ruby-graphviz
123
+ name: nestful
124
124
  prerelease: false
125
125
  requirement: &id008 !ruby/object:Gem::Requirement
126
126
  none: false
@@ -134,7 +134,7 @@ dependencies:
134
134
  type: :runtime
135
135
  version_requirements: *id008
136
136
  - !ruby/object:Gem::Dependency
137
- name: ruby2ruby
137
+ name: ParseTree
138
138
  prerelease: false
139
139
  requirement: &id009 !ruby/object:Gem::Requirement
140
140
  none: false
@@ -148,7 +148,7 @@ dependencies:
148
148
  type: :runtime
149
149
  version_requirements: *id009
150
150
  - !ruby/object:Gem::Dependency
151
- name: sexp_path
151
+ name: ruby-graphviz
152
152
  prerelease: false
153
153
  requirement: &id010 !ruby/object:Gem::Requirement
154
154
  none: false
@@ -162,7 +162,7 @@ dependencies:
162
162
  type: :runtime
163
163
  version_requirements: *id010
164
164
  - !ruby/object:Gem::Dependency
165
- name: superators
165
+ name: ruby2ruby
166
166
  prerelease: false
167
167
  requirement: &id011 !ruby/object:Gem::Requirement
168
168
  none: false
@@ -176,7 +176,7 @@ dependencies:
176
176
  type: :runtime
177
177
  version_requirements: *id011
178
178
  - !ruby/object:Gem::Dependency
179
- name: syntax
179
+ name: sexp_path
180
180
  prerelease: false
181
181
  requirement: &id012 !ruby/object:Gem::Requirement
182
182
  none: false
@@ -190,7 +190,7 @@ dependencies:
190
190
  type: :runtime
191
191
  version_requirements: *id012
192
192
  - !ruby/object:Gem::Dependency
193
- name: tokyocabinet
193
+ name: superators
194
194
  prerelease: false
195
195
  requirement: &id013 !ruby/object:Gem::Requirement
196
196
  none: false
@@ -204,7 +204,7 @@ dependencies:
204
204
  type: :runtime
205
205
  version_requirements: *id013
206
206
  - !ruby/object:Gem::Dependency
207
- name: uuid
207
+ name: syntax
208
208
  prerelease: false
209
209
  requirement: &id014 !ruby/object:Gem::Requirement
210
210
  none: false
@@ -217,6 +217,20 @@ dependencies:
217
217
  version: "0"
218
218
  type: :runtime
219
219
  version_requirements: *id014
220
+ - !ruby/object:Gem::Dependency
221
+ name: uuid
222
+ prerelease: false
223
+ requirement: &id015 !ruby/object:Gem::Requirement
224
+ none: false
225
+ requirements:
226
+ - - ">="
227
+ - !ruby/object:Gem::Version
228
+ hash: 3
229
+ segments:
230
+ - 0
231
+ version: "0"
232
+ type: :runtime
233
+ version_requirements: *id015
220
234
  description: A prototype of the Bloom distributed programming language, as a Ruby DSL.
221
235
  email:
222
236
  - bloomdevs@gmail.com
@@ -224,6 +238,7 @@ executables:
224
238
  - rebl
225
239
  - budplot
226
240
  - budvis
241
+ - budtimelines
227
242
  extensions: []
228
243
 
229
244
  extra_rdoc_files: []
@@ -239,31 +254,36 @@ files:
239
254
  - lib/bud/deploy/countatomicdelivery.rb
240
255
  - lib/bud/deploy/deployer.rb
241
256
  - lib/bud/deploy/ec2deploy.rb
242
- - lib/bud/deploy/localdeploy.rb
257
+ - lib/bud/deploy/forkdeploy.rb
258
+ - lib/bud/deploy/threaddeploy.rb
243
259
  - lib/bud/errors.rb
244
260
  - lib/bud/graphs.rb
245
261
  - lib/bud/joins.rb
262
+ - lib/bud/monkeypatch.rb
246
263
  - lib/bud/rebl.rb
247
264
  - lib/bud/rewrite.rb
248
265
  - lib/bud/rtrace.rb
249
266
  - lib/bud/server.rb
250
267
  - lib/bud/state.rb
268
+ - lib/bud/storage/dbm.rb
251
269
  - lib/bud/storage/tokyocabinet.rb
252
270
  - lib/bud/storage/zookeeper.rb
253
271
  - lib/bud/stratify.rb
254
272
  - lib/bud/viz.rb
273
+ - lib/bud/viz_util.rb
255
274
  - lib/bud.rb
275
+ - lib/bud.rb.orig
256
276
  - bin/budplot
277
+ - bin/budtimelines
257
278
  - bin/budvis
258
279
  - bin/rebl
259
280
  - docs/bfs.md
260
- - docs/bfs.raw
261
281
  - docs/bfs_arch.png
262
282
  - docs/bloom-loop.png
263
283
  - docs/bust.md
284
+ - docs/c.html
264
285
  - docs/cheat.md
265
286
  - docs/deploy.md
266
- - docs/diffs
267
287
  - docs/getstarted.md
268
288
  - docs/intro.md
269
289
  - docs/modules.md
@@ -273,8 +293,6 @@ files:
273
293
  - docs/ruby_hooks.md
274
294
  - docs/visualizations.md
275
295
  - examples/basics/hello.rb
276
- - examples/basics/out
277
- - examples/basics/out.new
278
296
  - examples/basics/paths.rb
279
297
  - examples/bust/bustclient-example.rb
280
298
  - examples/bust/bustinspector.html
@@ -284,8 +302,11 @@ files:
284
302
  - examples/chat/chat_protocol.rb
285
303
  - examples/chat/chat_server.rb
286
304
  - examples/chat/README.md
305
+ - examples/deploy/deploy_ip_port
306
+ - examples/deploy/keys.rb
287
307
  - examples/deploy/tokenring-ec2.rb
288
- - examples/deploy/tokenring-local.rb
308
+ - examples/deploy/tokenring-fork.rb
309
+ - examples/deploy/tokenring-thread.rb
289
310
  - examples/deploy/tokenring.rb
290
311
  - examples/README
291
312
  - README
@@ -302,7 +323,7 @@ require_paths:
302
323
  required_ruby_version: !ruby/object:Gem::Requirement
303
324
  none: false
304
325
  requirements:
305
- - - ">="
326
+ - - ~>
306
327
  - !ruby/object:Gem::Version
307
328
  hash: 57
308
329
  segments:
data/docs/bfs.raw DELETED
@@ -1,251 +0,0 @@
1
- # BFS: A distributed file system in Bloom
2
-
3
- In this document we'll use what we've learned to build a piece of systems software using Bloom. The libraries that ship with Bud provide many of the building blocks we'll need to create a distributed,
4
- ``chunked'' file system in the style of the Google File System (GFS):
5
-
6
- * a [key-value store](https://github.com/bloom-lang/bud-sandbox/blob/master/kvs/kvs.rb) (KVS)
7
- * [nonce generation](https://github.com/bloom-lang/bud-sandbox/blob/master/ordering/nonce.rb)
8
- * a [heartbeat protocol](https://github.com/bloom-lang/bud-sandbox/blob/master/heartbeat/heartbeat.rb)
9
-
10
- ## High-level architecture
11
-
12
- ![BFS Architecture](bfs_arch.png?raw=true)
13
-
14
- BFS implements a chunked, distributed file system (mostly) in the Bloom
15
- language. BFS is architecturally based on [BOOM-FS](http://db.cs.berkeley.edu/papers/eurosys10-boom.pdf), which is itself based on
16
- the Google File System (GFS). As in GFS, a single master node manages
17
- file system metadata, while data blocks are replicated and stored on a large
18
- number of storage nodes. Writing or reading data involves a multi-step
19
- protocol in which clients interact with the master, retrieving metadata and
20
- possibly changing state, then interact with storage nodes to read or write
21
- chunks. Background jobs running on the master will contact storage nodes to
22
- orchestrate chunk migrations, during which storage nodes communicate with
23
- other storage nodes. As in BOOM-FS, the communication protocols and the data
24
- channel used for bulk data transfer between clients and datanodes and between
25
- datanodes is written outside Bloom (in Ruby).
26
-
27
- ## [Basic File System](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/fs_master.rb)
28
-
29
- Before we worry about any of the details of distribution, we need to implement the basic file system metadata operations: _create_, _remove_, _mkdir_ and _ls_.
30
- There are many choices for how to implement these operations, and it makes sense to keep them separate from the (largely orthogonal) distributed file system logic.
31
- That way, it will be possible later to choose a different implementation of the metadata operations without impacting the rest of the system.
32
- Another benefit of modularizing the metadata logic is that it can be independently tested and debugged. We want to get the core of the file system
33
- working correctly before we even send a whisper over the network, let alone add any complex features.
34
-
35
- ### Protocol
36
-
37
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|12-20
38
-
39
- We create an input interface for each of the operations, and a single output interface for the return for any operation: given a request id, __status__ is a boolean
40
- indicating whether the request succeeded, and __data__ may contain return values (e.g., _fsls_ should return an array containing the array contents).
41
-
42
- ### Implementation
43
-
44
- We already have a library that provides an updateable flat namespace: the key-value store. We can easily implement the tree structure of a file system over a key-value store
45
- in the following way:
46
-
47
- 1. keys are paths
48
- 2. directories have arrays containing child entries (base names)
49
- 3. files values are their contents
50
-
51
- <!--- (**JMH**: I find it a bit confusing how you toggle from the discussion above to this naive file-storage design here. Can you warn us a bit more clearly that this is a starting point focused on metadata, with (3) being a strawman for data storage that is intended to be overriden later?)
52
- --->
53
- Note that (3) is a strawman: it will cease to apply when we implement chunked storage later. It is tempting, however, to support (3) so that the resulting program is a working
54
- standalone file system.
55
-
56
- We begin our implementation of a KVS-backed metadata system in the following way:
57
-
58
-
59
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|33-36
60
-
61
- If we wanted to replicate the master node's metadata we could consider mixing in a replicated KVS implementation instead of __BasicKVS__ -- but more on that later.
62
-
63
- ### Directory Listing
64
-
65
- The directory listing operation is implemented by a simple block of Bloom statements:
66
-
67
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|51-57
68
-
69
- If we get a __fsls__ request, probe the key-value store for the requested by projecting _reqid_, _path_ from the __fsls__ tuple into __kvget__. If the given path
70
- is a key, __kvget_response__ will contain a tuple with the same _reqid_, and the join on the second line will succeed. In this case, we insert the value
71
- associated with that key into __fsret__. Otherwise, the third rule will fire, inserting a failure tuple into __fsret__.
72
-
73
-
74
- ### Mutation
75
-
76
- The logic for file and directory creation and deletion follow a similar logic with regard to the parent directory:
77
-
78
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|61-71
79
-
80
- Unlike a directory listing, however, these operations change the state of the file system. In general, any state change will involve
81
- carrying out two mutating operations to the key-value store atomically:
82
-
83
- 1. update the value (child array) associated with the parent directory entry
84
- 2. update the key-value pair associated with the object in question (a file or directory being created or destroyed).
85
-
86
- The following Bloom code carries this out:
87
-
88
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|73-73
89
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|80-108
90
-
91
-
92
- <!--- (**JMH**: This next sounds awkward. You *do* take care: by using <= and understanding the atomicity of timesteps in Bloom. I think what you mean to say is that Bloom's atomic timestep model makes this easy compared to ... something.)
93
- Note that we need not take any particular care to ensure that the two inserts into __kvput__ occur together atomically. Because both statements use the synchronous
94
- -->
95
- Note that because both inserts into the __kvput__ collection use the synchronous operator (`<=`), we know that they will occur together in the same fixpoint computation or not at all.
96
- Therefore we need not be concerned with explicitly sequencing the operations (e.g., ensuring that the directory entries is created _after_ the file entry) to deal with concurrency:
97
- there can be no visible state of the database in which only one of the operations has succeeded.
98
-
99
- If the request is a deletion, we need some additional logic to enforce the constraint that only an empty directory may be removed:
100
-
101
-
102
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|74-78
103
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|110-115
104
-
105
-
106
- Recall that when we created KVSFS we mixed in __TimestepNonce__, one of the nonce libraries. While we were able to use the _reqid_ field from the input operation as a unique identifier
107
- for one of our KVS operations, we need a fresh, unique request id for the second KVS operation in the atomic pair described above. By joining __nonce__, we get
108
- an identifier that is unique to this timestep.
109
-
110
-
111
- ## [File Chunking](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/chunking.rb)
112
-
113
- Now that we have a module providing a basic file system, we can extend it to support chunked storage of file contents. The metadata master will contain, in addition to the KVS
114
- structure for directory information, a relation mapping a set of chunk identifiers to each file
115
-
116
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|26-26
117
-
118
- and relations associating a chunk with a set of datanodes that host a replica of the chunk.
119
-
120
- ==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|12-12
121
-
122
- (**JMH**: ambiguous reference ahead "these latter")
123
- The latter (defined in __HBMaster__) is soft-state, kept up to data by heartbeat messages from datanodes (described in the next section).
124
-
125
- To support chunked storage, we add a few metadata operations to those already defined by FSProtocol:
126
-
127
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|6-16
128
-
129
- * __fschunklist__ returns the set of chunks belonging to a given file.
130
- * __fschunklocations__ returns the set of datanodes in possession of a given chunk.
131
- * __fsaddchunk__ returns a new chunkid for appending to an existing file, guaranteed to be higher than any existing chunkids for that file, and a list of candidate datanodes that can store a replica of the new chunk.
132
-
133
- We continue to use __fsret__ for return values.
134
-
135
- ### Lookups
136
-
137
- Lines 34-44 are a similar pattern to what we saw in the basic FS: whenever we get a __fschunklist__ or __fsaddchunk__ request, we must first ensure that the given file
138
- exists, and error out if not. If it does, and the operation was __fschunklist__, we join the metadata relation __chunk__ and return the set of chunks owned
139
- by the given (existent) file:
140
-
141
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|47-49
142
-
143
- ### Add chunk
144
-
145
- If it was a __fsaddchunk__ request, we need to generate a unique id for a new chunk and return a list of target datanodes. We reuse __TimestepNonce__ to do the former, and join a relation
146
- called __available__ that is exported by __HBMaster__ (described in the next section) for the latter:
147
-
148
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|69-76
149
-
150
- Finally, it was a __fschunklocations__ request, we have another possible error scenario, because the nodes associated with chunks are a part of our soft state. Even if the file
151
- exists, it may not be the case that we have fresh information in our cache about what datanodes own a replica of the given chunk:
152
-
153
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|54-58
154
-
155
- Otherwise, __chunk_cache__ has information about the given chunk, which we may return to the client:
156
-
157
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|61-64
158
-
159
-
160
- ## Datanodes and Heartbeats
161
-
162
- ### [Datanode](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/datanode.rb)
163
-
164
- A datanode runs both Bud code (to support the heartbeat and control protocols) and pure Ruby (to support the data transfer protocol). A datanode's main job is keeping the master
165
- aware of it existence and its state, and participating when necessary in data pipelines to read or write chunk data to and from its local storage.
166
-
167
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|11-15
168
-
169
- By mixing in HeartbeatAgent, the datanode includes the machinery necessary to regularly send status messages to the master. __HeartbeatAgent__ provides an input interface
170
- called __payload__ that allows an agent to optionally include additional information in heartbeat messages: in our case, we wish to include state deltas which ensure that
171
- the master has an accurate view of the set of chunks owned by the datanode.
172
-
173
- When a datanode is constructed, it takes a port at which the embedded data protocol server will listen, and starts the server in the background:
174
-
175
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|61-62
176
-
177
- At regular intervals, a datanode polls its local chunk directory (which is independently written to by the data protocol):
178
-
179
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|26-31
180
-
181
- We update the payload that we send to the master if our recent poll found files that we don't believe the master knows about:
182
-
183
-
184
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|33-40
185
-
186
- Our view of what the master ``knows'' about reflects our local cache of acknowledgement messages from the master. This logic is defined in __HBMaster__.
187
-
188
- ### [Heartbeat master logic](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/hb_master.rb)
189
-
190
- On the master side of heartbeats, we always send an ack when we get a heartbeat, so that the datanode doesn't need to keep resending its
191
- payload of local chunks:
192
-
193
- ==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|30-32
194
-
195
- At the same time, we use the Ruby _flatmap_ method to flatten the array of chunks in the heartbeat payload into a set of tuples, which we
196
- associate with the heartbeating datanode and the time of receipt in __chunk_cache__:
197
-
198
- ==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|22-28
199
-
200
- We periodically garbage-collect this cached, removing entries for datanodes from whom we have not received a heartbeat in a configurable amount of time.
201
- __last_heartbeat__ is an output interface provided by the __HeartbeatAgent__ module, and contains the most recent, non-stale heartbeat contents:
202
-
203
- ==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|34-36
204
-
205
-
206
- ## [BFS Client](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/bfs_client.rb)
207
-
208
- One of the most complicated parts of the basic GFS design is the client component. To minimize load on the centralized master, we take it off the critical
209
- path of file transfers. The client therefore needs to pick up this work.
210
-
211
- We won't spend too much time on the details of the client code, as it is nearly all _plain old Ruby_. The basic idea is:
212
-
213
- 1. Pure metadata operations
214
- * _mkdir_, _create_, _ls_, _rm_
215
- * Send the request to the master and inform the caller of the status.
216
- * If _ls_, return the directory listing to the caller.
217
- 2. Append
218
- * Send a __fsaddchunk__ request to the master, which should return a new chunkid and a list of datanodes.
219
- * Read a chunk worth of data from the input stream.
220
- * Connect to the first datanode in the list. Send a header containing the chunkid and the remaining datanodes.
221
- * Stream the file contents. The target datanode will then ``play client'' and continue the pipeline to the next datanode, and so on.
222
- 2. Read
223
- * Send a __getchunks__ request to the master for the given file. It should return the list of chunks owned by the file.
224
- * For each chunk,
225
- * Send a __fschunklocations__ request to the master, which should return a list of datanodes in possession of the chunk (returning a list allows the client to perform retries without more communication with the master, should some of the datanodes fail).
226
- * Connect to a datanode from the list and stream the chunk to a local buffer.
227
- * As chunks become available, stream them to the caller.
228
-
229
-
230
- ## [Data transfer protocol](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/data_protocol.rb)
231
-
232
- The data transfer protocol comprises a set of support functions for the bulk data transfer protocol whose use is described in the previous section.
233
- Because it is _plain old Ruby_ it is not as interesting as the other modules. It provides:
234
-
235
- * The TCP server code that runs at each datanode, which parses headers and writes stream data to the local FS (these files are later detected by the directory poll).
236
- * Client API calls to connect to datanodes and stream data. Datanodes also use this protocol to pipeline chunks to downstream datanodes.
237
- * Master API code invoked by a background process to replicate chunks from datanodes to other datanodes, when the replication factor for a chunk is too low.
238
-
239
- ## [Master background process](https://github.com/bloom-lang/bud-sandbox/blob/master/bfs/background.rb)
240
-
241
- So far, we have implemented the BFS master as a strictly reactive system: when clients make requests, it queries and possibly updates local state.
242
- To maintain the durability requirement that `REP_FACTOR` copies of every chunk are stored on distinct nodes, the master must be an active system
243
- that maintains a near-consistent view of global state, and takes steps to correct violated requirements.
244
-
245
- __chunk_cache__ is the master's view of datanode state, maintained as described by collecting and pruning heartbeat messages.
246
-
247
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/background.rb|24-27
248
-
249
- After defining some helper aggregates (__chunk_cnts_chunk__ or replica count by chunk, and __chunk_cnt_host__ or datanode fill factor),
250
-
251
- ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/background.rb|29-36
data/docs/diffs DELETED
@@ -1,181 +0,0 @@
1
- 24c24
2
- < channel used for communication (**JMH**: "bulk data transfer"?) between clients and datanodes and between
3
- ---
4
- > channel used for communication between clients and datanodes and between
5
- 35,43c35
6
- < module FSProtocol
7
- < state do
8
- < interface input, :fsls, [:reqid, :path]
9
- < interface input, :fscreate, [] => [:reqid, :name, :path, :data]
10
- < interface input, :fsmkdir, [] => [:reqid, :name, :path]
11
- < interface input, :fsrm, [] => [:reqid, :name, :path]
12
- < interface output, :fsret, [:reqid, :status, :data]
13
- < end
14
- < end
15
- ---
16
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|12-20
17
- 57,58d48
18
- < (**JMH**: I find it a bit confusing how you toggle from the discussion above to this naive file-storage design here. Can you warn us a bit more clearly that this is a starting point focused on metadata, with a strawman for data storage to be overriden later?)
19
- <
20
- 62,65c52
21
- < module KVSFS
22
- < include FSProtocol
23
- < include BasicKVS
24
- < include TimestepNonce
25
- ---
26
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|33-36
27
- 67c54
28
- < If we wanted to replicate the metadata master (**JMH**: "the master node's metadata"?), we could consider mixing in a replicated KVS implementation instead of __BasicKVS__ -- but more on that later.
29
- ---
30
- > If we wanted to replicate the metadata master, we could consider mixing in a replicated KVS implementation instead of __BasicKVS__ -- but more on that later.
31
- 71c58
32
- < The directory listing operation is implemented by a simple block of Bloom statements:
33
- ---
34
- > The directory listing operation is very simple:
35
- 73,81c60
36
- < bloom :elles do
37
- < kvget <= fsls.map{ |l| [l.reqid, l.path] }
38
- < fsret <= join([kvget_response, fsls], [kvget_response.reqid, fsls.reqid]).map{ |r, i| [r.reqid, true, r.value] }
39
- < fsret <= fsls.map do |l|
40
- < unless kvget_response.map{ |r| r.reqid}.include? l.reqid
41
- < [l.reqid, false, nil]
42
- < end
43
- < end
44
- < end
45
- ---
46
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|51-59
47
- 96d74
48
- < (**JMH**: Transition: "The following Bloom code carries this out for ...")
49
- 98,134c76
50
- < dir_exists = join [check_parent_exists, kvget_response, nonce], [check_parent_exists.reqid, kvget_response.reqid]
51
- <
52
- < check_is_empty <= join([fsrm, nonce]).map{|m, n| [n.ident, m.reqid, terminate_with_slash(m.path) + m.name] }
53
- < kvget <= check_is_empty.map{|c| [c.reqid, c.name] }
54
- < can_remove <= join([kvget_response, check_is_empty], [kvget_response.reqid, check_is_empty.reqid]).map do |r, c|
55
- < [c.reqid, c.orig_reqid, c.name] if r.value.length == 0
56
- < end
57
- <
58
- < fsret <= dir_exists.map do |c, r, n|
59
- < if c.mtype == :rm
60
- < unless can_remove.map{|can| can.orig_reqid}.include? c.reqid
61
- < [c.reqid, false, "directory #{} not empty"]
62
- < end
63
- < end
64
- < end
65
- <
66
- < # update dir entry
67
- < # note that it is unnecessary to ensure that a file is created before its corresponding
68
- < # directory entry, as both inserts into :kvput below will co-occur in the same timestep.
69
- < kvput <= dir_exists.map do |c, r, n|
70
- < if c.mtype == :rm
71
- < if can_remove.map{|can| can.orig_reqid}.include? c.reqid
72
- < [ip_port, c.path, n.ident, r.value.clone.reject{|item| item == c.name}]
73
- < end
74
- < else
75
- < [ip_port, c.path, n.ident, r.value.clone.push(c.name)]
76
- < end
77
- < end
78
- <
79
- < kvput <= dir_exists.map do |c, r, n|
80
- < case c.mtype
81
- < when :mkdir
82
- < [ip_port, terminate_with_slash(c.path) + c.name, c.reqid, []]
83
- < when :create
84
- < [ip_port, terminate_with_slash(c.path) + c.name, c.reqid, "LEAF"]
85
- < end
86
- < end
87
- ---
88
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/fs_master.rb|74-110
89
- 136d77
90
- < (**JMH**: This next sounds awkward. You *do* take care: by using <= and understanding the atomicity of timesteps in Bloom. I think what you mean to say is that Bloom's atomic timestep model makes this easy compared to ... something.)
91
- 150c91
92
- < table :chunk, [:chunkid, :file, :siz]
93
- ---
94
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|26-26
95
- 154c95
96
- < table :chunk_cache, [:node, :chunkid, :time]
97
- ---
98
- > ==https://github.com/bloom-lang/bud-sandbox/raw/5c7734912e900c28087e39b3424a1e0191e13704/bfs/hb_master.rb|12-12
99
- 156d96
100
- < (**JMH**: ambiguous reference ahead "these latter")
101
- 161,171c101
102
- < module ChunkedFSProtocol
103
- < include FSProtocol
104
- <
105
- < state do
106
- < interface :input, :fschunklist, [:reqid, :file]
107
- < interface :input, :fschunklocations, [:reqid, :chunkid]
108
- < interface :input, :fsaddchunk, [:reqid, :file]
109
- < # note that no output interface is defined.
110
- < # we use :fsret (defined in FSProtocol) for output.
111
- < end
112
- < end
113
- ---
114
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|6-16
115
- 185,187c115
116
- < chunk_buffer <= join([fschunklist, kvget_response, chunk], [fschunklist.reqid, kvget_response.reqid], [fschunklist.file, chunk.file]).map{ |l, r, c| [l.reqid, c.chunkid] }
117
- < chunk_buffer2 <= chunk_buffer.group([chunk_buffer.reqid], accum(chunk_buffer.chunkid))
118
- < fsret <= chunk_buffer2.map{ |c| [c.reqid, true, c.chunklist] }
119
- ---
120
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|47-49
121
- 191d118
122
- < (**JMH**: Ambiguous ref "If it")
123
- 195,202c122
124
- < minted_chunk = join([kvget_response, fsaddchunk, available, nonce], [kvget_response.reqid, fsaddchunk.reqid])
125
- < chunk <= minted_chunk.map{ |r, a, v, n| [n.ident, a.file, 0] }
126
- < fsret <= minted_chunk.map{ |r, a, v, n| [r.reqid, true, [n.ident, v.pref_list.slice(0, (REP_FACTOR + 2))]] }
127
- < fsret <= join([kvget_response, fsaddchunk], [kvget_response.reqid, fsaddchunk.reqid]).map do |r, a|
128
- < if available.empty? or available.first.pref_list.length < REP_FACTOR
129
- < [r.reqid, false, "datanode set cannot satisfy REP_FACTOR = #{REP_FACTOR} with [#{available.first.nil? ? "NIL" : available.first.pref_list.inspect}]"]
130
- < end
131
- < end
132
- ---
133
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|69-76
134
- 204d123
135
- < (**JMH**: Ambiguous ref "If it")
136
- 208,212c127
137
- < fsret <= fschunklocations.map do |l|
138
- < unless chunk_cache.map{|c| c.chunkid}.include? l.chunkid
139
- < [l.reqid, false, "no datanodes found for #{l.chunkid} in cc, now #{chunk_cache.length}"]
140
- < end
141
- < end
142
- ---
143
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|54-58
144
- 216,219c131
145
- < chunkjoin = join [fschunklocations, chunk_cache], [fschunklocations.chunkid, chunk_cache.chunkid]
146
- < host_buffer <= chunkjoin.map{|l, c| [l.reqid, c.node] }
147
- < host_buffer2 <= host_buffer.group([host_buffer.reqid], accum(host_buffer.host))
148
- < fsret <= host_buffer2.map{|c| [c.reqid, true, c.hostlist] }
149
- ---
150
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/chunking.rb|61-64
151
- 229,233c141
152
- < module BFSDatanode
153
- < include HeartbeatAgent
154
- < include StaticMembership
155
- < include TimestepNonce
156
- < include BFSHBProtocol
157
- ---
158
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|9-13
159
- 241c149
160
- < @dp_server = DataProtocolServer.new(dataport)
161
- ---
162
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|53-53
163
- 245,250c153
164
- < dir_contents <= hb_timer.flat_map do |t|
165
- < dir = Dir.new("#{DATADIR}/#{@data_port}")
166
- < files = dir.to_a.map{|d| d.to_i unless d =~ /^\./}.uniq!
167
- < dir.close
168
- < files.map {|f| [f, Time.parse(t.val).to_f]}
169
- < end
170
- ---
171
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|24-29
172
- 255,259c158
173
- < to_payload <= join([dir_contents, nonce]).map do |c, n|
174
- < unless server_knows.map{|s| s.file}.include? c.file
175
- < [n.ident, c.file, c.time]
176
- < end
177
- < end
178
- ---
179
- > ==https://github.com/bloom-lang/bud-sandbox/raw/master/bfs/datanode.rb|31-35
180
- 275d173
181
- < ### I am autogenerated. Please do not edit me.