bud 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/README +33 -16
  2. data/bin/budplot +42 -65
  3. data/bin/budtimelines +235 -0
  4. data/bin/budvis +24 -122
  5. data/bin/rebl +1 -0
  6. data/docs/README.md +21 -10
  7. data/docs/bfs.md +4 -6
  8. data/docs/c.html +251 -0
  9. data/docs/cheat.md +45 -30
  10. data/docs/deploy.md +26 -26
  11. data/docs/getstarted.md +6 -4
  12. data/docs/visualizations.md +43 -31
  13. data/examples/chat/chat.rb +4 -9
  14. data/examples/chat/chat_server.rb +1 -8
  15. data/examples/deploy/deploy_ip_port +1 -0
  16. data/examples/deploy/keys.rb +5 -0
  17. data/examples/deploy/tokenring-ec2.rb +9 -9
  18. data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
  19. data/examples/deploy/tokenring-thread.rb +15 -0
  20. data/examples/deploy/tokenring.rb +25 -17
  21. data/lib/bud/aggs.rb +87 -25
  22. data/lib/bud/bud_meta.rb +48 -31
  23. data/lib/bud/bust/bust.rb +16 -15
  24. data/lib/bud/collections.rb +207 -232
  25. data/lib/bud/depanalysis.rb +1 -0
  26. data/lib/bud/deploy/countatomicdelivery.rb +8 -20
  27. data/lib/bud/deploy/deployer.rb +16 -16
  28. data/lib/bud/deploy/ec2deploy.rb +34 -35
  29. data/lib/bud/deploy/forkdeploy.rb +90 -0
  30. data/lib/bud/deploy/threaddeploy.rb +38 -0
  31. data/lib/bud/graphs.rb +103 -199
  32. data/lib/bud/joins.rb +190 -41
  33. data/lib/bud/monkeypatch.rb +84 -0
  34. data/lib/bud/rebl.rb +8 -1
  35. data/lib/bud/rewrite.rb +152 -49
  36. data/lib/bud/server.rb +1 -0
  37. data/lib/bud/state.rb +24 -10
  38. data/lib/bud/storage/dbm.rb +170 -0
  39. data/lib/bud/storage/tokyocabinet.rb +5 -1
  40. data/lib/bud/stratify.rb +6 -7
  41. data/lib/bud/viz.rb +31 -17
  42. data/lib/bud/viz_util.rb +204 -0
  43. data/lib/bud.rb +271 -244
  44. data/lib/bud.rb.orig +806 -0
  45. metadata +43 -22
  46. data/docs/bfs.raw +0 -251
  47. data/docs/diffs +0 -181
  48. data/examples/basics/out +0 -1103
  49. data/examples/basics/out.new +0 -856
  50. data/lib/bud/deploy/localdeploy.rb +0 -53
data/lib/bud/joins.rb CHANGED
@@ -3,12 +3,12 @@ module Bud
3
3
  attr_accessor :rels, :origrels, :origpreds # :nodoc: all
4
4
  attr_reader :hash_tables # :nodoc: all
5
5
 
6
- def initialize(rellist, bud_instance, preds=nil) # :nodoc: all
6
+ def initialize(rellist, bud_instance, preds=[]) # :nodoc: all
7
7
  @schema = []
8
- otherpreds = nil
9
8
  @origpreds = preds
10
9
  @bud_instance = bud_instance
11
10
  @localpreds = nil
11
+ @hashpreds = nil
12
12
 
13
13
  # if any elements on rellist are BudJoins, suck up their contents
14
14
  tmprels = []
@@ -25,7 +25,7 @@ module Bud
25
25
 
26
26
  # recurse to form a tree of binary BudJoins
27
27
  @rels = [rellist[0]]
28
- @rels << (rellist.length == 2 ? rellist[1] : BudJoin.new(rellist[1..rellist.length-1], @bud_instance, nil))
28
+ @rels << (rellist.length == 2 ? rellist[1] : BudJoin.new(rellist[1..rellist.length-1], @bud_instance))
29
29
  # derive schema: one column for each table.
30
30
  # duplicated inputs get distinguishing numeral
31
31
  @schema = []
@@ -39,7 +39,7 @@ module Bud
39
39
  memo
40
40
  end
41
41
 
42
- setup_preds(preds) unless preds.nil? or preds.empty?
42
+ setup_preds(preds) unless preds.empty?
43
43
  setup_state
44
44
  end
45
45
 
@@ -76,7 +76,7 @@ module Bud
76
76
  # similar to <tt>SELECT * FROM ... WHERE...</tt> block in SQL.
77
77
  public
78
78
  def flatten(*preds)
79
- setup_preds(preds) unless preds.nil? or preds.size == 0
79
+ setup_preds(preds) unless preds.empty?
80
80
  flat_schema = @rels.map{|r| r.schema}.flatten(1)
81
81
  dupfree_schema = []
82
82
  # while loop here (inefficiently) ensures no collisions
@@ -127,8 +127,8 @@ module Bud
127
127
 
128
128
  methods.each do |left_rel|
129
129
  methods.each do |right_rel|
130
- next if (mode == :delta and left_rel == :storage and right_rel == :storage)
131
- if @localpreds.nil? or @localpreds.empty?
130
+ next if (mode == :both and left_rel == :storage and right_rel == :storage)
131
+ if @hashpreds.nil? or @hashpreds.empty?
132
132
  nestloop_join(left_rel, right_rel, &block)
133
133
  else
134
134
  hash_join(left_rel, right_rel, &block)
@@ -138,6 +138,156 @@ module Bud
138
138
  tick_hash_deltas
139
139
  end
140
140
 
141
+ # given a * expression over n collections, form all combinations of items
142
+ # subject to an array of predicates, pred
143
+ # currently supports two options for equijoin predicates:
144
+ # general form: an array of arrays capturing a conjunction of equiv. classes
145
+ # [[table1.col1, table2.col2, table3.col3], [table1.col2, table2.col3]]
146
+ # common form: a hash capturing equality of a column on left with one on right.
147
+ # :col1 => :col2 (same as lefttable.col1 => righttable.col2)
148
+ public
149
+ def pairs(*preds, &blk)
150
+ @origpreds = preds
151
+ setup_preds(preds) unless preds.empty?
152
+ # given new preds, the state for the join will be different. set it up again.
153
+ setup_state if self.class <= Bud::BudJoin
154
+ blk.nil? ? self : map(&blk)
155
+ end
156
+
157
+ alias combos pairs
158
+
159
+ # the natural join: given a * expression over n collections, form all
160
+ # combinations of items that have the same values in matching fields
161
+ public
162
+ def matches(&blk)
163
+ preds = BudJoin::natural_preds(@bud_instance, @origrels)
164
+ pairs(*preds, &blk)
165
+ end
166
+
167
+ # given a * expression over 2 collections, form all combinations of items
168
+ # that satisfy the predicates +preds+, and project only onto the attributes
169
+ # of the first collection
170
+ public
171
+ def lefts(*preds, &blk)
172
+ setup_preds(preds) unless preds.empty?
173
+ # given new preds, the state for the join will be different. set it up again.
174
+ setup_state if self.class <= Bud::BudJoin
175
+ map{ |l,r| blk.nil? ? l : blk.call(l) }
176
+ end
177
+
178
+ # given a * expression over 2 collections, form all combinations of items
179
+ # that satisfy the predicates +preds+, and project only onto the attributes
180
+ # of the second item
181
+ public
182
+ def rights(*preds, &blk)
183
+ setup_preds(preds) unless preds.empty?
184
+ # given new preds, the state for the join will be different. set it up again.
185
+ setup_state if self.class <= Bud::BudJoin
186
+ map{ |l,r| blk.nil? ? r : blk.call(r) }
187
+ end
188
+
189
+ # given a * expression over 2 collections, form all combos of items that
190
+ # satisfy +preds+, and for any item from the 1st collection that has no
191
+ # matches in the 2nd, nil-pad it and include it in the output.
192
+ public
193
+ def outer(*preds)
194
+ @origpreds = preds
195
+ @localpreds = disambiguate_preds(preds)
196
+ self.extend(Bud::BudOuterJoin)
197
+ map
198
+ end
199
+
200
+ # extract predicates on rellist[0] and recurse to right side with remainder
201
+ protected
202
+ def setup_preds(preds) # :nodoc: all
203
+ allpreds = disambiguate_preds(preds)
204
+ allpreds = canonicalize_localpreds(@rels, allpreds)
205
+ # check for refs to collections that aren't being joined, Issue 191
206
+ unless @rels[1].class <= Bud::BudJoin
207
+ tabnames = @rels.map{ |r| r.tabname }
208
+ allpreds.each do |p|
209
+ unless tabnames.include? p[0][0] and tabnames.include? p[1][0]
210
+ raise Bud::CompileError, "illegal predicate: collection #{} is not being joined"
211
+ end
212
+ end
213
+ end
214
+ @hashpreds = allpreds.reject { |p| p[0][0] != @rels[0].tabname or p[1][0] == @rels[0].tabname }
215
+ @localpreds = @hashpreds
216
+ @localpreds += allpreds.map do |p|
217
+ p if p[0][0] == p[1][0] and (p[0][0] == @rels[0].tabname or p[0][0] == @rels[1].tabname)
218
+ end.compact
219
+ otherpreds = allpreds - @localpreds
220
+ unless otherpreds.empty?
221
+ unless @rels[1].class <= Bud::BudJoin
222
+ raise Bud::CompileError, "join predicates don't match tables being joined: #{otherpreds.inspect}"
223
+ end
224
+ @rels[1].setup_preds(otherpreds)
225
+ end
226
+ end
227
+
228
+ protected
229
+ def disambiguate_preds(preds) # :nodoc: all
230
+ if preds.size == 1 and preds[0].class <= Hash
231
+ predarray = preds[0].map do |k,v|
232
+ if k.class != v.class
233
+ raise Bud::CompileError, "inconsistent attribute ref style #{k.inspect} => #{v.inspect}"
234
+ elsif k.class <= Array
235
+ [k,v]
236
+ elsif k.class <= Symbol
237
+ if @origrels and @origrels.length == 2
238
+ [find_attr_match(k, @origrels[0]), find_attr_match(v, @origrels[1])]
239
+ else
240
+ [find_attr_match(k), find_attr_match(v)]
241
+ end
242
+ else
243
+ raise Bud::CompileError, "invalid attribute ref in #{k.inspect} => #{v.inspect}"
244
+ end
245
+ end
246
+ return decomp_preds(*predarray)
247
+ else
248
+ return decomp_preds(*preds)
249
+ end
250
+ end
251
+
252
+ # find element in @origrels that contains this +aname+ method
253
+ # if +rel+ is non-nil, only check that collection.
254
+ # after found, return the result of invoking +aname+ from chosen collection
255
+ protected
256
+ def find_attr_match(aname, rel=nil) # :nodoc: all
257
+ dorels = (rel.nil? ? @origrels : [rel])
258
+ match = nil
259
+ dorels.each do |r|
260
+ match ||= r if r.respond_to?(aname)
261
+ if r.respond_to?(aname) and match != r
262
+ raise Bud::CompileError, "ambiguous attribute :#{aname} in both #{match.tabname} and #{r.tabname}"
263
+ end
264
+ end
265
+ if match.nil?
266
+ raise Bud::CompileError, "attribute :#{aname} not found in any of #{dorels.map{|t| t.tabname}.inspect}"
267
+ end
268
+ match.send(aname)
269
+ end
270
+
271
+ protected
272
+ def decomp_preds(*preds) # :nodoc:all
273
+ # decompose each pred into a binary pred
274
+ return nil if preds.empty? or preds == [nil]
275
+ newpreds = []
276
+ preds.each do |p|
277
+ p.each_with_index do |c, i|
278
+ newpreds << [p[i], p[i+1]] unless p[i+1].nil?
279
+ end
280
+ end
281
+ newpreds
282
+ end
283
+
284
+ protected
285
+ def canonicalize_localpreds(rel_list, preds) # :nodoc:all
286
+ retval = preds.map do |p|
287
+ p[1][0] == rel_list[0].tabname ? p.reverse : p
288
+ end
289
+ end
290
+
141
291
  public
142
292
  def each_from_sym(buf_syms, &block) # :nodoc: all
143
293
  buf_syms.each do |s|
@@ -154,8 +304,16 @@ module Bud
154
304
  # check remainder of the predicates
155
305
  @localpreds.each do |pred|
156
306
  next if skips.include? pred
157
- r_offset, s_index, s_offset = join_offsets(pred)
158
- if r[r_offset] != s[s_index][s_offset]
307
+ vals = []
308
+ (0..1).each do |i|
309
+ if pred[i][0] == @rels[0].tabname
310
+ vals[i] = r[pred[i][1] ]
311
+ else
312
+ ix, off = join_offset(pred[i])
313
+ vals[i] = s[ix][off]
314
+ end
315
+ end
316
+ if vals[0] != vals[1]
159
317
  retval = false
160
318
  break
161
319
  end
@@ -175,28 +333,27 @@ module Bud
175
333
  end
176
334
 
177
335
  private
178
- # calculate the attribute position for the left table in the join ("left_offset")
179
- # the right table may itself be a nested tuple from a join, so calculate
180
- # the tuple offset ("right_subtuple") and the attribute position within it
181
- # ("right_offset")
182
- def join_offsets(pred)
183
- right_entry = pred[1]
184
- right_name, right_offset = right_entry[0], right_entry[1]
185
- left_entry = pred[0]
186
- left_name, left_offset = left_entry[0], left_entry[1]
187
-
188
- # determine which subtuple of right collection contains the table
189
- # referenced in RHS of pred. note that right collection doesn't contain the
190
- # first entry in rels, which is the left collection
191
- right_subtuple = 0
336
+ # calculate the position for a field in the result of a join:
337
+ # the tuple offset ("subtuple") and the attribute position within it
338
+ # ("offset")
339
+ def join_offset(entry)
340
+ name, offset = entry[0], entry[1]
341
+
342
+ # determine which subtuple of the collection contains the table
343
+ # referenced in entry. note that origrels[0] is a base table
344
+ # on the left of the join, hence we shouldn't be calling join_offset on it
345
+ subtuple = 0
346
+ if origrels[0].tabname == entry[0]
347
+ raise BudError, "join_offset called on the result of a single collection #{entry[0]}"
348
+ end
192
349
  origrels[1..origrels.length].each_with_index do |t,i|
193
- if t.tabname == pred[1][0]
194
- right_subtuple = i
350
+ if t.tabname == entry[0]
351
+ subtuple = i
195
352
  break
196
353
  end
197
354
  end
198
355
 
199
- return left_offset, right_subtuple, right_offset
356
+ return subtuple, offset
200
357
  end
201
358
 
202
359
  def tick_hash_deltas
@@ -204,7 +361,7 @@ module Bud
204
361
  return if @hash_tables.nil?
205
362
  (0..1).each do |i|
206
363
  @hash_tables[i][:storage].merge!(@hash_tables[i][:delta]) do |k,l,r|
207
- l+r
364
+ l+r
208
365
  end
209
366
  @hash_tables[i][:delta] = {}
210
367
  end
@@ -213,7 +370,9 @@ module Bud
213
370
  # semi-naive symmetric hash join on first predicate
214
371
  private
215
372
  def hash_join(left_sym, right_sym, &block)
216
- left_offset, right_subtuple, right_offset = join_offsets(@localpreds.first)
373
+ # we know that a hashpred has been canonicalized with @rels[0] in left offset
374
+ left_offset = @hashpreds.first[0][1]
375
+ right_subtuple, right_offset = join_offset(@hashpreds.first[1])
217
376
 
218
377
  syms = [left_sym, right_sym]
219
378
 
@@ -249,25 +408,14 @@ module Bud
249
408
  left = s_tup; right = r
250
409
  end
251
410
  retval = left + right
252
- yield retval if test_locals(left[0], right, @localpreds.first)
411
+ yield retval if test_locals(left[0], right, @hashpreds.first)
253
412
  end
254
413
  end
255
414
  end
256
415
  end
257
416
  end
258
417
 
259
- class BudLeftJoin < BudJoin # :nodoc: all
260
- def initialize(rellist, bud_instance, preds=nil)
261
- raise(BudError, "Left Join only defined for two relations") unless rellist.length == 2
262
- super(rellist, bud_instance, preds)
263
- @origpreds = preds
264
- preds.each do |k,v|
265
- if k.class <= Array
266
- raise Bud::CompileError, "in leftjoin, attribute refs must have style :col1 => :col2"
267
- end
268
- end
269
- end
270
-
418
+ module BudOuterJoin
271
419
  public
272
420
  def each(&block) # :nodoc:all
273
421
  super(&block)
@@ -298,3 +446,4 @@ module Bud
298
446
  end
299
447
  end
300
448
  end
449
+
@@ -0,0 +1,84 @@
1
+ # We monkeypatch Module to add support for Bloom state and code declarations.
2
+ class Module
3
+ # import another module and assign to a qualifier symbol: <tt>import MyModule => :m</tt>
4
+ def import(spec)
5
+ raise Bud::CompileError unless (spec.class <= Hash and spec.length == 1)
6
+ mod, local_name = spec.first
7
+ raise Bud::CompileError unless (mod.class <= Module and local_name.class <= Symbol)
8
+
9
+ # To correctly expand qualified references to an imported module, we keep a
10
+ # table with the local bind names of all the modules imported by this
11
+ # module. To handle nested references (a.b.c.d etc.), the import table for
12
+ # module X points to X's own nested import table.
13
+ @bud_import_tbl ||= {}
14
+ child_tbl = mod.bud_import_table
15
+ raise Bud::CompileError if @bud_import_tbl.has_key? local_name
16
+ @bud_import_tbl[local_name] = child_tbl.clone # XXX: clone needed?
17
+
18
+ rewritten_mod_name = ModuleRewriter.do_import(self, mod, local_name)
19
+ self.module_eval "include #{rewritten_mod_name}"
20
+ end
21
+
22
+ # the block of Bloom collection declarations. one per module.
23
+ def state(&block)
24
+ meth_name = Module.make_state_meth_name(self)
25
+ define_method(meth_name, &block)
26
+ end
27
+
28
+ # a ruby block to be run before timestep 1. one per module.
29
+ def bootstrap(&block)
30
+ meth_name = "__bootstrap__#{Module.get_class_name(self)}".to_sym
31
+ define_method(meth_name, &block)
32
+ end
33
+
34
+ # bloom statements to be registered with Bud runtime. optional +block_name+
35
+ # allows for multiple bloom blocks per module, and overriding
36
+ def bloom(block_name=nil, &block)
37
+ # If no block name was specified, generate a unique name
38
+ if block_name.nil?
39
+ @block_id ||= 0
40
+ block_name = "#{Module.get_class_name(self)}__#{@block_id.to_s}"
41
+ @block_id += 1
42
+ else
43
+ unless block_name.class <= Symbol
44
+ raise Bud::CompileError, "Bloom block names must be a symbol: #{block_name}"
45
+ end
46
+ end
47
+
48
+ # Note that we don't encode the module name ("self") into the name of the
49
+ # method. This allows named blocks to be overridden (via inheritance or
50
+ # mixin) in the same way as normal Ruby methods.
51
+ meth_name = "__bloom__#{block_name}"
52
+
53
+ # Don't allow duplicate named bloom blocks to be defined within a single
54
+ # module; this indicates a likely programmer error.
55
+ if instance_methods(false).include? meth_name
56
+ raise Bud::CompileError, "Duplicate named bloom block: '#{block_name}' in #{self}"
57
+ end
58
+ define_method(meth_name.to_sym, &block)
59
+ end
60
+
61
+ def bud_import_table() #:nodoc: all
62
+ @bud_import_tbl ||= {}
63
+ @bud_import_tbl
64
+ end
65
+
66
+ private
67
+ # Return a string with a version of the class name appropriate for embedding
68
+ # into a method name. Annoyingly, if you define class X nested inside
69
+ # class/module Y, X's class name is the string "Y::X". We don't want to define
70
+ # method names with semicolons in them, so just return "X" instead.
71
+ def self.get_class_name(klass)
72
+ klass.name.split("::").last
73
+ end
74
+
75
+ # State method blocks are named using an auto-incrementing counter. This is to
76
+ # ensure that we can rediscover the possible dependencies between these blocks
77
+ # after module import (see Bud#call_state_methods).
78
+ def self.make_state_meth_name(klass)
79
+ @state_meth_id ||= 0
80
+ r = "__state#{@state_meth_id}__#{Module.get_class_name(klass)}".to_sym
81
+ @state_meth_id += 1
82
+ return r
83
+ end
84
+ end
data/lib/bud/rebl.rb CHANGED
@@ -72,7 +72,7 @@ class ReblShell
72
72
  # LibRebl that is created; testcases call this directly.
73
73
  def self.setup
74
74
  Signal.trap("INT") {do_exit}
75
- Signal.trap("TRAP") {do_exit}
75
+ Signal.trap("TERM") {do_exit}
76
76
 
77
77
  ipport = ARGV[0] ? ARGV[0].split(":") : []
78
78
  lib = LibRebl.new(*[(ipport[0] or "localhost"), (ipport[1] or 0)])
@@ -287,6 +287,13 @@ class LibRebl
287
287
  @rebl_class_inst.tables.merge!(@old_inst.tables.reject do |k,v|
288
288
  @@builtin_tables.include? k
289
289
  end)
290
+ @rebl_class_inst.channels.merge!(@old_inst.channels.reject do |k,v|
291
+ @@builtin_tables.include? k
292
+ end)
293
+ @rebl_class_inst.tc_tables.merge! @old_inst.tc_tables
294
+ @rebl_class_inst.dbm_tables.merge! @old_inst.dbm_tables
295
+ @rebl_class_inst.zk_tables.merge! @old_inst.zk_tables
296
+
290
297
  # Fix the bud instance pointers from copied tables.
291
298
  @rebl_class_inst.tables.values.each do |v|
292
299
  v.bud_instance = @rebl_class_inst