bud 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (62) hide show
  1. data/LICENSE +9 -0
  2. data/README +30 -0
  3. data/bin/budplot +134 -0
  4. data/bin/budvis +201 -0
  5. data/bin/rebl +4 -0
  6. data/docs/README.md +13 -0
  7. data/docs/bfs.md +379 -0
  8. data/docs/bfs.raw +251 -0
  9. data/docs/bfs_arch.png +0 -0
  10. data/docs/bloom-loop.png +0 -0
  11. data/docs/bust.md +83 -0
  12. data/docs/cheat.md +291 -0
  13. data/docs/deploy.md +96 -0
  14. data/docs/diffs +181 -0
  15. data/docs/getstarted.md +296 -0
  16. data/docs/intro.md +36 -0
  17. data/docs/modules.md +112 -0
  18. data/docs/operational.md +96 -0
  19. data/docs/rebl.md +99 -0
  20. data/docs/ruby_hooks.md +19 -0
  21. data/docs/visualizations.md +75 -0
  22. data/examples/README +1 -0
  23. data/examples/basics/hello.rb +12 -0
  24. data/examples/basics/out +1103 -0
  25. data/examples/basics/out.new +856 -0
  26. data/examples/basics/paths.rb +51 -0
  27. data/examples/bust/README.md +9 -0
  28. data/examples/bust/bustclient-example.rb +23 -0
  29. data/examples/bust/bustinspector.html +135 -0
  30. data/examples/bust/bustserver-example.rb +18 -0
  31. data/examples/chat/README.md +9 -0
  32. data/examples/chat/chat.rb +45 -0
  33. data/examples/chat/chat_protocol.rb +8 -0
  34. data/examples/chat/chat_server.rb +29 -0
  35. data/examples/deploy/tokenring-ec2.rb +26 -0
  36. data/examples/deploy/tokenring-local.rb +17 -0
  37. data/examples/deploy/tokenring.rb +39 -0
  38. data/lib/bud/aggs.rb +126 -0
  39. data/lib/bud/bud_meta.rb +185 -0
  40. data/lib/bud/bust/bust.rb +126 -0
  41. data/lib/bud/bust/client/idempotence.rb +10 -0
  42. data/lib/bud/bust/client/restclient.rb +49 -0
  43. data/lib/bud/collections.rb +937 -0
  44. data/lib/bud/depanalysis.rb +44 -0
  45. data/lib/bud/deploy/countatomicdelivery.rb +50 -0
  46. data/lib/bud/deploy/deployer.rb +67 -0
  47. data/lib/bud/deploy/ec2deploy.rb +200 -0
  48. data/lib/bud/deploy/localdeploy.rb +41 -0
  49. data/lib/bud/errors.rb +15 -0
  50. data/lib/bud/graphs.rb +405 -0
  51. data/lib/bud/joins.rb +300 -0
  52. data/lib/bud/rebl.rb +314 -0
  53. data/lib/bud/rewrite.rb +523 -0
  54. data/lib/bud/rtrace.rb +27 -0
  55. data/lib/bud/server.rb +43 -0
  56. data/lib/bud/state.rb +108 -0
  57. data/lib/bud/storage/tokyocabinet.rb +170 -0
  58. data/lib/bud/storage/zookeeper.rb +178 -0
  59. data/lib/bud/stratify.rb +83 -0
  60. data/lib/bud/viz.rb +65 -0
  61. data/lib/bud.rb +797 -0
  62. metadata +330 -0
@@ -0,0 +1,51 @@
1
+ # simple shortest paths
2
+ # note use of program.tick at bottom to run a single timestemp
3
+ # and inspect relations
4
+ require 'rubygems'
5
+ require 'bud'
6
+
7
+ class ShortestPaths
8
+ include Bud
9
+
10
+ state do
11
+ table :link, [:from, :to, :cost]
12
+ table :path, [:from, :to, :next, :cost]
13
+ table :shortest, [:from, :to] => [:next, :cost]
14
+ end
15
+
16
+ # recursive rules to define all paths from links
17
+ bloom :make_paths do
18
+ # base case: every link is a path
19
+ path <= link {|e| [e.from, e.to, e.to, e.cost]}
20
+
21
+ # inductive case: make path of length n+1 by connecting a link to a path of length n
22
+ temp :j <= (link*path).pairs(:to => :from)
23
+ path <= j { |l,p| [l.from, p.to, p.from, l.cost+p.cost] }
24
+ end
25
+
26
+ # find the shortest path between each connected pair of nodes
27
+ bloom :find_shortest do
28
+ shortest <= path.argmin([path.from, path.to], path.cost)
29
+ end
30
+ end
31
+
32
+ # compute shortest paths.
33
+ program = ShortestPaths.new
34
+
35
+ # populate our little example. we put two links between 'a' and 'b'
36
+ # to see whether our shortest-paths code does the right thing.
37
+ program.link <= [['a', 'b', 1],
38
+ ['a', 'b', 4],
39
+ ['b', 'c', 1],
40
+ ['c', 'd', 1],
41
+ ['d', 'e', 1]]
42
+
43
+ program.tick # one timestamp is enough for this simple program
44
+ program.shortest.sort.each {|t| puts t.inspect}
45
+
46
+ puts "----"
47
+
48
+ # now lets add an extra link and recompute
49
+ program.link << ['e', 'f', 1]
50
+ program.tick
51
+ program.shortest.sort.each {|t| puts t.inspect}
@@ -0,0 +1,9 @@
1
+ To run the bust twitter example client, do the following:
2
+
3
+ # ruby bustclient-example.rb
4
+
5
+ To test out bust inspector, do the following:
6
+
7
+ # ruby bustserver-example.rb
8
+
9
+ Then, launch bustinspector.html in a web browser (only Firefox 4 has been tested thus far) on either the computer where you launched bustserver-example.rb, or any computer that has TCP port 8080 forwarded to the computer where you launched bustserver-example.rb.
@@ -0,0 +1,23 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+ require 'bud/bust/client/restclient'
4
+
5
+ class TwitterExample # :nodoc: all
6
+ include Bud
7
+ include RestClient
8
+
9
+ bootstrap do
10
+ # get the 20 most recent tweets from twitter
11
+ rest_req <= [[1, :get, :json,
12
+ 'http://api.twitter.com/1/statuses/public_timeline']]
13
+ end
14
+
15
+ bloom :print_recent_tweets do
16
+ # print the tweets with user screen names
17
+ stdio <~ rest_response do |r|
18
+ [r.resp.map {|s| s["user"]["screen_name"] + ": " + s["text"]}] if r.rid==1
19
+ end
20
+ end
21
+ end
22
+
23
+ TwitterExample.new.run_fg
@@ -0,0 +1,135 @@
1
+ <html>
2
+ <head>
3
+ <title>BUST Inspector</title>
4
+
5
+ <script type="text/javascript">
6
+ function pollState()
7
+ {
8
+
9
+ // 1. GET the list of tables
10
+ var xmlhttp = new XMLHttpRequest();
11
+ xmlhttp.open("GET","http://localhost:8080/t_table_info",false);
12
+ xmlhttp.send();
13
+
14
+ if (xmlhttp.readyState==4) {
15
+ if (xmlhttp.status == 404){
16
+ alert("error fetching 't_table_info'");
17
+ }
18
+ if (xmlhttp.status == 200){
19
+ tableArray = JSON.parse(xmlhttp.responseText);
20
+
21
+ var tablenames = document.getElementById("tablenames");
22
+
23
+ for (var i = 0; i < tableArray.length; i++) {
24
+ var tablename = tableArray[i][0]
25
+ // if the collection is new
26
+ if (!document.getElementById("coll_" + tablename)) {
27
+ var newspan = document.createElement("span");
28
+ newspan.innerHTML = '<input type="checkbox" id="coll_' + tablename + '"/>' + tablename + '(<small><b>' + tableArray[i][1] + '</b></small>) <br/>'
29
+ while (newspan.firstChild) {
30
+ tablenames.appendChild(newspan.firstChild);
31
+ }
32
+ }
33
+ }
34
+
35
+ }
36
+ }
37
+
38
+
39
+ // 2. GET the schema table
40
+ var schemareq = new XMLHttpRequest();
41
+ schemareq.open("GET","http://localhost:8080/t_table_schema",false);
42
+ schemareq.send();
43
+
44
+ if (schemareq.status == 404) {
45
+ alert("error fetching schema info");
46
+ }
47
+ if (schemareq.status == 200) {
48
+ schemaArray = JSON.parse(schemareq.responseText)
49
+ }
50
+
51
+
52
+ // 3. GET the list of checked tables
53
+ form_elts = document.forms[0].elements
54
+ for (var i = 0; i < form_elts.length; i++) {
55
+ var tablename = form_elts[i].id.substr(5);
56
+
57
+ //look it up in the schema
58
+ var schemaIdx;
59
+ var found = false;
60
+ for (schemaIdx = 0; schemaIdx < schemaArray.length; schemaIdx++) {
61
+ if (schemaArray[schemaIdx][0] == tablename) {
62
+ found = true;
63
+ break;
64
+ }
65
+ }
66
+ if (!found) {
67
+ alert ("no schema information for table " + tablename);
68
+ }
69
+
70
+ var ourSchema = schemaArray[schemaIdx][1];
71
+
72
+ if (form_elts[i].checked) {
73
+ var tablereq = new XMLHttpRequest();
74
+ tablereq.open("GET","http://localhost:8080/"+tablename,false);
75
+ tablereq.send();
76
+
77
+ if (tablereq.status == 404){
78
+ alert("error fetching table " + tablename);
79
+ }
80
+ if (tablereq.status == 200){
81
+ responseArray = JSON.parse(tablereq.responseText)
82
+
83
+ //schema
84
+ tableText = "<table style='border-width: 1px; border-spacing: 2px; border-style: outset; border-color: gray; border-collapse: separate;'><tr>";
85
+ for (var j = 0; j < ourSchema.length; j++) {
86
+ tableText += "<td style='border-width: 1px; padding: 1px; border-style: inset; border-color: gray;'><b>" + ourSchema[j] + "</b></td>";
87
+ }
88
+ tableText += "</tr>";
89
+
90
+ //data
91
+ for (var k = 0; k < responseArray.length; k++) {
92
+ tableText += "<tr>";
93
+ for (var l = 0; l < responseArray[k].length; l++) {
94
+ tableText += "<td style='border-width: 1px; padding: 1px; border-style: inset; border-color: gray;'>" + responseArray[k][l] + "</td>";
95
+ }
96
+ tableText += "</tr>";
97
+ }
98
+ tableText += "</table>";
99
+
100
+ if (!document.getElementById("disp_" + tablename)) {
101
+ document.getElementById("tabledisplay").innerHTML += '<div style="border: 1px solid black; float:left; padding: 0px; margin:5px;" id="disp_' + tablename + '"><h4 style="margin-top:0px;">' + tablename + '</h4>' + tableText + '</div>';
102
+ } else {
103
+ document.getElementById("disp_" + tablename).innerHTML = '<h4 style="margin-top:0px;">' + tablename + '</h4>' + tableText;
104
+ }
105
+ }
106
+ } else { //if not checked
107
+ if (document.getElementById("disp_" + tablename)) {
108
+ document.getElementById("tabledisplay").removeChild(document.getElementById("disp_" + tablename))
109
+ }
110
+ }
111
+
112
+ } //end for
113
+
114
+
115
+ setTimeout('pollState()', 1000);
116
+ }
117
+
118
+ </script>
119
+
120
+ </head>
121
+
122
+ <body onLoad="setTimeout('pollState()', 1000);">
123
+
124
+ <div style="width:100%;"><h3>BUST Inspector</h3></div>
125
+ <div style="height:100%; border: 1px solid black; float:left; padding:0px; margin:5px;">
126
+ <h4 style="margin-top:0px;">Collections</h4>
127
+ <form id="tableform">
128
+ <span id="tablenames">
129
+ </span>
130
+ </form>
131
+ </div>
132
+ <span id="tabledisplay">
133
+ </span>
134
+ </body>
135
+ </html>
@@ -0,0 +1,18 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+ require 'bud/bust/bust'
4
+
5
+ class BustExample # :nodoc: all
6
+ include Bud
7
+ include Bust
8
+
9
+ state do
10
+ table :foo, [:bar, :baz, :qux]
11
+ end
12
+
13
+ bloom do
14
+ stdio <~ foo {|t| [t.inspect]}
15
+ end
16
+ end
17
+
18
+ BustExample.new.run_fg
@@ -0,0 +1,9 @@
1
+ To run the chat example, do each of the following in a different terminal:
2
+
3
+ # ruby chat_server.rb
4
+
5
+ # ruby chat.rb alice
6
+
7
+ # ruby chat.rb bob
8
+
9
+ # ruby chat.rb harvey
@@ -0,0 +1,45 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+ require 'chat_protocol'
4
+
5
+ class ChatClient
6
+ include Bud
7
+ include ChatProtocol
8
+
9
+ def initialize(nick, server, opts={})
10
+ @nick = nick
11
+ @server = server
12
+ super opts
13
+ end
14
+
15
+ bootstrap do
16
+ connect <~ [[@server, [ip_port, @nick]]]
17
+ end
18
+
19
+ bloom do
20
+ mcast <~ stdio do |s|
21
+ [@server, [ip_port, @nick, Time.new.strftime("%I:%M.%S"), s.line]]
22
+ end
23
+
24
+ stdio <~ mcast { |m| [pretty_print(m.val)] }
25
+ end
26
+
27
+ # format chat messages with timestamp on the right of the screen
28
+ def pretty_print(val)
29
+ str = val[1].to_s + ": " + (val[3].to_s || '')
30
+ pad = "(" + val[2].to_s + ")"
31
+ return str + " "*[66 - str.length,2].max + pad
32
+ end
33
+ end
34
+
35
+
36
+
37
+ if ARGV.length == 2
38
+ server = ARGV[1]
39
+ else
40
+ server = ChatProtocol::DEFAULT_ADDR
41
+ end
42
+
43
+ puts "Server address: #{server}"
44
+ program = ChatClient.new(ARGV[0], server, :read_stdin => true)
45
+ program.run_fg
@@ -0,0 +1,8 @@
1
+ module ChatProtocol
2
+ state do
3
+ channel :mcast
4
+ channel :connect
5
+ end
6
+
7
+ DEFAULT_ADDR = "localhost:12345"
8
+ end
@@ -0,0 +1,29 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+ require 'chat_protocol'
4
+
5
+ class ChatServer
6
+ include Bud
7
+ include ChatProtocol
8
+
9
+ state { table :nodelist }
10
+
11
+ bloom do
12
+ nodelist <= connect.payloads
13
+ mcast <~ (mcast * nodelist).pairs { |m,n| [n.key, m.val] }
14
+ end
15
+ end
16
+
17
+
18
+
19
+ # ruby command-line wrangling
20
+ if ARGV.first
21
+ addr = ARGV.first
22
+ else
23
+ addr = ChatProtocol::DEFAULT_ADDR
24
+ end
25
+
26
+ ip, port = addr.split(":")
27
+ puts "Server address: #{ip}:#{port}"
28
+ program = ChatServer.new(:ip => ip, :port => port.to_i)
29
+ program.run_fg
@@ -0,0 +1,26 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+ require 'tokenring'
4
+ require 'bud/deploy/ec2deploy'
5
+
6
+ class RingLocal
7
+ include Bud
8
+ include TokenRing
9
+ include EC2Deploy
10
+
11
+ deploystrap do
12
+ node_count << [10]
13
+ eval(IO.read('keys.rb'), binding) if File.exists?('keys.rb')
14
+ ruby_command << ["ruby tokenring-ec2.rb"]
15
+ init_dir << ["."]
16
+ end
17
+
18
+ end
19
+
20
+ ip, port = ARGV[0].split(':')
21
+ ext_ip, ext_port = ARGV[1].split(':')
22
+ RingLocal.new(:ip => ip,
23
+ :ext_ip => ext_ip,
24
+ :port => port,
25
+ :ext_port => ext_port,
26
+ :deploy => ARGV[2]).run_fg
@@ -0,0 +1,17 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+ require 'tokenring'
4
+ require 'bud/deploy/localdeploy'
5
+
6
+ class RingLocal
7
+ include Bud
8
+ include TokenRing
9
+ include LocalDeploy
10
+
11
+ deploystrap do
12
+ node_count << [10]
13
+ end
14
+
15
+ end
16
+
17
+ RingLocal.new(:deploy => true).run_fg
@@ -0,0 +1,39 @@
1
+ require 'rubygems'
2
+ require 'bud'
3
+
4
+ module TokenRing
5
+
6
+ state do
7
+ table :next_node, [] => [:node]
8
+ channel :token, [:@loc]
9
+ table :token_persist, [:loc]
10
+ end
11
+
12
+ bloom :make_ring do
13
+ initial_data <= node.map do |n|
14
+ # Calculate the successor node (splice deployer into the list)
15
+ succ_id = (n.uid + 1) % (node_count[[]].num + 1)
16
+ succ_node = (succ_id == node_count[[]].num) ? [ip_port] :
17
+ [node[[succ_id]].node]
18
+ [ n.uid, :next_node, [ succ_node ] ]
19
+ end
20
+
21
+ # Initial data for deployer (not part of "node" set)
22
+ next_node <= node.map do
23
+ [node[[0]].node] if @options[:deploy]
24
+ end
25
+ token <~ node.map do
26
+ [node[[0]].node] if @options [:deploy]
27
+ end
28
+ end
29
+
30
+ bloom :pass_token do
31
+ # Persist the token for as long as necessary
32
+ token_persist <= token
33
+ token_persist <- join([token_persist, next_node]).map {|t,_| [t]}
34
+ # Pass on the token
35
+ token <~ join([token_persist, next_node]).map {[next_node[[]].node]}
36
+ stdio <~ token.map {["#{ip_port}: Got token!"]}
37
+ end
38
+
39
+ end
data/lib/bud/aggs.rb ADDED
@@ -0,0 +1,126 @@
1
+ module Bud
2
+ ######## Agg definitions
3
+ class Agg #:nodoc: all
4
+ def init(val)
5
+ val
6
+ end
7
+
8
+ def final(state)
9
+ state
10
+ end
11
+ end
12
+
13
+ class Exemplary < Agg #:nodoc: all
14
+ end
15
+
16
+ # ArgExemplary aggs are used by argagg. Canonical examples are min/min (argmin/max)
17
+ # They must have a trivial final method and be monotonic, i.e. once a value v
18
+ # is discarded in favor of another, v can never be the final result
19
+
20
+ class ArgExemplary < Agg #:nodoc: all
21
+ def tie(state, val)
22
+ (state == val)
23
+ end
24
+ def final(state)
25
+ state
26
+ end
27
+ end
28
+
29
+ class Min < ArgExemplary #:nodoc: all
30
+ def trans(state, val)
31
+ state < val ? state : val
32
+ end
33
+ end
34
+ # exemplary aggregate method to be used in Bud::BudCollection.group.
35
+ # computes minimum of x entries aggregated.
36
+ def min(x)
37
+ [Min.new, x]
38
+ end
39
+
40
+ class Max < ArgExemplary #:nodoc: all
41
+ def trans(state, val)
42
+ state > val ? state : val
43
+ end
44
+ end
45
+ # exemplary aggregate method to be used in Bud::BudCollection.group.
46
+ # computes maximum of x entries aggregated.
47
+ def max(x)
48
+ [Max.new, x]
49
+ end
50
+
51
+ class Choose < ArgExemplary #:nodoc: all
52
+ def trans(state, val)
53
+ state.nil? ? val : state
54
+ end
55
+ def tie(state, val)
56
+ false
57
+ end
58
+ end
59
+
60
+ # exemplary aggregate method to be used in Bud::BudCollection.group.
61
+ # arbitrarily but deterministically chooses among x entries being aggregated.
62
+ def choose(x)
63
+ [Choose.new, x]
64
+ end
65
+
66
+ class Sum < Agg #:nodoc: all
67
+ def trans(state, val)
68
+ state + val
69
+ end
70
+ end
71
+
72
+ # aggregate method to be used in Bud::BudCollection.group.
73
+ # computes sum of x entries aggregated.
74
+ def sum(x)
75
+ [Sum.new, x]
76
+ end
77
+
78
+ class Count < Agg #:nodoc: all
79
+ def init(x=nil)
80
+ 1
81
+ end
82
+ def trans(state, x=nil)
83
+ state + 1
84
+ end
85
+ end
86
+
87
+ # aggregate method to be used in Bud::BudCollection.group.
88
+ # counts number of entries aggregated. argument is ignored.
89
+ def count(x=nil)
90
+ [Count.new]
91
+ end
92
+
93
+ class Avg < Agg #:nodoc: all
94
+ def init(val)
95
+ [val, 1]
96
+ end
97
+ def trans(state, val)
98
+ retval = [state[0] + val]
99
+ retval << (state[1] + 1)
100
+ end
101
+ def final(state)
102
+ state[0]*1.0 / state[1]
103
+ end
104
+ end
105
+
106
+ # aggregate method to be used in Bud::BudCollection.group.
107
+ # computes average of a multiset of x values
108
+ def avg(x)
109
+ [Avg.new, x]
110
+ end
111
+
112
+ class Accum < Agg #:nodoc: all
113
+ def init(x)
114
+ [x]
115
+ end
116
+ def trans(state, val)
117
+ state << val
118
+ end
119
+ end
120
+
121
+ # aggregate method to be used in Bud::BudCollection.group.
122
+ # accumulates all x inputs into an array
123
+ def accum(x)
124
+ [Accum.new, x]
125
+ end
126
+ end