bud 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +33 -16
- data/bin/budplot +42 -65
- data/bin/budtimelines +235 -0
- data/bin/budvis +24 -122
- data/bin/rebl +1 -0
- data/docs/README.md +21 -10
- data/docs/bfs.md +4 -6
- data/docs/c.html +251 -0
- data/docs/cheat.md +45 -30
- data/docs/deploy.md +26 -26
- data/docs/getstarted.md +6 -4
- data/docs/visualizations.md +43 -31
- data/examples/chat/chat.rb +4 -9
- data/examples/chat/chat_server.rb +1 -8
- data/examples/deploy/deploy_ip_port +1 -0
- data/examples/deploy/keys.rb +5 -0
- data/examples/deploy/tokenring-ec2.rb +9 -9
- data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
- data/examples/deploy/tokenring-thread.rb +15 -0
- data/examples/deploy/tokenring.rb +25 -17
- data/lib/bud/aggs.rb +87 -25
- data/lib/bud/bud_meta.rb +48 -31
- data/lib/bud/bust/bust.rb +16 -15
- data/lib/bud/collections.rb +207 -232
- data/lib/bud/depanalysis.rb +1 -0
- data/lib/bud/deploy/countatomicdelivery.rb +8 -20
- data/lib/bud/deploy/deployer.rb +16 -16
- data/lib/bud/deploy/ec2deploy.rb +34 -35
- data/lib/bud/deploy/forkdeploy.rb +90 -0
- data/lib/bud/deploy/threaddeploy.rb +38 -0
- data/lib/bud/graphs.rb +103 -199
- data/lib/bud/joins.rb +190 -41
- data/lib/bud/monkeypatch.rb +84 -0
- data/lib/bud/rebl.rb +8 -1
- data/lib/bud/rewrite.rb +152 -49
- data/lib/bud/server.rb +1 -0
- data/lib/bud/state.rb +24 -10
- data/lib/bud/storage/dbm.rb +170 -0
- data/lib/bud/storage/tokyocabinet.rb +5 -1
- data/lib/bud/stratify.rb +6 -7
- data/lib/bud/viz.rb +31 -17
- data/lib/bud/viz_util.rb +204 -0
- data/lib/bud.rb +271 -244
- data/lib/bud.rb.orig +806 -0
- metadata +43 -22
- data/docs/bfs.raw +0 -251
- data/docs/diffs +0 -181
- data/examples/basics/out +0 -1103
- data/examples/basics/out.new +0 -856
- data/lib/bud/deploy/localdeploy.rb +0 -53
data/lib/bud/depanalysis.rb
CHANGED
@@ -10,41 +10,29 @@ module CountAtomicDelivery # :nodoc: all
|
|
10
10
|
channel :atomic_data_chan, [:@loc, :tuple]
|
11
11
|
table :atomic_data_recv, [:loc, :tuple]
|
12
12
|
|
13
|
-
scratch :atomic_count, [:loc, :cnt]
|
14
13
|
channel :atomic_count_chan,[:@loc, :cnt]
|
15
14
|
table :atomic_count_recv, [] => [:cnt]
|
16
15
|
|
17
16
|
scratch :atomic_recv_count, [:loc] => [:cnt]
|
18
|
-
scratch :
|
17
|
+
scratch :atomic_data, [:loc, :tuple]
|
19
18
|
scratch :atomic_data_out, [:tuple]
|
20
19
|
end
|
21
20
|
|
22
21
|
bloom :countatomicdelivery do
|
23
|
-
atomic_count <= atomic_data_in.group([:loc], count)
|
22
|
+
temp :atomic_count <= atomic_data_in.group([:loc], count)
|
24
23
|
|
25
24
|
atomic_data_chan <~ atomic_data_in
|
26
25
|
atomic_count_chan <~ atomic_count
|
27
26
|
|
28
|
-
atomic_count_recv <= atomic_count_chan
|
27
|
+
atomic_count_recv <= atomic_count_chan {|c| [c.cnt]}
|
29
28
|
atomic_data_recv <= atomic_data_chan
|
30
29
|
|
31
30
|
atomic_recv_count <= atomic_data_recv.group([:loc], count)
|
32
31
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
atomic_data_atomic <= ((if atomic_recv_count.first and
|
39
|
-
atomic_count_recv.first and
|
40
|
-
atomic_recv_count.first.cnt ==
|
41
|
-
atomic_count_recv.first.cnt
|
42
|
-
atomic_data_recv
|
43
|
-
end) or [])
|
44
|
-
|
45
|
-
# Commented out due to bug #85.
|
46
|
-
# atomic_data_recv <- atomic_data_atomic
|
47
|
-
# Idempotence hack inserted due to bug #85.
|
48
|
-
atomic_data_out <= atomic_data_atomic.map {|a| [a.tuple] if depl_idempotent [[:atomic_data_out, a]]}
|
32
|
+
atomic_data <= (atomic_recv_count * atomic_count_recv * atomic_data_recv).combos(atomic_recv_count.cnt => atomic_count_recv.cnt) {|rc, cr, d| d}
|
33
|
+
|
34
|
+
atomic_data_recv <- atomic_data
|
35
|
+
|
36
|
+
atomic_data_out <= atomic_data {|a| [a.tuple]}
|
49
37
|
end
|
50
38
|
end
|
data/lib/bud/deploy/deployer.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'bud'
|
3
|
-
require 'open-uri'
|
4
3
|
require 'bud/deploy/countatomicdelivery'
|
5
4
|
|
6
5
|
class Module
|
@@ -14,8 +13,12 @@ module Deployer # :nodoc: all
|
|
14
13
|
include CountAtomicDelivery
|
15
14
|
|
16
15
|
state do
|
17
|
-
table :node, [:uid] => [:
|
16
|
+
table :node, [:uid] => [:addr]
|
18
17
|
table :node_count, [] => [:num]
|
18
|
+
# At the deployer node, this collection will contain a fact for the first
|
19
|
+
# tick in which the given node is ready
|
20
|
+
scratch :node_ready, [:uid]
|
21
|
+
|
19
22
|
table :initial_data, [:uid, :pred, :data]
|
20
23
|
channel :dont_care, [:@loc]
|
21
24
|
table :dead, [:dead]
|
@@ -33,11 +36,9 @@ module Deployer # :nodoc: all
|
|
33
36
|
end
|
34
37
|
end
|
35
38
|
|
36
|
-
def initialize(opt
|
39
|
+
def initialize(opt={})
|
37
40
|
super
|
38
|
-
if opt[:deploy]
|
39
|
-
do_deploystrap
|
40
|
-
end
|
41
|
+
do_deploystrap if opt[:deploy]
|
41
42
|
end
|
42
43
|
|
43
44
|
# Distribute the EDB to each node.
|
@@ -46,22 +47,21 @@ module Deployer # :nodoc: all
|
|
46
47
|
# before any messages are received. In order to fix this, we would probably
|
47
48
|
# need to globally synchronize to ensure that "timestamp 0" gets "fully
|
48
49
|
# evaluated" before any messages can be sent.
|
49
|
-
|
50
50
|
bloom :distribute_data do
|
51
|
-
atomic_data_in <=
|
52
|
-
|
53
|
-
[n.node, [i.pred, i.data]] if depl_idempotent [[n.node, i.pred, i.data]]
|
51
|
+
atomic_data_in <= (node_ready * node * initial_data).combos(node_ready.uid => node.uid, node.uid => initial_data.uid) do |nr, n, i|
|
52
|
+
[n.addr, [i.pred, i.data]]
|
54
53
|
end
|
55
54
|
|
56
55
|
# Add all tuples at once.
|
57
|
-
dont_care <~ atomic_data_out
|
58
|
-
|
59
|
-
a.tuple[
|
60
|
-
eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
|
61
|
-
end
|
62
|
-
[ip_port]
|
56
|
+
dont_care <~ atomic_data_out do |a|
|
57
|
+
a.tuple[1].map do |d|
|
58
|
+
eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
|
63
59
|
end
|
60
|
+
[ip_port]
|
64
61
|
end
|
62
|
+
end
|
65
63
|
|
64
|
+
bloom :print_ready do
|
65
|
+
stdio <~ node_ready {|nr| ["Child node ready: #{nr.uid}"]}
|
66
66
|
end
|
67
67
|
end
|
data/lib/bud/deploy/ec2deploy.rb
CHANGED
@@ -6,7 +6,11 @@ require 'net/scp'
|
|
6
6
|
require 'bud'
|
7
7
|
require 'bud/deploy/deployer'
|
8
8
|
|
9
|
-
#
|
9
|
+
# Logic to deploy Bloom programs on EC2
|
10
|
+
# TODO:
|
11
|
+
# * add support for deploy_child_opts
|
12
|
+
# * avoid blocking I/O calls
|
13
|
+
# * emit "node_ready" event when each node is ready
|
10
14
|
module EC2Deploy
|
11
15
|
include Deployer
|
12
16
|
|
@@ -46,23 +50,23 @@ module EC2Deploy
|
|
46
50
|
unless @options[:deploy]
|
47
51
|
# Send message to the deployer telling 'em we's up.
|
48
52
|
File.open("deploy_ip_port", "r") do |f|
|
49
|
-
ready <~ [[f.readline.
|
53
|
+
ready <~ [[f.readline.rstrip, ip_port]]
|
50
54
|
end
|
51
55
|
end
|
52
56
|
end
|
53
57
|
|
54
58
|
bloom :spinup do
|
55
|
-
ec2_conn <=
|
59
|
+
ec2_conn <= (access_key_id * secret_access_key).pairs do
|
56
60
|
if depl_idempotent [:ec2_comm]
|
57
61
|
[AWS::EC2::Base.new(:access_key_id => access_key_id[[]].key,
|
58
62
|
:secret_access_key => secret_access_key[[]].key)]
|
59
63
|
end
|
60
64
|
end
|
61
65
|
|
62
|
-
ec2_insts <=
|
66
|
+
ec2_insts <= (image_id * node_count * key_name * ec2_conn).combos do
|
63
67
|
if depl_idempotent [:ec2_insts]
|
64
68
|
print "Starting up EC2 instances"
|
65
|
-
|
69
|
+
$stdout.flush
|
66
70
|
# First, we create the security group.
|
67
71
|
begin
|
68
72
|
ec2_conn[[]].conn.create_security_group(:group_name => "bud", :group_description => "bud")
|
@@ -70,18 +74,18 @@ module EC2Deploy
|
|
70
74
|
# Group already exists; ok, maybe we created it previously.
|
71
75
|
else
|
72
76
|
# Add SSH permission.
|
73
|
-
ec2_conn[[]].conn.authorize_security_group_ingress(
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
77
|
+
ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
|
78
|
+
:ip_protocol => "tcp",
|
79
|
+
:from_port => 22,
|
80
|
+
:to_port => 22,
|
81
|
+
:cidr_ip => "0.0.0.0/0")
|
78
82
|
# Add unlimited UDP permission from any node not in the security group.
|
79
83
|
# XXX: make this more restrictive?
|
80
|
-
ec2_conn[[]].conn.authorize_security_group_ingress(
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
84
|
+
ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
|
85
|
+
:ip_protocol => "udp",
|
86
|
+
:from_port => 0,
|
87
|
+
:to_port => 65535,
|
88
|
+
:cidr_ip => "0.0.0.0/0")
|
85
89
|
end
|
86
90
|
|
87
91
|
# Finally, start up the instances.
|
@@ -93,19 +97,19 @@ module EC2Deploy
|
|
93
97
|
end
|
94
98
|
end
|
95
99
|
|
96
|
-
the_reservation <=
|
100
|
+
the_reservation <= (spinup_timer * ec2_conn * ec2_insts).combos do |t,c,i|
|
97
101
|
if depl_idempotent [[:the_reservation, t.val]] and not all_up.include? [true]
|
98
102
|
to_ret = nil
|
99
103
|
begin
|
100
|
-
to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |
|
101
|
-
|
104
|
+
to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |j|
|
105
|
+
j["reservationId"] == ec2_insts[[]].insts["reservationId"]
|
102
106
|
end]
|
103
107
|
rescue SocketError
|
104
108
|
print "E"
|
105
109
|
else
|
106
110
|
print "."
|
107
111
|
end
|
108
|
-
|
112
|
+
$stdout.flush
|
109
113
|
to_ret
|
110
114
|
end
|
111
115
|
end
|
@@ -113,7 +117,7 @@ module EC2Deploy
|
|
113
117
|
# XXX: No upsert operator, so we have to do this.
|
114
118
|
the_reservation_next <+ the_reservation
|
115
119
|
|
116
|
-
node_up <= ((
|
120
|
+
node_up <= (((ec2_insts * the_reservation).pairs do
|
117
121
|
if not all_up.include? [true]
|
118
122
|
the_reservation[[]].reservation["instancesSet"]["item"].map do |i|
|
119
123
|
[i, i["instanceState"]["code"] == "16"]
|
@@ -121,28 +125,27 @@ module EC2Deploy
|
|
121
125
|
end
|
122
126
|
end)[0] or [])
|
123
127
|
|
124
|
-
|
125
|
-
|
126
|
-
if node_up.find {|n| n.bool == false} == nil and node_up.find {|n| n.bool == true} != nil
|
128
|
+
all_up <+ node_up do
|
129
|
+
if node_up.find {|n1| n1.bool == false} == nil and node_up.find {|n2| n2.bool == true} != nil
|
127
130
|
if depl_idempotent [:nodes_all_up]
|
128
131
|
puts "done"
|
129
|
-
|
132
|
+
$stdout.flush
|
130
133
|
[true]
|
131
134
|
end
|
132
135
|
end
|
133
136
|
end
|
134
137
|
|
135
138
|
# XXX: Fixed port 54321
|
136
|
-
temp_node <=
|
139
|
+
temp_node <= (all_up * the_reservation_next).pairs do
|
137
140
|
break(((0..(the_reservation_next[[]].reservation["instancesSet"]["item"].size-1)).to_a.zip(the_reservation_next[[]].reservation["instancesSet"]["item"].map {|i| [i["ipAddress"], i["privateIpAddress"]]})).map {|n,ips| [n, ips[0] + ":54321", ips[1] + ":54321"]})
|
138
141
|
end
|
139
142
|
|
140
|
-
deploy_node <=
|
143
|
+
deploy_node <= (temp_node * init_dir * ruby_command).combos do |t, i, r|
|
141
144
|
if depl_idempotent [[:node_startup, t.node]]
|
142
145
|
ip = t.node.split(":")[0]
|
143
146
|
port = t.node.split(":")[1]
|
144
147
|
print "Deploying to #{ip} (#{t.uid}/#{node_count[[]].num-1})."
|
145
|
-
|
148
|
+
$stdout.flush
|
146
149
|
|
147
150
|
# Upload files and run commands.
|
148
151
|
ctr = 0
|
@@ -170,7 +173,7 @@ module EC2Deploy
|
|
170
173
|
rescue Exception
|
171
174
|
ctr += 1
|
172
175
|
print "."
|
173
|
-
|
176
|
+
$stdout.flush
|
174
177
|
sleep 10
|
175
178
|
next
|
176
179
|
end
|
@@ -180,21 +183,17 @@ module EC2Deploy
|
|
180
183
|
[t.uid, t.node]
|
181
184
|
end
|
182
185
|
end
|
183
|
-
|
184
186
|
end
|
185
187
|
|
186
188
|
bloom :all_nodes do
|
187
|
-
stdio <~ ready
|
189
|
+
stdio <~ ready {|_,s| ["Ready: #{s}"]}
|
188
190
|
# Persist ready messages
|
189
|
-
ready_tab <= ready
|
191
|
+
ready_tab <= ready {|_, s| [s]}
|
190
192
|
# Compute a count of ready messages
|
191
193
|
ready_count <= ready_tab.group(nil, count)
|
192
194
|
# Copy deploy_node into node when all nodes are up
|
193
|
-
node <=
|
194
|
-
[ready_count.num, node_count.num]).map do
|
195
|
+
node <= (ready_count * node_count).pairs(:num => :num) do
|
195
196
|
break deploy_node
|
196
197
|
end
|
197
|
-
|
198
198
|
end
|
199
|
-
|
200
199
|
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'bud/deploy/deployer'
|
2
|
+
|
3
|
+
module ForkDeployProtocol
|
4
|
+
state do
|
5
|
+
channel :child_ack, [:@loc, :node_id] => [:node_addr]
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
module ForkDeployChild
|
10
|
+
include ForkDeployProtocol
|
11
|
+
|
12
|
+
bootstrap do
|
13
|
+
child_ack <~ [[@deployer_addr, @node_id, ip_port]]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# An implementation of the Deployer that runs instances using forked local
|
18
|
+
# processes (listening on an ephemeral port).
|
19
|
+
#
|
20
|
+
# Note that this module is included in both the deployer process and in the
|
21
|
+
# deployed instances. To write code that only runs in one type of process,
|
22
|
+
# consult the ":deploy" Bud option (which is false in deployed children).
|
23
|
+
module ForkDeploy
|
24
|
+
include Deployer
|
25
|
+
include ForkDeployProtocol
|
26
|
+
|
27
|
+
def initialize(opts={})
|
28
|
+
super
|
29
|
+
@child_modules = [ForkDeployChild]
|
30
|
+
@child_pids = []
|
31
|
+
@dead_pids = []
|
32
|
+
end
|
33
|
+
|
34
|
+
bloom :child_info do
|
35
|
+
node <= child_ack {|a| [a.node_id, a.node_addr]}
|
36
|
+
node_ready <= child_ack {|a| [a.node_id]}
|
37
|
+
end
|
38
|
+
|
39
|
+
bootstrap do
|
40
|
+
return unless @options[:deploy]
|
41
|
+
|
42
|
+
Signal.trap("CHLD") do
|
43
|
+
# We receive SIGCHLD when a child process changes state; unfortunately,
|
44
|
+
# there's no easy way to tell whether the child process we're getting the
|
45
|
+
# signal for is one of ForkDeploy's children. Hence, check if any of the
|
46
|
+
# forked children have exited. We also ignore Errno::ECHILD, because
|
47
|
+
# someone else's waitpid() could easily race with us.
|
48
|
+
@child_pids.each do |c|
|
49
|
+
begin
|
50
|
+
pid = Process.waitpid(c, Process::WNOHANG)
|
51
|
+
unless pid.nil?
|
52
|
+
@dead_pids << pid
|
53
|
+
end
|
54
|
+
rescue Errno::ECHILD
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
on_shutdown do
|
60
|
+
# NB: Setting the SIGCHLD handler to "IGNORE" results in waitpid() being
|
61
|
+
# called automatically (to cleanup zombies), at least on OSX. This is not
|
62
|
+
# what we want, since it would cause a subsequent waitpid() to fail.
|
63
|
+
Signal.trap("CHLD", "DEFAULT")
|
64
|
+
pids = @child_pids - @dead_pids
|
65
|
+
pids.each do |p|
|
66
|
+
begin
|
67
|
+
Process.kill("TERM", p)
|
68
|
+
Process.waitpid(p)
|
69
|
+
rescue Errno::ESRCH
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
child_opts = @options[:deploy_child_opts]
|
75
|
+
child_opts ||= {}
|
76
|
+
deployer_addr = self.ip_port
|
77
|
+
node_count[[]].num.times do |i|
|
78
|
+
@child_pids << Bud.do_fork do
|
79
|
+
@child_modules.each do |m|
|
80
|
+
# XXX: Can this be done without "instance_eval"?
|
81
|
+
self.class.instance_eval "include #{m}"
|
82
|
+
end
|
83
|
+
child = self.class.new(child_opts)
|
84
|
+
child.instance_variable_set('@deployer_addr', deployer_addr)
|
85
|
+
child.instance_variable_set('@node_id', i)
|
86
|
+
child.run_fg
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'bud/deploy/deployer'
|
2
|
+
|
3
|
+
# An implementation of the Deployer that runs instances using the current Ruby
|
4
|
+
# process (listening on an ephemeral port). ThreadDeploy is probably not the
|
5
|
+
# best name: all the spawned instances are run by a single thread, they are just
|
6
|
+
# multiplexed via EventMachine.
|
7
|
+
#
|
8
|
+
# Note that this module is included in both the deployer process and in the
|
9
|
+
# deployed instances. To write code that only runs in one type of process,
|
10
|
+
# consult the ":deploy" Bud option (which is false in deployed children).
|
11
|
+
module ThreadDeploy
|
12
|
+
include Deployer
|
13
|
+
|
14
|
+
bootstrap do
|
15
|
+
return unless @options[:deploy]
|
16
|
+
|
17
|
+
@instances = []
|
18
|
+
on_shutdown do
|
19
|
+
@instances.each {|b| b.stop_bg}
|
20
|
+
end
|
21
|
+
|
22
|
+
print "Spawning threads"
|
23
|
+
child_opts = @options[:deploy_child_opts]
|
24
|
+
child_opts ||= {}
|
25
|
+
deployer_addr = self.ip_port
|
26
|
+
node_count[[]].num.times do |i|
|
27
|
+
b = self.class.new(child_opts)
|
28
|
+
b.instance_variable_set('@deployer_addr', deployer_addr)
|
29
|
+
b.instance_variable_set('@node_id', i)
|
30
|
+
b.run_bg
|
31
|
+
@instances << b
|
32
|
+
node << [i, b.ip_port]
|
33
|
+
node_ready << [i]
|
34
|
+
print "."
|
35
|
+
end
|
36
|
+
puts "done"
|
37
|
+
end
|
38
|
+
end
|