bud 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/README +33 -16
  2. data/bin/budplot +42 -65
  3. data/bin/budtimelines +235 -0
  4. data/bin/budvis +24 -122
  5. data/bin/rebl +1 -0
  6. data/docs/README.md +21 -10
  7. data/docs/bfs.md +4 -6
  8. data/docs/c.html +251 -0
  9. data/docs/cheat.md +45 -30
  10. data/docs/deploy.md +26 -26
  11. data/docs/getstarted.md +6 -4
  12. data/docs/visualizations.md +43 -31
  13. data/examples/chat/chat.rb +4 -9
  14. data/examples/chat/chat_server.rb +1 -8
  15. data/examples/deploy/deploy_ip_port +1 -0
  16. data/examples/deploy/keys.rb +5 -0
  17. data/examples/deploy/tokenring-ec2.rb +9 -9
  18. data/examples/deploy/{tokenring-local.rb → tokenring-fork.rb} +3 -5
  19. data/examples/deploy/tokenring-thread.rb +15 -0
  20. data/examples/deploy/tokenring.rb +25 -17
  21. data/lib/bud/aggs.rb +87 -25
  22. data/lib/bud/bud_meta.rb +48 -31
  23. data/lib/bud/bust/bust.rb +16 -15
  24. data/lib/bud/collections.rb +207 -232
  25. data/lib/bud/depanalysis.rb +1 -0
  26. data/lib/bud/deploy/countatomicdelivery.rb +8 -20
  27. data/lib/bud/deploy/deployer.rb +16 -16
  28. data/lib/bud/deploy/ec2deploy.rb +34 -35
  29. data/lib/bud/deploy/forkdeploy.rb +90 -0
  30. data/lib/bud/deploy/threaddeploy.rb +38 -0
  31. data/lib/bud/graphs.rb +103 -199
  32. data/lib/bud/joins.rb +190 -41
  33. data/lib/bud/monkeypatch.rb +84 -0
  34. data/lib/bud/rebl.rb +8 -1
  35. data/lib/bud/rewrite.rb +152 -49
  36. data/lib/bud/server.rb +1 -0
  37. data/lib/bud/state.rb +24 -10
  38. data/lib/bud/storage/dbm.rb +170 -0
  39. data/lib/bud/storage/tokyocabinet.rb +5 -1
  40. data/lib/bud/stratify.rb +6 -7
  41. data/lib/bud/viz.rb +31 -17
  42. data/lib/bud/viz_util.rb +204 -0
  43. data/lib/bud.rb +271 -244
  44. data/lib/bud.rb.orig +806 -0
  45. metadata +43 -22
  46. data/docs/bfs.raw +0 -251
  47. data/docs/diffs +0 -181
  48. data/examples/basics/out +0 -1103
  49. data/examples/basics/out.new +0 -856
  50. data/lib/bud/deploy/localdeploy.rb +0 -53
@@ -8,6 +8,7 @@ class DepAnalysis #:nodoc: all
8
8
  table :providing, [:pred, :input]
9
9
  table :depends_tc, [:head, :body, :via, :neg, :temporal]
10
10
  table :underspecified, [:pred, :input]
11
+
11
12
 
12
13
  table :source, [:pred]
13
14
  table :sink, [:pred]
@@ -10,41 +10,29 @@ module CountAtomicDelivery # :nodoc: all
10
10
  channel :atomic_data_chan, [:@loc, :tuple]
11
11
  table :atomic_data_recv, [:loc, :tuple]
12
12
 
13
- scratch :atomic_count, [:loc, :cnt]
14
13
  channel :atomic_count_chan,[:@loc, :cnt]
15
14
  table :atomic_count_recv, [] => [:cnt]
16
15
 
17
16
  scratch :atomic_recv_count, [:loc] => [:cnt]
18
- scratch :atomic_data_atomic, [:loc, :tuple]
17
+ scratch :atomic_data, [:loc, :tuple]
19
18
  scratch :atomic_data_out, [:tuple]
20
19
  end
21
20
 
22
21
  bloom :countatomicdelivery do
23
- atomic_count <= atomic_data_in.group([:loc], count)
22
+ temp :atomic_count <= atomic_data_in.group([:loc], count)
24
23
 
25
24
  atomic_data_chan <~ atomic_data_in
26
25
  atomic_count_chan <~ atomic_count
27
26
 
28
- atomic_count_recv <= atomic_count_chan.map {|c| [c.cnt]}
27
+ atomic_count_recv <= atomic_count_chan {|c| [c.cnt]}
29
28
  atomic_data_recv <= atomic_data_chan
30
29
 
31
30
  atomic_recv_count <= atomic_data_recv.group([:loc], count)
32
31
 
33
- # atomic_data_atomic <= join([atomic_recv_count, atomic_count_recv,
34
- # atomic_data_recv],
35
- # [atomic_recv_count.cnt,
36
- # atomic_count_recv.cnt]).map{ |rc, cr, d| d }
37
-
38
- atomic_data_atomic <= ((if atomic_recv_count.first and
39
- atomic_count_recv.first and
40
- atomic_recv_count.first.cnt ==
41
- atomic_count_recv.first.cnt
42
- atomic_data_recv
43
- end) or [])
44
-
45
- # Commented out due to bug #85.
46
- # atomic_data_recv <- atomic_data_atomic
47
- # Idempotence hack inserted due to bug #85.
48
- atomic_data_out <= atomic_data_atomic.map {|a| [a.tuple] if depl_idempotent [[:atomic_data_out, a]]}
32
+ atomic_data <= (atomic_recv_count * atomic_count_recv * atomic_data_recv).combos(atomic_recv_count.cnt => atomic_count_recv.cnt) {|rc, cr, d| d}
33
+
34
+ atomic_data_recv <- atomic_data
35
+
36
+ atomic_data_out <= atomic_data {|a| [a.tuple]}
49
37
  end
50
38
  end
@@ -1,6 +1,5 @@
1
1
  require 'rubygems'
2
2
  require 'bud'
3
- require 'open-uri'
4
3
  require 'bud/deploy/countatomicdelivery'
5
4
 
6
5
  class Module
@@ -14,8 +13,12 @@ module Deployer # :nodoc: all
14
13
  include CountAtomicDelivery
15
14
 
16
15
  state do
17
- table :node, [:uid] => [:node]
16
+ table :node, [:uid] => [:addr]
18
17
  table :node_count, [] => [:num]
18
+ # At the deployer node, this collection will contain a fact for the first
19
+ # tick in which the given node is ready
20
+ scratch :node_ready, [:uid]
21
+
19
22
  table :initial_data, [:uid, :pred, :data]
20
23
  channel :dont_care, [:@loc]
21
24
  table :dead, [:dead]
@@ -33,11 +36,9 @@ module Deployer # :nodoc: all
33
36
  end
34
37
  end
35
38
 
36
- def initialize(opt = {})
39
+ def initialize(opt={})
37
40
  super
38
- if opt[:deploy]
39
- do_deploystrap
40
- end
41
+ do_deploystrap if opt[:deploy]
41
42
  end
42
43
 
43
44
  # Distribute the EDB to each node.
@@ -46,22 +47,21 @@ module Deployer # :nodoc: all
46
47
  # before any messages are received. In order to fix this, we would probably
47
48
  # need to globally synchronize to ensure that "timestamp 0" gets "fully
48
49
  # evaluated" before any messages can be sent.
49
-
50
50
  bloom :distribute_data do
51
- atomic_data_in <= join([node, initial_data],
52
- [node.uid, initial_data.uid]).map do |n, i|
53
- [n.node, [i.pred, i.data]] if depl_idempotent [[n.node, i.pred, i.data]]
51
+ atomic_data_in <= (node_ready * node * initial_data).combos(node_ready.uid => node.uid, node.uid => initial_data.uid) do |nr, n, i|
52
+ [n.addr, [i.pred, i.data]]
54
53
  end
55
54
 
56
55
  # Add all tuples at once.
57
- dont_care <~ atomic_data_out.map do |a|
58
- if depl_idempotent a
59
- a.tuple[1].map do |d|
60
- eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
61
- end
62
- [ip_port]
56
+ dont_care <~ atomic_data_out do |a|
57
+ a.tuple[1].map do |d|
58
+ eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
63
59
  end
60
+ [ip_port]
64
61
  end
62
+ end
65
63
 
64
+ bloom :print_ready do
65
+ stdio <~ node_ready {|nr| ["Child node ready: #{nr.uid}"]}
66
66
  end
67
67
  end
@@ -6,7 +6,11 @@ require 'net/scp'
6
6
  require 'bud'
7
7
  require 'bud/deploy/deployer'
8
8
 
9
- # logic to deploy Bloom programs on EC2
9
+ # Logic to deploy Bloom programs on EC2
10
+ # TODO:
11
+ # * add support for deploy_child_opts
12
+ # * avoid blocking I/O calls
13
+ # * emit "node_ready" event when each node is ready
10
14
  module EC2Deploy
11
15
  include Deployer
12
16
 
@@ -46,23 +50,23 @@ module EC2Deploy
46
50
  unless @options[:deploy]
47
51
  # Send message to the deployer telling 'em we's up.
48
52
  File.open("deploy_ip_port", "r") do |f|
49
- ready <~ [[f.readline.chop, ip_port]]
53
+ ready <~ [[f.readline.rstrip, ip_port]]
50
54
  end
51
55
  end
52
56
  end
53
57
 
54
58
  bloom :spinup do
55
- ec2_conn <= join([access_key_id, secret_access_key]).map do
59
+ ec2_conn <= (access_key_id * secret_access_key).pairs do
56
60
  if depl_idempotent [:ec2_comm]
57
61
  [AWS::EC2::Base.new(:access_key_id => access_key_id[[]].key,
58
62
  :secret_access_key => secret_access_key[[]].key)]
59
63
  end
60
64
  end
61
65
 
62
- ec2_insts <= join([image_id, node_count, key_name, ec2_conn]).map do
66
+ ec2_insts <= (image_id * node_count * key_name * ec2_conn).combos do
63
67
  if depl_idempotent [:ec2_insts]
64
68
  print "Starting up EC2 instances"
65
- STDOUT.flush
69
+ $stdout.flush
66
70
  # First, we create the security group.
67
71
  begin
68
72
  ec2_conn[[]].conn.create_security_group(:group_name => "bud", :group_description => "bud")
@@ -70,18 +74,18 @@ module EC2Deploy
70
74
  # Group already exists; ok, maybe we created it previously.
71
75
  else
72
76
  # Add SSH permission.
73
- ec2_conn[[]].conn.authorize_security_group_ingress( :group_name => "bud",
74
- :ip_protocol => "tcp",
75
- :from_port => 22,
76
- :to_port => 22,
77
- :cidr_ip => "0.0.0.0/0" )
77
+ ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
78
+ :ip_protocol => "tcp",
79
+ :from_port => 22,
80
+ :to_port => 22,
81
+ :cidr_ip => "0.0.0.0/0")
78
82
  # Add unlimited UDP permission from any node not in the security group.
79
83
  # XXX: make this more restrictive?
80
- ec2_conn[[]].conn.authorize_security_group_ingress( :group_name => "bud",
81
- :ip_protocol => "udp",
82
- :from_port => 0,
83
- :to_port => 65535,
84
- :cidr_ip => "0.0.0.0/0" )
84
+ ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
85
+ :ip_protocol => "udp",
86
+ :from_port => 0,
87
+ :to_port => 65535,
88
+ :cidr_ip => "0.0.0.0/0")
85
89
  end
86
90
 
87
91
  # Finally, start up the instances.
@@ -93,19 +97,19 @@ module EC2Deploy
93
97
  end
94
98
  end
95
99
 
96
- the_reservation <= join([spinup_timer, ec2_conn, ec2_insts]).map do |t,c,i|
100
+ the_reservation <= (spinup_timer * ec2_conn * ec2_insts).combos do |t,c,i|
97
101
  if depl_idempotent [[:the_reservation, t.val]] and not all_up.include? [true]
98
102
  to_ret = nil
99
103
  begin
100
- to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |i|
101
- i["reservationId"] == ec2_insts[[]].insts["reservationId"]
104
+ to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |j|
105
+ j["reservationId"] == ec2_insts[[]].insts["reservationId"]
102
106
  end]
103
107
  rescue SocketError
104
108
  print "E"
105
109
  else
106
110
  print "."
107
111
  end
108
- STDOUT.flush
112
+ $stdout.flush
109
113
  to_ret
110
114
  end
111
115
  end
@@ -113,7 +117,7 @@ module EC2Deploy
113
117
  # XXX: No upsert operator, so we have to do this.
114
118
  the_reservation_next <+ the_reservation
115
119
 
116
- node_up <= ((join([ec2_insts, the_reservation]).map do
120
+ node_up <= (((ec2_insts * the_reservation).pairs do
117
121
  if not all_up.include? [true]
118
122
  the_reservation[[]].reservation["instancesSet"]["item"].map do |i|
119
123
  [i, i["instanceState"]["code"] == "16"]
@@ -121,28 +125,27 @@ module EC2Deploy
121
125
  end
122
126
  end)[0] or [])
123
127
 
124
-
125
- all_up <+ node_up.map do
126
- if node_up.find {|n| n.bool == false} == nil and node_up.find {|n| n.bool == true} != nil
128
+ all_up <+ node_up do
129
+ if node_up.find {|n1| n1.bool == false} == nil and node_up.find {|n2| n2.bool == true} != nil
127
130
  if depl_idempotent [:nodes_all_up]
128
131
  puts "done"
129
- STDOUT.flush
132
+ $stdout.flush
130
133
  [true]
131
134
  end
132
135
  end
133
136
  end
134
137
 
135
138
  # XXX: Fixed port 54321
136
- temp_node <= join([all_up, the_reservation_next]).map do
139
+ temp_node <= (all_up * the_reservation_next).pairs do
137
140
  break(((0..(the_reservation_next[[]].reservation["instancesSet"]["item"].size-1)).to_a.zip(the_reservation_next[[]].reservation["instancesSet"]["item"].map {|i| [i["ipAddress"], i["privateIpAddress"]]})).map {|n,ips| [n, ips[0] + ":54321", ips[1] + ":54321"]})
138
141
  end
139
142
 
140
- deploy_node <= join([temp_node, init_dir, ruby_command]).map do |t, i, r|
143
+ deploy_node <= (temp_node * init_dir * ruby_command).combos do |t, i, r|
141
144
  if depl_idempotent [[:node_startup, t.node]]
142
145
  ip = t.node.split(":")[0]
143
146
  port = t.node.split(":")[1]
144
147
  print "Deploying to #{ip} (#{t.uid}/#{node_count[[]].num-1})."
145
- STDOUT.flush
148
+ $stdout.flush
146
149
 
147
150
  # Upload files and run commands.
148
151
  ctr = 0
@@ -170,7 +173,7 @@ module EC2Deploy
170
173
  rescue Exception
171
174
  ctr += 1
172
175
  print "."
173
- STDOUT.flush
176
+ $stdout.flush
174
177
  sleep 10
175
178
  next
176
179
  end
@@ -180,21 +183,17 @@ module EC2Deploy
180
183
  [t.uid, t.node]
181
184
  end
182
185
  end
183
-
184
186
  end
185
187
 
186
188
  bloom :all_nodes do
187
- stdio <~ ready.map {|_,s| ["Ready: #{s}"]}
189
+ stdio <~ ready {|_,s| ["Ready: #{s}"]}
188
190
  # Persist ready messages
189
- ready_tab <= ready.map {|_, s| [s]}
191
+ ready_tab <= ready {|_, s| [s]}
190
192
  # Compute a count of ready messages
191
193
  ready_count <= ready_tab.group(nil, count)
192
194
  # Copy deploy_node into node when all nodes are up
193
- node <= join([ready_count, node_count],
194
- [ready_count.num, node_count.num]).map do
195
+ node <= (ready_count * node_count).pairs(:num => :num) do
195
196
  break deploy_node
196
197
  end
197
-
198
198
  end
199
-
200
199
  end
@@ -0,0 +1,90 @@
1
+ require 'bud/deploy/deployer'
2
+
3
+ module ForkDeployProtocol
4
+ state do
5
+ channel :child_ack, [:@loc, :node_id] => [:node_addr]
6
+ end
7
+ end
8
+
9
+ module ForkDeployChild
10
+ include ForkDeployProtocol
11
+
12
+ bootstrap do
13
+ child_ack <~ [[@deployer_addr, @node_id, ip_port]]
14
+ end
15
+ end
16
+
17
+ # An implementation of the Deployer that runs instances using forked local
18
+ # processes (listening on an ephemeral port).
19
+ #
20
+ # Note that this module is included in both the deployer process and in the
21
+ # deployed instances. To write code that only runs in one type of process,
22
+ # consult the ":deploy" Bud option (which is false in deployed children).
23
+ module ForkDeploy
24
+ include Deployer
25
+ include ForkDeployProtocol
26
+
27
+ def initialize(opts={})
28
+ super
29
+ @child_modules = [ForkDeployChild]
30
+ @child_pids = []
31
+ @dead_pids = []
32
+ end
33
+
34
+ bloom :child_info do
35
+ node <= child_ack {|a| [a.node_id, a.node_addr]}
36
+ node_ready <= child_ack {|a| [a.node_id]}
37
+ end
38
+
39
+ bootstrap do
40
+ return unless @options[:deploy]
41
+
42
+ Signal.trap("CHLD") do
43
+ # We receive SIGCHLD when a child process changes state; unfortunately,
44
+ # there's no easy way to tell whether the child process we're getting the
45
+ # signal for is one of ForkDeploy's children. Hence, check if any of the
46
+ # forked children have exited. We also ignore Errno::ECHILD, because
47
+ # someone else's waitpid() could easily race with us.
48
+ @child_pids.each do |c|
49
+ begin
50
+ pid = Process.waitpid(c, Process::WNOHANG)
51
+ unless pid.nil?
52
+ @dead_pids << pid
53
+ end
54
+ rescue Errno::ECHILD
55
+ end
56
+ end
57
+ end
58
+
59
+ on_shutdown do
60
+ # NB: Setting the SIGCHLD handler to "IGNORE" results in waitpid() being
61
+ # called automatically (to cleanup zombies), at least on OSX. This is not
62
+ # what we want, since it would cause a subsequent waitpid() to fail.
63
+ Signal.trap("CHLD", "DEFAULT")
64
+ pids = @child_pids - @dead_pids
65
+ pids.each do |p|
66
+ begin
67
+ Process.kill("TERM", p)
68
+ Process.waitpid(p)
69
+ rescue Errno::ESRCH
70
+ end
71
+ end
72
+ end
73
+
74
+ child_opts = @options[:deploy_child_opts]
75
+ child_opts ||= {}
76
+ deployer_addr = self.ip_port
77
+ node_count[[]].num.times do |i|
78
+ @child_pids << Bud.do_fork do
79
+ @child_modules.each do |m|
80
+ # XXX: Can this be done without "instance_eval"?
81
+ self.class.instance_eval "include #{m}"
82
+ end
83
+ child = self.class.new(child_opts)
84
+ child.instance_variable_set('@deployer_addr', deployer_addr)
85
+ child.instance_variable_set('@node_id', i)
86
+ child.run_fg
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,38 @@
1
+ require 'bud/deploy/deployer'
2
+
3
+ # An implementation of the Deployer that runs instances using the current Ruby
4
+ # process (listening on an ephemeral port). ThreadDeploy is probably not the
5
+ # best name: all the spawned instances are run by a single thread, they are just
6
+ # multiplexed via EventMachine.
7
+ #
8
+ # Note that this module is included in both the deployer process and in the
9
+ # deployed instances. To write code that only runs in one type of process,
10
+ # consult the ":deploy" Bud option (which is false in deployed children).
11
+ module ThreadDeploy
12
+ include Deployer
13
+
14
+ bootstrap do
15
+ return unless @options[:deploy]
16
+
17
+ @instances = []
18
+ on_shutdown do
19
+ @instances.each {|b| b.stop_bg}
20
+ end
21
+
22
+ print "Spawning threads"
23
+ child_opts = @options[:deploy_child_opts]
24
+ child_opts ||= {}
25
+ deployer_addr = self.ip_port
26
+ node_count[[]].num.times do |i|
27
+ b = self.class.new(child_opts)
28
+ b.instance_variable_set('@deployer_addr', deployer_addr)
29
+ b.instance_variable_set('@node_id', i)
30
+ b.run_bg
31
+ @instances << b
32
+ node << [i, b.ip_port]
33
+ node_ready << [i]
34
+ print "."
35
+ end
36
+ puts "done"
37
+ end
38
+ end