bud 0.0.3 → 0.0.4

data/lib/bud/depanalysis.rb

@@ -8,6 +8,7 @@ class DepAnalysis #:nodoc: all
     table :providing, [:pred, :input]
     table :depends_tc, [:head, :body, :via, :neg, :temporal]
     table :underspecified, [:pred, :input]
+    
 
     table :source, [:pred]
     table :sink, [:pred]

data/lib/bud/deploy/countatomicdelivery.rb

@@ -10,41 +10,29 @@ module CountAtomicDelivery # :nodoc: all
     channel :atomic_data_chan, [:@loc, :tuple]
     table :atomic_data_recv, [:loc, :tuple]
 
-    scratch :atomic_count, [:loc, :cnt]
     channel :atomic_count_chan,[:@loc, :cnt]
     table :atomic_count_recv, [] => [:cnt]
 
     scratch :atomic_recv_count, [:loc] => [:cnt]
-    scratch :atomic_data_atomic, [:loc, :tuple]
+    scratch :atomic_data, [:loc, :tuple]
     scratch :atomic_data_out, [:tuple]
   end
 
   bloom :countatomicdelivery do
-    atomic_count <= atomic_data_in.group([:loc], count)
+    temp :atomic_count <= atomic_data_in.group([:loc], count)
 
     atomic_data_chan <~ atomic_data_in
     atomic_count_chan <~ atomic_count
 
-    atomic_count_recv <= atomic_count_chan.map {|c| [c.cnt]}
+    atomic_count_recv <= atomic_count_chan {|c| [c.cnt]}
     atomic_data_recv <= atomic_data_chan
 
     atomic_recv_count <= atomic_data_recv.group([:loc], count)
 
-#    atomic_data_atomic <= join([atomic_recv_count, atomic_count_recv,
-#                                atomic_data_recv],
-#                               [atomic_recv_count.cnt,
-#                                atomic_count_recv.cnt]).map{ |rc, cr, d| d }
-
-    atomic_data_atomic <= ((if atomic_recv_count.first and
-                                atomic_count_recv.first and
-                                atomic_recv_count.first.cnt ==
-                                atomic_count_recv.first.cnt
-                              atomic_data_recv
-                            end) or [])
-
-    # Commented out due to bug #85.
-    # atomic_data_recv <- atomic_data_atomic
-    # Idempotence hack inserted due to bug #85.
-    atomic_data_out <= atomic_data_atomic.map {|a| [a.tuple] if depl_idempotent [[:atomic_data_out, a]]}
+    atomic_data <= (atomic_recv_count * atomic_count_recv * atomic_data_recv).combos(atomic_recv_count.cnt => atomic_count_recv.cnt) {|rc, cr, d| d}
+
+    atomic_data_recv <- atomic_data
+
+    atomic_data_out <= atomic_data {|a| [a.tuple]}
   end
 end

data/lib/bud/deploy/deployer.rb

@@ -1,6 +1,5 @@
 require 'rubygems'
 require 'bud'
-require 'open-uri'
 require 'bud/deploy/countatomicdelivery'
 
 class Module
@@ -14,8 +13,12 @@ module Deployer # :nodoc: all
   include CountAtomicDelivery
 
   state do
-    table :node, [:uid] => [:node]
+    table :node, [:uid] => [:addr]
     table :node_count, [] => [:num]
+    # At the deployer node, this collection will contain a fact for the first
+    # tick in which the given node is ready
+    scratch :node_ready, [:uid]
+
     table :initial_data, [:uid, :pred, :data]
     channel :dont_care, [:@loc]
     table :dead, [:dead]
@@ -33,11 +36,9 @@ module Deployer # :nodoc: all
     end
   end
 
-  def initialize(opt = {})
+  def initialize(opt={})
     super
-    if opt[:deploy]
-      do_deploystrap
-    end
+    do_deploystrap if opt[:deploy]
   end
 
   # Distribute the EDB to each node.
@@ -46,22 +47,21 @@ module Deployer # :nodoc: all
   # before any messages are received.  In order to fix this, we would probably
   # need to globally synchronize to ensure that "timestamp 0" gets "fully
   # evaluated" before any messages can be sent.
-
   bloom :distribute_data do
-    atomic_data_in <= join([node, initial_data],
-                           [node.uid, initial_data.uid]).map do |n, i|
-      [n.node, [i.pred, i.data]] if depl_idempotent [[n.node, i.pred, i.data]]
+    atomic_data_in <= (node_ready * node * initial_data).combos(node_ready.uid => node.uid, node.uid => initial_data.uid) do |nr, n, i|
+      [n.addr, [i.pred, i.data]]
     end
 
     # Add all tuples at once.
-    dont_care <~ atomic_data_out.map do |a|
-      if depl_idempotent a
-        a.tuple[1].map do |d|
-          eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
-        end
-        [ip_port]
+    dont_care <~ atomic_data_out do |a|
+      a.tuple[1].map do |d|
+        eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
       end
+      [ip_port]
     end
+  end
 
+  bloom :print_ready do
+    stdio <~ node_ready {|nr| ["Child node ready: #{nr.uid}"]}
   end
 end

data/lib/bud/deploy/ec2deploy.rb

@@ -6,7 +6,11 @@ require 'net/scp'
 require 'bud'
 require 'bud/deploy/deployer'
 
-# logic to deploy Bloom programs on EC2
+# Logic to deploy Bloom programs on EC2
+# TODO:
+#  * add support for deploy_child_opts
+#  * avoid blocking I/O calls
+#  * emit "node_ready" event when each node is ready
 module EC2Deploy
   include Deployer
 
@@ -46,23 +50,23 @@ module EC2Deploy
     unless @options[:deploy]
       # Send message to the deployer telling 'em we's up.
       File.open("deploy_ip_port", "r") do |f|
-        ready <~ [[f.readline.chop, ip_port]]
+        ready <~ [[f.readline.rstrip, ip_port]]
       end
     end
   end
 
   bloom :spinup do
-    ec2_conn <= join([access_key_id, secret_access_key]).map do
+    ec2_conn <= (access_key_id * secret_access_key).pairs do
       if depl_idempotent [:ec2_comm]
         [AWS::EC2::Base.new(:access_key_id => access_key_id[[]].key,
                             :secret_access_key => secret_access_key[[]].key)]
       end
     end
 
-    ec2_insts <= join([image_id, node_count, key_name, ec2_conn]).map do
+    ec2_insts <= (image_id * node_count * key_name * ec2_conn).combos do
       if depl_idempotent [:ec2_insts]
         print "Starting up EC2 instances"
-        STDOUT.flush
+        $stdout.flush
         # First, we create the security group.
         begin
           ec2_conn[[]].conn.create_security_group(:group_name => "bud", :group_description => "bud")
@@ -70,18 +74,18 @@ module EC2Deploy
           # Group already exists; ok, maybe we created it previously.
         else
           # Add SSH permission.
-          ec2_conn[[]].conn.authorize_security_group_ingress( :group_name => "bud",
-                                                              :ip_protocol => "tcp",
-                                                              :from_port => 22,
-                                                              :to_port => 22,
-                                                              :cidr_ip => "0.0.0.0/0" )
+          ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
+                                                             :ip_protocol => "tcp",
+                                                             :from_port => 22,
+                                                             :to_port => 22,
+                                                             :cidr_ip => "0.0.0.0/0")
           # Add unlimited UDP permission from any node not in the security group.
           # XXX: make this more restrictive?
-          ec2_conn[[]].conn.authorize_security_group_ingress( :group_name => "bud",
-                                                              :ip_protocol => "udp",
-                                                              :from_port => 0,
-                                                              :to_port => 65535,
-                                                              :cidr_ip => "0.0.0.0/0" )
+          ec2_conn[[]].conn.authorize_security_group_ingress(:group_name => "bud",
+                                                             :ip_protocol => "udp",
+                                                             :from_port => 0,
+                                                             :to_port => 65535,
+                                                             :cidr_ip => "0.0.0.0/0")
         end
 
         # Finally, start up the instances.
@@ -93,19 +97,19 @@ module EC2Deploy
       end
     end
 
-    the_reservation <= join([spinup_timer, ec2_conn, ec2_insts]).map do |t,c,i|
+    the_reservation <= (spinup_timer * ec2_conn * ec2_insts).combos do |t,c,i|
       if depl_idempotent [[:the_reservation, t.val]] and not all_up.include? [true]
         to_ret = nil
         begin
-          to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |i|
-                      i["reservationId"] == ec2_insts[[]].insts["reservationId"]
+          to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |j|
+                      j["reservationId"] == ec2_insts[[]].insts["reservationId"]
                     end]
         rescue SocketError
           print "E"
         else
           print "."
         end
-        STDOUT.flush
+        $stdout.flush
         to_ret
       end
     end
@@ -113,7 +117,7 @@ module EC2Deploy
     # XXX: No upsert operator, so we have to do this.
     the_reservation_next <+ the_reservation
 
-    node_up <= ((join([ec2_insts, the_reservation]).map do
+    node_up <= (((ec2_insts * the_reservation).pairs do
                    if not all_up.include? [true]
                      the_reservation[[]].reservation["instancesSet"]["item"].map do |i|
                        [i, i["instanceState"]["code"] == "16"]
@@ -121,28 +125,27 @@ module EC2Deploy
                    end
                  end)[0] or [])
 
-
-    all_up <+ node_up.map do
-      if node_up.find {|n| n.bool == false} == nil and node_up.find {|n| n.bool == true} != nil
+    all_up <+ node_up do
+      if node_up.find {|n1| n1.bool == false} == nil and node_up.find {|n2| n2.bool == true} != nil
         if depl_idempotent [:nodes_all_up]
           puts "done"
-          STDOUT.flush
+          $stdout.flush
           [true]
         end
       end
     end
 
     # XXX: Fixed port 54321
-    temp_node <= join([all_up, the_reservation_next]).map do
+    temp_node <= (all_up * the_reservation_next).pairs do
       break(((0..(the_reservation_next[[]].reservation["instancesSet"]["item"].size-1)).to_a.zip(the_reservation_next[[]].reservation["instancesSet"]["item"].map {|i| [i["ipAddress"], i["privateIpAddress"]]})).map {|n,ips| [n, ips[0] + ":54321", ips[1] + ":54321"]})
     end
 
-    deploy_node <= join([temp_node, init_dir, ruby_command]).map do |t, i, r|
+    deploy_node <= (temp_node * init_dir * ruby_command).combos do |t, i, r|
       if depl_idempotent [[:node_startup, t.node]]
         ip = t.node.split(":")[0]
         port = t.node.split(":")[1]
         print "Deploying to #{ip} (#{t.uid}/#{node_count[[]].num-1})."
-        STDOUT.flush
+        $stdout.flush
 
         # Upload files and run commands.
         ctr = 0
@@ -170,7 +173,7 @@ module EC2Deploy
           rescue Exception
             ctr += 1
             print "."
-            STDOUT.flush
+            $stdout.flush
             sleep 10
             next
           end
@@ -180,21 +183,17 @@ module EC2Deploy
         [t.uid, t.node]
       end
     end
-
   end
 
   bloom :all_nodes do
-    stdio <~ ready.map {|_,s| ["Ready: #{s}"]}
+    stdio <~ ready {|_,s| ["Ready: #{s}"]}
     # Persist ready messages
-    ready_tab <= ready.map {|_, s| [s]}
+    ready_tab <= ready {|_, s| [s]}
     # Compute a count of ready messages
     ready_count <= ready_tab.group(nil, count)
     # Copy deploy_node into node when all nodes are up
-    node <= join([ready_count, node_count],
-                 [ready_count.num, node_count.num]).map do
+    node <= (ready_count * node_count).pairs(:num => :num) do
       break deploy_node
     end
-
   end
-
 end

data/lib/bud/deploy/forkdeploy.rb

@@ -0,0 +1,90 @@
+require 'bud/deploy/deployer'
+
+module ForkDeployProtocol
+  state do
+    channel :child_ack, [:@loc, :node_id] => [:node_addr]
+  end
+end
+
+module ForkDeployChild
+  include ForkDeployProtocol
+
+  bootstrap do
+    child_ack <~ [[@deployer_addr, @node_id, ip_port]]
+  end
+end
+
+# An implementation of the Deployer that runs instances using forked local
+# processes (listening on an ephemeral port).
+#
+# Note that this module is included in both the deployer process and in the
+# deployed instances. To write code that only runs in one type of process,
+# consult the ":deploy" Bud option (which is false in deployed children).
+module ForkDeploy
+  include Deployer
+  include ForkDeployProtocol
+
+  def initialize(opts={})
+    super
+    @child_modules = [ForkDeployChild]
+    @child_pids = []
+    @dead_pids = []
+  end
+
+  bloom :child_info do
+    node <= child_ack {|a| [a.node_id, a.node_addr]}
+    node_ready <= child_ack {|a| [a.node_id]}
+  end
+
+  bootstrap do
+    return unless @options[:deploy]
+
+    Signal.trap("CHLD") do
+      # We receive SIGCHLD when a child process changes state; unfortunately,
+      # there's no easy way to tell whether the child process we're getting the
+      # signal for is one of ForkDeploy's children. Hence, check if any of the
+      # forked children have exited. We also ignore Errno::ECHILD, because
+      # someone else's waitpid() could easily race with us.
+      @child_pids.each do |c|
+        begin
+          pid = Process.waitpid(c, Process::WNOHANG)
+          unless pid.nil?
+            @dead_pids << pid
+          end
+        rescue Errno::ECHILD
+        end
+      end
+    end
+
+    on_shutdown do
+      # NB: Setting the SIGCHLD handler to "IGNORE" results in waitpid() being
+      # called automatically (to cleanup zombies), at least on OSX. This is not
+      # what we want, since it would cause a subsequent waitpid() to fail.
+      Signal.trap("CHLD", "DEFAULT")
+      pids = @child_pids - @dead_pids
+      pids.each do |p|
+        begin
+          Process.kill("TERM", p)
+          Process.waitpid(p)
+        rescue Errno::ESRCH
+        end
+      end
+    end
+
+    child_opts = @options[:deploy_child_opts]
+    child_opts ||= {}
+    deployer_addr = self.ip_port
+    node_count[[]].num.times do |i|
+      @child_pids << Bud.do_fork do
+        @child_modules.each do |m|
+          # XXX: Can this be done without "instance_eval"?
+          self.class.instance_eval "include #{m}"
+        end
+        child = self.class.new(child_opts)
+        child.instance_variable_set('@deployer_addr', deployer_addr)
+        child.instance_variable_set('@node_id', i)
+        child.run_fg
+      end
+    end
+  end
+end

data/lib/bud/deploy/threaddeploy.rb

@@ -0,0 +1,38 @@
+require 'bud/deploy/deployer'
+
+# An implementation of the Deployer that runs instances using the current Ruby
+# process (listening on an ephemeral port). ThreadDeploy is probably not the
+# best name: all the spawned instances are run by a single thread, they are just
+# multiplexed via EventMachine.
+#
+# Note that this module is included in both the deployer process and in the
+# deployed instances. To write code that only runs in one type of process,
+# consult the ":deploy" Bud option (which is false in deployed children).
+module ThreadDeploy
+  include Deployer
+
+  bootstrap do
+    return unless @options[:deploy]
+
+    @instances = []
+    on_shutdown do
+      @instances.each {|b| b.stop_bg}
+    end
+
+    print "Spawning threads"
+    child_opts = @options[:deploy_child_opts]
+    child_opts ||= {}
+    deployer_addr = self.ip_port
+    node_count[[]].num.times do |i|
+      b = self.class.new(child_opts)
+      b.instance_variable_set('@deployer_addr', deployer_addr)
+      b.instance_variable_set('@node_id', i)
+      b.run_bg
+      @instances << b
+      node << [i, b.ip_port]
+      node_ready << [i]
+      print "."
+    end
+    puts "done"
+  end
+end