bud 0.0.2

data/lib/bud/depanalysis.rb

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'bud'
+
+class DepAnalysis #:nodoc: all
+  include Bud
+
+  state do
+    table :providing, [:pred, :input]
+    table :depends_tc, [:head, :body, :via, :neg, :temporal]
+    table :underspecified, [:pred, :input]
+
+    table :source, [:pred]
+    table :sink, [:pred]
+  end
+
+  def declaration
+    strata[0] = lambda {
+      source <= providing do |p|
+        if p.input and !depends_tc.map{|d| d.head}.include? p.pred
+          [p.pred]
+        end
+      end
+
+      sink <= providing do |p|
+        if !p.input and !depends_tc.map{|d| d.body}.include? p.pred
+          [p.pred]
+        end
+      end
+
+      underspecified <= providing do |p|
+        if p.input
+          unless depends_tc.map{|d| d.body if d.head != d.body}.include? p.pred
+            [p.pred, true]
+          end
+        else
+          unless depends_tc.map{|d| d.head if d.head != d.body}.include? p.pred
+            [p.pred, false]
+          end
+        end
+      end
+    }
+  end
+end
+

data/lib/bud/deploy/countatomicdelivery.rb

@@ -0,0 +1,50 @@
+require 'rubygems'
+require 'bud'
+
+# XXX: delivery modules are a massive hack.  we need to think about
+# aspect-oriented programming here, or allow users to extend the definitions of
+# existing table types
+module CountAtomicDelivery # :nodoc: all
+  state do
+    scratch :atomic_data_in, [:loc, :tuple]
+    channel :atomic_data_chan, [:@loc, :tuple]
+    table :atomic_data_recv, [:loc, :tuple]
+
+    scratch :atomic_count, [:loc, :cnt]
+    channel :atomic_count_chan,[:@loc, :cnt]
+    table :atomic_count_recv, [] => [:cnt]
+
+    scratch :atomic_recv_count, [:loc] => [:cnt]
+    scratch :atomic_data_atomic, [:loc, :tuple]
+    scratch :atomic_data_out, [:tuple]
+  end
+
+  bloom :countatomicdelivery do
+    atomic_count <= atomic_data_in.group([:loc], count)
+
+    atomic_data_chan <~ atomic_data_in
+    atomic_count_chan <~ atomic_count
+
+    atomic_count_recv <= atomic_count_chan.map {|c| [c.cnt]}
+    atomic_data_recv <= atomic_data_chan
+
+    atomic_recv_count <= atomic_data_recv.group([:loc], count)
+
+#    atomic_data_atomic <= join([atomic_recv_count, atomic_count_recv,
+#                                atomic_data_recv],
+#                               [atomic_recv_count.cnt,
+#                                atomic_count_recv.cnt]).map{ |rc, cr, d| d }
+
+    atomic_data_atomic <= ((if atomic_recv_count.first and
+                                atomic_count_recv.first and
+                                atomic_recv_count.first.cnt ==
+                                atomic_count_recv.first.cnt
+                              atomic_data_recv
+                            end) or [])
+
+    # Commented out due to bug #85.
+    # atomic_data_recv <- atomic_data_atomic
+    # Idempotence hack inserted due to bug #85.
+    atomic_data_out <= atomic_data_atomic.map {|a| [a.tuple] if depl_idempotent [[:atomic_data_out, a]]}
+  end
+end

data/lib/bud/deploy/deployer.rb

@@ -0,0 +1,67 @@
+require 'rubygems'
+require 'bud'
+require 'open-uri'
+require 'bud/deploy/countatomicdelivery'
+
+class Module
+  def deploystrap(&block)
+    meth_name = "__deploystrap__#{Module.get_class_name(self)}".to_sym
+    define_method(meth_name, &block)
+  end
+end
+
+module Deployer # :nodoc: all
+  include CountAtomicDelivery
+
+  state do
+    table :node, [:uid] => [:node]
+    table :node_count, [] => [:num]
+    table :initial_data, [:uid, :pred, :data]
+    channel :dont_care, [:@loc]
+    table :dead, [:dead]
+  end
+
+  def depl_idempotent(r) (dead.include? r) ? false : dead.insert(r) end
+
+  def do_deploystrap
+    self.class.ancestors.each do |anc|
+      anc.instance_methods(false).each do |m|
+        if /^__deploystrap__/.match m
+          self.method(m.to_sym).call
+        end
+      end
+    end
+  end
+
+  def initialize(opt = {})
+    super
+    if opt[:deploy]
+      do_deploystrap
+    end
+  end
+
+  # Distribute the EDB to each node.
+  #
+  # XXX: this may break coordination protocols that assume the EDB is present
+  # before any messages are received.  In order to fix this, we would probably
+  # need to globally synchronize to ensure that "timestamp 0" gets "fully
+  # evaluated" before any messages can be sent.
+
+  bloom :distribute_data do
+    atomic_data_in <= join([node, initial_data],
+                           [node.uid, initial_data.uid]).map do |n, i|
+      [n.node, [i.pred, i.data]] if depl_idempotent [[n.node, i.pred, i.data]]
+    end
+
+    # Add all tuples at once.
+    dont_care <~ atomic_data_out.map do |a|
+      if depl_idempotent a
+        a.tuple[1].map do |d|
+          eval a.tuple[0].to_s + " <+ [" + d.inspect + "]"
+        end
+        [ip_port]
+      end
+    end
+
+  end
+end

data/lib/bud/deploy/ec2deploy.rb

@@ -0,0 +1,200 @@
+require 'rubygems'
+require 'AWS'
+require 'pp'
+require 'net/ssh'
+require 'net/scp'
+require 'bud'
+require 'bud/deploy/deployer'
+
+# logic to deploy Bloom programs on EC2
+module EC2Deploy
+  include Deployer
+
+  state do
+    table :access_key_id, [] => [:key]
+    table :secret_access_key, [] => [:key]
+    table :image_id, [] => [:img]
+    table :key_name, [] => [:name]
+    table :ec2_key_location, [] => [:loc]
+    table :ec2_conn, [] => [:conn]
+    table :ec2_insts, [] => [:insts]
+    table :reservation_id, [] => [:rid]
+    periodic :spinup_timer, 6
+    scratch :the_reservation, [] => [:reservation]
+    scratch :the_reservation_next, [] => [:reservation]
+    scratch :node_up, [:node] => [:bool]
+    table :init_dir, [] => [:dir]
+    table :temp_node, [:uid, :node, :localip]
+    table :all_up, [:bool]
+    table :ruby_command, [] => [:cmd]
+    table :deploy_node, [:uid] => [:node]
+    channel :ready, [:@loc, :sender]
+    table :ready_tab, [:sender]
+    scratch :ready_count, [:num]
+  end
+
+  deploystrap do
+    # Write the IP & port to a file; we'll send this to each EC2 node.
+    File.open("deploy_ip_port", "w") do |f|
+      f.puts ip_port
+    end
+  end
+
+  bootstrap do
+    # The official BUD AMI.
+    image_id <= [["ami-f434c99d"]]
+    unless @options[:deploy]
+      # Send message to the deployer telling 'em we's up.
+      File.open("deploy_ip_port", "r") do |f|
+        ready <~ [[f.readline.chop, ip_port]]
+      end
+    end
+  end
+
+  bloom :spinup do
+    ec2_conn <= join([access_key_id, secret_access_key]).map do
+      if depl_idempotent [:ec2_comm]
+        [AWS::EC2::Base.new(:access_key_id => access_key_id[[]].key,
+                            :secret_access_key => secret_access_key[[]].key)]
+      end
+    end
+
+    ec2_insts <= join([image_id, node_count, key_name, ec2_conn]).map do
+      if depl_idempotent [:ec2_insts]
+        print "Starting up EC2 instances"
+        STDOUT.flush
+        # First, we create the security group.
+        begin
+          ec2_conn[[]].conn.create_security_group(:group_name => "bud", :group_description => "bud")
+        rescue AWS::InvalidGroupDuplicate
+          # Group already exists; ok, maybe we created it previously.
+        else
+          # Add SSH permission.
+          ec2_conn[[]].conn.authorize_security_group_ingress( :group_name => "bud",
+                                                              :ip_protocol => "tcp",
+                                                              :from_port => 22,
+                                                              :to_port => 22,
+                                                              :cidr_ip => "0.0.0.0/0" )
+          # Add unlimited UDP permission from any node not in the security group.
+          # XXX: make this more restrictive?
+          ec2_conn[[]].conn.authorize_security_group_ingress( :group_name => "bud",
+                                                              :ip_protocol => "udp",
+                                                              :from_port => 0,
+                                                              :to_port => 65535,
+                                                              :cidr_ip => "0.0.0.0/0" )
+        end
+
+        # Finally, start up the instances.
+        [ec2_conn[[]].conn.run_instances(:image_id => image_id[[]].img,
+                                         :min_count => node_count[[]].num,
+                                         :max_count => node_count[[]].num,
+                                         :key_name => key_name[[]].name,
+                                         :security_group => "bud")]
+      end
+    end
+
+    the_reservation <= join([spinup_timer, ec2_conn, ec2_insts]).map do |t,c,i|
+      if depl_idempotent [[:the_reservation, t.val]] and not all_up.include? [true]
+        to_ret = nil
+        begin
+          to_ret = [ec2_conn[[]].conn.describe_instances()["reservationSet"]["item"].find do |i|
+                      i["reservationId"] == ec2_insts[[]].insts["reservationId"]
+                    end]
+        rescue SocketError
+          print "E"
+        else
+          print "."
+        end
+        STDOUT.flush
+        to_ret
+      end
+    end
+
+    # XXX: No upsert operator, so we have to do this.
+    the_reservation_next <+ the_reservation
+
+    node_up <= ((join([ec2_insts, the_reservation]).map do
+                   if not all_up.include? [true]
+                     the_reservation[[]].reservation["instancesSet"]["item"].map do |i|
+                       [i, i["instanceState"]["code"] == "16"]
+                     end
+                   end
+                 end)[0] or [])
+
+
+    all_up <+ node_up.map do
+      if node_up.find {|n| n.bool == false} == nil and node_up.find {|n| n.bool == true} != nil
+        if depl_idempotent [:nodes_all_up]
+          puts "done"
+          STDOUT.flush
+          [true]
+        end
+      end
+    end
+
+    # XXX: Fixed port 54321
+    temp_node <= join([all_up, the_reservation_next]).map do
+      break(((0..(the_reservation_next[[]].reservation["instancesSet"]["item"].size-1)).to_a.zip(the_reservation_next[[]].reservation["instancesSet"]["item"].map {|i| [i["ipAddress"], i["privateIpAddress"]]})).map {|n,ips| [n, ips[0] + ":54321", ips[1] + ":54321"]})
+    end
+
+    deploy_node <= join([temp_node, init_dir, ruby_command]).map do |t, i, r|
+      if depl_idempotent [[:node_startup, t.node]]
+        ip = t.node.split(":")[0]
+        port = t.node.split(":")[1]
+        print "Deploying to #{ip} (#{t.uid}/#{node_count[[]].num-1})."
+        STDOUT.flush
+
+        # Upload files and run commands.
+        ctr = 0
+        while ctr < 10
+          begin
+            Net::SSH.start(ip, 'ec2-user', :keys => [ec2_key_location[[]].loc],
+                           :timeout => 5, :paranoid => false) do |session|
+              # Upload init_dir, and the IP and port of the deployer
+              session.scp.upload!("deploy_ip_port", "/home/ec2-user")
+              session.scp.upload!(init_dir[[]].dir, "/home/ec2-user",
+                                  :recursive  => true)
+              # Update the Bud gem
+              channel = session.open_channel do |ch|
+                channel.request_pty do |_, success|
+                  raise "Couldn't open a PTY on #{t.node}" if !success
+                end
+                channel.exec("sudo gem update --no-ri --no-rdoc bud")
+                channel.wait
+              end
+              # Run the ruby_command
+              session.exec!('nohup ' + ruby_command[[]].cmd + ' ' + t.localip +
+                            ' ' + t.node + ' >metarecv.out 2>metarecv.err </dev/null &')
+            end
+            break true
+          rescue Exception
+            ctr += 1
+            print "."
+            STDOUT.flush
+            sleep 10
+            next
+          end
+        end or raise "EC2 SSH failed after 10 retries"
+
+        puts "done"
+        [t.uid, t.node]
+      end
+    end
+
+  end
+
+  bloom :all_nodes do
+    stdio <~ ready.map {|_,s| ["Ready: #{s}"]}
+    # Persist ready messages
+    ready_tab <= ready.map {|_, s| [s]}
+    # Compute a count of ready messages
+    ready_count <= ready_tab.group(nil, count)
+    # Copy deploy_node into node when all nodes are up
+    node <= join([ready_count, node_count],
+                 [ready_count.num, node_count.num]).map do
+      break deploy_node
+    end
+
+  end
+
+end

data/lib/bud/deploy/localdeploy.rb

@@ -0,0 +1,41 @@
+require 'rubygems'
+require 'bud'
+require 'thread'
+require 'bud/deploy/deployer'
+
+# Starts up a bunch of Bud instances locally on 127.0.0.1, with ephemeral ports.
+module LocalDeploy
+  include Deployer
+
+  trap("CLD") {
+    pid = Process.wait
+    puts "Child pid #{pid}: terminated"
+  }
+
+  deploystrap do
+    read, write = IO.pipe
+    if node_count[[]]
+      print "Forking local processes"
+      (0..node_count[[]].num-1).map do |i|
+        Process.fork do
+          # Don't want to inherit our parent's random stuff.
+          srand
+          foo = self.class.new
+          foo.run_bg
+          print "."
+          $stdout.flush
+          # Processes write their port to a pipe.
+          write.print foo.port.to_s + "\n"
+          EventMachine.reactor_thread.join
+        end
+      end
+
+      # Read ports from pipe.
+      (0..node_count[[]].num-1).map do |i|
+        node << [i, "localhost:" + read.readline.rstrip]
+      end
+      puts "done"
+    end
+  end
+
+end

data/lib/bud/errors.rb

@@ -0,0 +1,15 @@
+module Bud
+  # Root Bud exception type.
+  class BudError < StandardError; end
+
+  # Raised (at runtime) when a type mismatch occurs (e.g., supplying a
+  # non-Enumerable object to the RHS of a Bud statement).
+  class BudTypeError < BudError; end
+
+  # Raised when a primary key constraint is violated.
+  class KeyConstraintError < BudError; end
+
+  # Raised when the input program fails to compile (e.g., due to illegal
+  # syntax).
+  class CompileError < BudError; end
+end