bud 0.0.3 → 0.0.4

data/lib/bud.rb

@@ -5,105 +5,33 @@ require 'socket'
 require 'superators'
 require 'thread'
 
+require 'bud/monkeypatch'
+
 require 'bud/aggs'
 require 'bud/bud_meta'
 require 'bud/collections'
+require 'bud/depanalysis'
+require 'bud/deploy/forkdeploy'
+require 'bud/deploy/threaddeploy'
 require 'bud/errors'
 require 'bud/joins'
 require 'bud/rtrace'
 require 'bud/server'
 require 'bud/state'
+require 'bud/storage/dbm'
 require 'bud/storage/tokyocabinet'
 require 'bud/storage/zookeeper'
+require 'bud/stratify'
 require 'bud/viz'
 
-$em_stopped = Queue.new
-
-# We monkeypatch Module to add support for Bloom state and code declarations.
-class Module
-  
-  # import another module and assign to a qualifier symbol: <tt>import MyModule => :m</tt>
-  def import(spec)
-    raise Bud::CompileError unless (spec.class <= Hash and spec.length == 1)
-    mod, local_name = spec.first
-    raise Bud::CompileError unless (mod.class <= Module and local_name.class <= Symbol)
-
-    # To correctly expand qualified references to an imported module, we keep a
-    # table with the local bind names of all the modules imported by this
-    # module. To handle nested references (a.b.c.d etc.), the import table for
-    # module X points to X's own nested import table.
-    @bud_import_tbl ||= {}
-    child_tbl = mod.bud_import_table
-    raise Bud::CompileError if @bud_import_tbl.has_key? local_name
-    @bud_import_tbl[local_name] = child_tbl.clone # XXX: clone needed?
-
-    rewritten_mod_name = ModuleRewriter.do_import(self, mod, local_name)
-    self.module_eval "include #{rewritten_mod_name}"
-  end
-
-  # the block of Bloom collection declarations.  one per module.
-  def state(&block)
-    meth_name = Module.make_state_meth_name(self)
-    define_method(meth_name, &block)
-  end
-
-  # a ruby block to be run before timestep 1.  one per module.
-  def bootstrap(&block)
-    meth_name = "__bootstrap__#{Module.get_class_name(self)}".to_sym
-    define_method(meth_name, &block)
-  end
-
-  # bloom statements to be registered with Bud runtime.  optional +block_name+ 
-  # allows for multiple bloom blocks per module, and overriding
-  def bloom(block_name=nil, &block)
-    # If no block name was specified, generate a unique name
-    if block_name.nil?
-      @block_id ||= 0
-      block_name = "#{Module.get_class_name(self)}__#{@block_id.to_s}"
-      @block_id += 1
-    else
-      unless block_name.class <= Symbol
-        raise Bud::CompileError, "Bloom block names must be a symbol: #{block_name}"
-      end
-    end
-
-    # Note that we don't encode the module name ("self") into the name of the
-    # method. This allows named blocks to be overridden (via inheritance or
-    # mixin) in the same way as normal Ruby methods.
-    meth_name = "__bloom__#{block_name}"
-
-    # Don't allow duplicate named bloom blocks to be defined within a single
-    # module; this indicates a likely programmer error.
-    if instance_methods(false).include? meth_name
-      raise Bud::CompileError, "Duplicate named bloom block: '#{block_name}' in #{self}"
-    end
-    define_method(meth_name.to_sym, &block)
-  end
-
-  def bud_import_table() #:nodoc: all
-    @bud_import_tbl ||= {}
-    @bud_import_tbl
-  end
+ILLEGAL_INSTANCE_ID = -1
+SIGNAL_CHECK_PERIOD = 0.2
 
-  private
-  # Return a string with a version of the class name appropriate for embedding
-  # into a method name. Annoyingly, if you define class X nested inside
-  # class/module Y, X's class name is the string "Y::X". We don't want to define
-  # method names with semicolons in them, so just return "X" instead.
-  def self.get_class_name(klass)
-    klass.name.split("::").last
-  end
-
-  # State method blocks are named using an auto-incrementing counter. This is to
-  # ensure that we can rediscover the possible dependencies between these blocks
-  # after module import (see Bud#call_state_methods).
-  def self.make_state_meth_name(klass)
-    @state_meth_id ||= 0
-    r = "__state#{@state_meth_id}__#{Module.get_class_name(klass)}".to_sym
-    @state_meth_id += 1
-    return r
-  end
-end
+$signal_lock = Mutex.new
+$got_shutdown_signal = false
+$signal_handler_setup = false
+$instance_id = 0
+$bud_instances = {}        # Map from instance id => Bud instance
 
 # The root Bud module. To cause an instance of Bud to begin executing, there are
 # three main options:
@@ -128,51 +56,64 @@ end
 # :main: Bud
 module Bud
   attr_reader :strata, :budtime, :inbound, :options, :meta_parser, :viz, :rtracer
-  attr_reader :dsock
-  attr_reader :tables, :ip, :port
-  attr_reader :stratum_first_iter
+  attr_reader :dsock, :ip, :port
+  attr_reader :tables, :channels, :tc_tables, :zk_tables, :dbm_tables
+  attr_reader :stratum_first_iter, :joinstate
   attr_accessor :lazy # This can be changed on-the-fly by REBL
+  attr_accessor :stratum_collection_map
 
-  # options to the bud runtime are passed in a hash, with the following keys
+  # options to the Bud runtime are passed in a hash, with the following keys
   # * network configuration
   #   * <tt>:ip</tt>   IP address string for this instance
   #   * <tt>:port</tt>   port number for this instance
   #   * <tt>:ext_ip</tt>  IP address at which external nodes can contact this instance
-  #   * <tt>:ext_port</tt>   port number to go with :ext_ip
-  #   * <tt>:bust_port</tt>  port number for the restful http messages
+  #   * <tt>:ext_port</tt>   port number to go with <tt>:ext_ip</tt>
+  #   * <tt>:bust_port</tt>  port number for the restful HTTP messages
   # * operating system interaction
-  #   * <tt>:read_stdin</tt>  if true, captures stdin via the stdio collection
-  #   * <tt>:no_signal_handlers</tt> if true, runtime ignores SIGINT and SIGTERM
+  #   * <tt>:stdin</tt>  if non-nil, reading from the +stdio+ collection results in reading from this +IO+ handle
+  #   * <tt>:stdout</tt> writing to the +stdio+ collection results in writing to this +IO+ handle; defaults to <tt>$stdout</tt>
+  #   * <tt>:no_signal_handlers</tt> if true, runtime ignores +SIGINT+ and +SIGTERM+
   # * tracing and output
   #   * <tt>:quiet</tt> if true, suppress certain messages
-  #   * <tt>:trace</tt> if true, generate budvis outputs
-  #   * <tt>:rtrace</tt>  if true, generate budplot outputs
+  #   * <tt>:trace</tt> if true, generate +budvis+ outputs
+  #   * <tt>:rtrace</tt>  if true, generate +budplot+ outputs
   #   * <tt>:dump_rewrite</tt> if true, dump results of internal rewriting of Bloom code to a file
-  # * controlling execution 
+  # * controlling execution
   #   * <tt>:lazy</tt>  if true, prevents runtime from ticking except on external calls to +tick+
   #   * <tt>:tag</tt>  a name for this instance, suitable for display during tracing and visualization
   # * storage configuration
-  #   * <tt>:tc_dir</tt>  filesystem directory to hold TokyoCabinet data stores
-  #   * <tt>:tc_truncate</tt> if true, TokyoCabinet collections are opened with OTRUNC
+  #   * <tt>:dbm_dir</tt> filesystem directory to hold DBM-backed collections
+  #   * <tt>:dbm_truncate</tt> if true, DBM-backed collections are opened with +OTRUNC+
+  #   * <tt>:tc_dir</tt>  filesystem directory to hold TokyoCabinet-backed collections
+  #   * <tt>:tc_truncate</tt> if true, TokyoCabinet-backed collections are opened with +OTRUNC+
+  # * deployment
+  #   * <tt>:deploy</tt>  enable deployment
+  #   * <tt>:deploy_child_opts</tt> option hash to pass to deployed instances
   def initialize(options={})
     @tables = {}
     @table_meta = []
     @rewritten_strata = []
     @channels = {}
     @tc_tables = {}
+    @dbm_tables = {}
     @zk_tables = {}
     @callbacks = {}
     @callback_id = 0
+    @shutdown_callbacks = []
+    @post_shutdown_callbacks = []
     @timers = []
+    @inside_tick = false
+    @tick_clock_time = nil
     @budtime = 0
     @inbound = []
     @done_bootstrap = false
     @joinstate = {}  # joins are stateful, their state needs to be kept inside the Bud instance
+    @instance_id = ILLEGAL_INSTANCE_ID # Assigned when we start running
 
     # Setup options (named arguments), along with default values
-    @options = options
+    @options = options.clone
     @lazy = @options[:lazy] ||= false
-    @options[:ip] ||= "localhost"
+    @options[:ip] ||= "127.0.0.1"
     @ip = @options[:ip]
     @options[:port] ||= 0
     @options[:port] = @options[:port].to_i
@@ -188,12 +129,6 @@ module Bud
 
     init_state
 
-    # NB: Somewhat hacky. Dependency analysis and stratification are implemented
-    # by Bud programs, so in order for those programs to parse, we need the
-    # "Bud" class to have been defined first.
-    require 'bud/depanalysis'
-    require 'bud/stratify'
-
     @viz = VizOnline.new(self) if @options[:trace]
     @rtracer = RTrace.new(self) if @options[:rtrace]
 
@@ -208,6 +143,7 @@ module Bud
     # array, so we need to convert it before loading the rewritten strata.
     @strata = []
     @rule_src = []
+    @rule_orig_src = []
     declaration
     @strata.each_with_index do |s,i|
       raise BudError if s.class <= Array
@@ -219,9 +155,11 @@ module Bud
     @rewritten_strata.each_with_index do |src_ary,i|
       @strata[i] ||= []
       @rule_src[i] ||= []
-      src_ary.each do |src|
+      @rule_orig_src[i] ||= []
+      src_ary.each_with_index do |src, j|
         @strata[i] << eval("lambda { #{src} }")
         @rule_src[i] << src
+        @rule_orig_src[i] << @no_attr_rewrite_strata[i][j]
       end
     end
   end
@@ -313,7 +251,7 @@ module Bud
 
   def do_rewrite
     @meta_parser = BudMeta.new(self, @declarations)
-    @rewritten_strata = @meta_parser.meta_rewrite
+    @rewritten_strata, @no_attr_rewrite_strata = @meta_parser.meta_rewrite
   end
 
   public
@@ -349,19 +287,25 @@ module Bud
   # Run Bud in the "foreground" -- the caller's thread will be used to run the
   # Bud interpreter. This means this method won't return unless an error
   # occurs. It is often more useful to run Bud asynchronously -- see run_bg.
-  #
-  # Note that run_fg cannot be invoked if run_bg has already been called in the
-  # same Ruby process.
-  #
-  # Execution proceeds in time ticks, a la Dedalus.
-  # * Within each tick there may be multiple strata.
-  # * Within each stratum we do multiple semi-naive iterations.
   def run_fg
-    raise BudError if EventMachine::reactor_running?
+    # If we're called from the EventMachine thread (and EM is running), blocking
+    # the current thread would imply deadlocking ourselves.
+    if Thread.current == EventMachine::reactor_thread and EventMachine::reactor_running?
+      raise BudError, "Cannot invoke run_fg from inside EventMachine"
+    end
 
-    EventMachine::run {
-      start_bud
-    }
+    q = Queue.new
+    # Note that this must be a post-shutdown callback: if this is the only
+    # thread, then the program might exit after run_fg() returns. If run_fg()
+    # blocked on a normal shutdown callback, the program might exit before the
+    # other shutdown callbacks have a chance to run.
+    post_shutdown do
+      q.push(true)
+    end
+
+    run_bg
+    # Block caller's thread until Bud has shutdown
+    q.pop
   end
 
   # Shutdown a Bud instance that is running asynchronously. This method blocks
@@ -369,15 +313,34 @@ module Bud
   # loop is also shutdown; this will interfere with the execution of any other
   # Bud instances in the same process (as well as anything else that happens to
   # use EventMachine).
-  def stop_bg(stop_em=false)
+  def stop_bg(stop_em=false, do_shutdown_cb=true)
+    schedule_and_wait do
+      do_shutdown(do_shutdown_cb)
+    end
+
     if stop_em
-      schedule_shutdown(true)
-      # Wait until EM has completely shutdown before we return.
-      $em_stopped.pop
-    else
-      schedule_and_wait do
-        do_shutdown(false)
-      end
+      Bud.stop_em_loop
+      EventMachine::reactor_thread.join
+    end
+  end
+
+  # Register a callback that will be invoked when this instance of Bud is
+  # shutting down.
+  def on_shutdown(&blk)
+    # Start EM if not yet started
+    start_reactor
+    schedule_and_wait do
+      @shutdown_callbacks << blk
+    end
+  end
+
+  # Register a callback that will be invoked when *after* this instance of Bud
+  # has been shutdown.
+  def post_shutdown(&blk)
+    # Start EM if not yet started
+    start_reactor
+    schedule_and_wait do
+      @post_shutdown_callbacks << blk
     end
   end
 
@@ -414,7 +377,7 @@ module Bud
   end
 
   # Shutdown any persistent tables used by the current Bud instance. If you are
-  # running Bud via tick() and using `tctable` collections, you should call this
+  # running Bud via tick() and using +tctable+ collections, you should call this
   # after you're finished using Bud. Programs that use Bud via run_fg() or
   # run_bg() don't need to call this manually.
   def close_tables
@@ -485,7 +448,7 @@ module Bud
     return result
   end
 
-  # a common special case for sync_callback: block on a delta to a table.
+  # A common special case for sync_callback: block on a delta to a table.
   def delta(out_tbl)
     sync_callback(nil, nil, out_tbl)
   end
@@ -508,6 +471,7 @@ module Bud
     EventMachine::error_handler do |e|
       puts "Unexpected Bud error: #{e.inspect}"
       puts e.backtrace.join("\n")
+      Bud.shutdown_all_instances
       raise e
     end
 
@@ -517,10 +481,8 @@ module Bud
     # EventMachine's event loop.
     Thread.new do
       EventMachine.run do
-        q << true
+        q.push(true)
       end
-      # Executed only after EventMachine::stop_event_loop is done
-      $em_stopped << true
     end
     # Block waiting for EM's event loop to start up.
     q.pop
@@ -529,9 +491,12 @@ module Bud
   # Schedule a block to be evaluated by EventMachine in the future, and
   # block until this has happened.
   def schedule_and_wait
-    # Try to defend against error situations in which EM has stopped, but we've
-    # been called nonetheless. This is racy, but better than nothing.
-    raise BudError, "EM not running" unless EventMachine::reactor_running?
+    # If EM isn't running, just run the user's block immediately
+    # XXX: not clear that this is the right behavior
+    unless EventMachine::reactor_running?
+      yield
+      return
+    end
 
     q = Queue.new
     EventMachine::schedule do
@@ -548,43 +513,33 @@ module Bud
     raise resp if resp
   end
 
-  def do_shutdown(stop_em=false)
-    @timers.each do |t|
-      t.cancel
+  def do_shutdown(do_shutdown_cb=true)
+    # Silently ignore duplicate shutdown requests or attempts to shutdown an
+    # instance that hasn't been started yet.
+    return if @instance_id == ILLEGAL_INSTANCE_ID
+
+    $signal_lock.synchronize {
+      raise unless $bud_instances.has_key? @instance_id
+      $bud_instances.delete @instance_id
+      @instance_id = ILLEGAL_INSTANCE_ID
+    }
+
+    if do_shutdown_cb
+      @shutdown_callbacks.each {|cb| cb.call}
     end
+    @timers.each {|t| t.cancel}
     close_tables
-    @dsock.close_connection
-    # Note that this affects anyone else in the same process who happens to be
-    # using EventMachine! This is also a non-blocking call; to block until EM
-    # has completely shutdown, we use the @em_stopped queue.
-    EventMachine::stop_event_loop if stop_em
-  end
-
-  # Schedule a "graceful" shutdown for a future EM tick. If EM is not currently
-  # running, shutdown immediately.
-  def schedule_shutdown(stop_em=false)
-    if EventMachine::reactor_running?
-      EventMachine::schedule do
-        do_shutdown(stop_em)
-      end
-    else
-      do_shutdown(stop_em)
+    @dsock.close_connection if EventMachine::reactor_running?
+    if do_shutdown_cb
+      @post_shutdown_callbacks.each {|cb| cb.call}
     end
   end
 
+  private
   def start_bud
     raise BudError unless EventMachine::reactor_thread?
 
-    # If we get SIGINT or SIGTERM, shutdown gracefully
-    unless @options[:no_signal_handlers]
-      Signal.trap("INT") do
-        schedule_shutdown(true)
-      end
-      Signal.trap("TRAP") do
-        schedule_shutdown(true)
-      end
-    end
-
+    @instance_id = Bud.init_signal_handlers(self)
     do_start_server
 
     # Initialize periodics
@@ -594,7 +549,7 @@ module Bud
 
     # Arrange for Bud to read from stdin if enabled. Note that we can't do this
     # earlier because we need to wait for EventMachine startup.
-    @stdio.start_stdin_reader if @options[:read_stdin]
+    @stdio.start_stdin_reader if @options[:stdin]
     @zk_tables.each_value {|t| t.start_watchers}
 
     # Compute a fixpoint; this will also invoke any bootstrap blocks.
@@ -611,12 +566,13 @@ module Bud
 
   public
 
-  # Returns the ip and port of the Bud instance.  In addition to the local IP
-  # and port, the user may define an external IP and/or port. the external
-  # version of each is returned if available.  If not, the local version is
-  # returned.  There are use cases for mixing and matching local and external.
-  # local_ip:external_port would be if you have local port forwarding, and
-  # external_ip:local_port would be if you're in a DMZ, for example
+  # Returns the IP and port of the Bud instance as a string.  In addition to the
+  # local IP and port, the user may define an external IP and/or port. The
+  # external version of each is returned if available.  If not, the local
+  # version is returned.  There are use cases for mixing and matching local and
+  # external.  local_ip:external_port would be if you have local port
+  # forwarding, and external_ip:local_port would be if you're in a DMZ, for
+  # example.
   def ip_port
     raise BudError, "ip_port called before port defined" if @port.nil? and @options[:port] == 0 and not @options[:ext_port]
 
@@ -626,28 +582,43 @@ module Bud
     ip + ":" + port
   end
 
-  # Returns the internal IP and port.  See ip_port
+  # Returns the internal IP and port.  See ip_port.
   def int_ip_port
     raise BudError, "ip_port called before port defined" if @port.nil? and @options[:port] == 0
     @port.nil? ? "#{@ip}:#{@options[:port]}" : "#{@ip}:#{@port}"
   end
 
-  # manually trigger one timestep of Bloom execution.
+  # Manually trigger one timestep of Bloom execution.
   def tick
-    @tables.each_value do |t|
-      t.tick
-    end
-    
-    @joinstate = {}
+    begin
+      @inside_tick = true
+      @tables.each_value do |t|
+        t.tick
+      end
+
+      @joinstate = {}
+
+      do_bootstrap unless @done_bootstrap
+      receive_inbound
 
-    do_bootstrap unless @done_bootstrap
-    receive_inbound
+      @strata.each_with_index { |s,i| stratum_fixpoint(s, i) }
+      @viz.do_cards if @options[:trace]
+      do_flush
+      invoke_callbacks
+      @budtime += 1
+    ensure
+      @inside_tick = false
+      @tick_clock_time = nil
+    end
+  end
 
-    @strata.each_with_index { |s,i| stratum_fixpoint(s, i) }
-    @viz.do_cards if @options[:trace]
-    do_flush
-    invoke_callbacks
-    @budtime += 1
+  # Returns the wallclock time associated with the current Bud tick. That is,
+  # this value is guaranteed to remain the same for the duration of a single
+  # tick, but will likely change between ticks.
+  def bud_clock
+    raise BudError, "bud_clock undefined outside tick" unless @inside_tick
+    @tick_clock_time ||= Time.now
+    @tick_clock_time
   end
 
   private
@@ -656,18 +627,20 @@ module Bud
   # standard "state" syntax, but we want to ensure that builtin state is
   # initialized before user-defined state.
   def builtin_state
-    channel  :localtick, [:col1]
+    loopback  :localtick, [:col1]
     @stdio = terminal :stdio
     @periodics = table :periodics_tbl, [:pername] => [:ident, :period]
 
     # for BUD reflection
-    table :t_rules, [:rule_id] => [:lhs, :op, :src]
+    table :t_rules, [:rule_id] => [:lhs, :op, :src, :orig_src]
     table :t_depends, [:rule_id, :lhs, :op, :body] => [:nm]
     table :t_depends_tc, [:head, :body, :via, :neg, :temporal]
     table :t_provides, [:interface] => [:input]
     table :t_underspecified, t_provides.schema
     table :t_stratum, [:predicate] => [:stratum]
     table :t_cycle, [:predicate, :via, :neg, :temporal]
+    table :t_table_info, [:tab_name, :tab_type]
+    table :t_table_schema, [:tab_name, :col_name, :ord, :loc]
   end
 
   # Handle any inbound tuples off the wire and then clear. Received messages are
@@ -684,14 +657,15 @@ module Bud
   # 1. Emit outgoing tuples in channels and ZK tables.
   # 2. Commit to disk any changes made to on-disk tables.
   def do_flush
-    @channels.each { |c| @tables[c[0]].flush }
+    @channels.each_value { |c| c.flush }
     @zk_tables.each_value { |t| t.flush }
     @tc_tables.each_value { |t| t.flush }
+    @dbm_tables.each_value { |t| t.flush }
   end
 
   def stratum_fixpoint(strat, strat_num)
-    # This routine uses semi-naive evaluation to compute
-    # a fixpoint of the rules in strat.
+    # This routine uses semi-naive evaluation to compute a fixpoint of the rules
+    # in strat.
     #
     # As described in lib/collections.rb, each collection has three
     # sub-collections of note here:
@@ -699,13 +673,13 @@ module Bud
     #   @delta: tuples that should be used to drive derivation of new facts
     #   @new_delta: a place to store newly-derived facts
     #
-    # The first time through this loop we mark @stratum_first_iter=true,
-    # while tells the Join::each code to join up all its @storage subcollections
-    # to start. In subsequent iterations the join code uses some table's @delta
-    # to ensure that only new tuples are derived.
+    # The first time through this loop we mark @stratum_first_iter=true, which
+    # tells the Join::each code to join up all its @storage subcollections to
+    # start. In subsequent iterations the join code uses some table's @delta to
+    # ensure that only new tuples are derived.
     #
-    # Note that calling "each" on a non-Join collection will iterate through both
-    # storage and delta.
+    # Note that calling "each" on a non-Join collection will iterate through
+    # both storage and delta.
     #
     # At the end of each iteration of this loop we transition:
     # - @delta tuples are merged into @storage
@@ -713,22 +687,22 @@ module Bud
     # - @new_delta is set to empty
     #
     # XXX as a performance optimization, it would be nice to bypass the delta
-    # tables for any preds that don't participate in a rhs Join -- in that
-    # case there's pointless extra tuple movement letting tuples "graduate"
-    # through @new_delta and @delta.
-
-    # In semi-naive, the first iteration should join up tables
-    # on their storage fields; subsequent iterations do the
-    # delta-joins only.  The stratum_first_iter field here distinguishes
-    # these cases.
+    # tables for any preds that don't participate in a rhs Join -- in that case
+    # there's pointless extra tuple movement letting tuples "graduate" through
+    # @new_delta and @delta.
+
+    # In semi-naive, the first iteration should join up tables on their storage
+    # fields; subsequent iterations do the delta-joins only.  The
+    # stratum_first_iter field here distinguishes these cases.
     @stratum_first_iter = true
     begin
       strat.each_with_index do |r,i|
+        fixpoint = false
         begin
           r.call
         rescue Exception => e
           # Don't report source text for certain rules (old-style rule blocks)
-          rule_src = @rule_src[strat_num][i]
+          rule_src = @rule_orig_src[strat_num][i] unless @rule_orig_src[strat_num].nil?
           src_msg = ""
           unless rule_src == ""
             src_msg = "\nRule: #{rule_src}"
@@ -742,57 +716,110 @@ module Bud
         end
       end
       @stratum_first_iter = false
-      # XXX this next line is inefficient.
-      # we could call tick_deltas only on predicates in this stratum.
-      # but it's not easy right now (??) to pull out tables in a given stratum
-      @tables.each{|name,coll| coll.tick_deltas}
-    end while not @tables.all?{|name,coll| coll.new_delta.empty? and coll.delta.empty?}
+      fixpoint = true
+      # tick collections in this stratum; if we don't have info on that, tick all collections
+      colls = @stratum_collection_map[strat_num] if @stratum_collection_map
+      colls ||= @tables.keys
+      colls.each do |name|
+        begin
+          coll = self.send(name)
+          unless coll.delta.empty? and coll.new_delta.empty?
+            coll.tick_deltas
+            fixpoint = false
+          end
+        rescue
+          # ignore missing tables; rebl for example deletes them mid-stream
+        end
+      end
+    end while not fixpoint
   end
 
-  ####### Joins
-  def wrap_map(j, &blk)
-    if blk.nil?
-      return j
-    else
-      return j.map(&blk)
-    end
-  end
+  private
 
-  public
-  def joinstate # :nodoc: all
-    @joinstate
+  ######## ids and timers
+  def gen_id
+    Time.new.to_i.to_s << rand.to_s
   end
 
-  public
-  def join(collections, *preds, &blk) # :nodoc: all
-    # since joins are stateful, we want to allocate them once and store in this Bud instance
-    # we ID them on their tablenames, preds, and block
-    return wrap_map(BudJoin.new(collections, self, preds), &blk)
+  def set_periodic_timer(name, id, period)
+    EventMachine::PeriodicTimer.new(period) do
+      @tables[name].add_periodic_tuple(id)
+      tick
+    end
   end
 
-  def natjoin(collections, &blk) # :nodoc: all
-    # for all pairs of relations, add predicates on matching column names
-    preds = BudJoin::natural_preds(self, collections)
-    join(collections, *preds, &blk)
+  # Fork a new process. This is identical to Kernel#fork, except that it also
+  # cleans up Bud and EventMachine-related state. As with Kernel#fork, the
+  # caller supplies a code block that is run in the child process; the PID of
+  # the child is returned by this method.
+  def self.do_fork
+    Kernel.fork do
+      srand
+      # This is somewhat grotty: we basically clone what EM::fork_reactor does,
+      # except that we don't want the user-supplied block to be invoked by the
+      # reactor thread.
+      if EventMachine::reactor_running?
+        EventMachine::stop_event_loop
+        EventMachine::release_machine
+        EventMachine::instance_variable_set('@reactor_running', false)
+      end
+      # Shutdown all the Bud instances inherited from the parent process, but
+      # don't invoke their shutdown callbacks
+      Bud.shutdown_all_instances(false)
+
+      $got_shutdown_signal = false
+      $setup_signal_handler = false
+
+      yield
+    end
   end
 
-  # left-outer-join syntax to be used in rhs of Bloom statements.  
-  # first argument an array of 2 collections, second argument an array of predicates (as in Bud::BudCollection.pairs)
-  def leftjoin(collections, *preds, &blk)
-    return wrap_map(BudLeftJoin.new(collections, self, preds), &blk)
+  # Note that this affects anyone else in the same process who happens to be
+  # using EventMachine! This is also a non-blocking call; to block until EM
+  # has completely shutdown, join on EM::reactor_thread.
+  def self.stop_em_loop
+    EventMachine::stop_event_loop
+
+    # If another instance of Bud is started later, we'll need to reinitialize
+    # the signal handlers (since they depend on EM).
+    $signal_handler_setup = false
   end
 
-  private
+  # Signal handling. If multiple Bud instances are running inside a single
+  # process, we want a SIGINT or SIGTERM signal to cleanly shutdown all of them.
+  def self.init_signal_handlers(b)
+    $signal_lock.synchronize {
+      # If we setup signal handlers and then fork a new process, we want to
+      # reinitialize the signal handler in the child process.
+      unless b.options[:no_signal_handlers] or $signal_handler_setup
+        EventMachine::PeriodicTimer.new(SIGNAL_CHECK_PERIOD) do
+          if $got_shutdown_signal
+            Bud.shutdown_all_instances
+            Bud.stop_em_loop
+            $got_shutdown_signal = false
+          end
+        end
 
-  ######## ids and timers
-  def gen_id
-    Time.new.to_i.to_s << rand.to_s
+        ["INT", "TERM"].each do |signal|
+          Signal.trap(signal) {
+            $got_shutdown_signal = true
+          }
+        end
+        $setup_signal_handler_pid = true
+      end
+
+      $instance_id += 1
+      $bud_instances[$instance_id] = b
+      return $instance_id
+    }
   end
 
-  def set_periodic_timer(name, id, period)
-    EventMachine::PeriodicTimer.new(period) do
-      @tables[name] <+ [[id, Time.new.to_s]]
-      tick
-    end
+  def self.shutdown_all_instances(do_shutdown_cb=true)
+    instances = nil
+    $signal_lock.synchronize {
+      instances = $bud_instances.clone
+    }
+
+    instances.each_value {|b| b.stop_bg(false, do_shutdown_cb) }
   end
 end