bud 0.0.4 → 0.0.5

data/lib/bud/deploy/threaddeploy.rb

@@ -16,7 +16,7 @@ module ThreadDeploy
 
     @instances = []
     on_shutdown do
-      @instances.each {|b| b.stop_bg}
+      @instances.each {|b| b.stop}
     end
 
     print "Spawning threads"

data/lib/bud/errors.rb

@@ -12,4 +12,7 @@ module Bud
   # Raised when the input program fails to compile (e.g., due to illegal
   # syntax).
   class CompileError < BudError; end
+  
+  # Raised when evaluation halts with outstanding callbacks
+  class BudShutdownWithCallbacksError < BudError; end
 end

data/lib/bud/graphs.rb

@@ -27,18 +27,16 @@ class GraphGen #:nodoc: all
     cycle.each do |c|
       # assumption: !(c[2] and !c[3]), or stratification would have bombed out
       if c[2] and c[3]
-        if !@redcycle[c[0]]
-          @redcycle[c[0]] = []
-        end
+        @redcycle[c[0]] ||= []
         @redcycle[c[0]] << c[1]
-      end      
+      end
     end
-    
+
     @nodes = {}
     @edges = {}
     @labels = {}
   end
-  
+
   def name_bag(predicate, bag)
     if bag[predicate]
       return bag
@@ -46,10 +44,10 @@ class GraphGen #:nodoc: all
       bag[predicate] = true
       res = bag
       if @redcycle[predicate].nil?
-        return res 
+        return res
       end
       @redcycle[predicate].each do |rp|
-        res = name_bag(rp, res)      
+        res = name_bag(rp, res)
       end
     end
 
@@ -65,7 +63,7 @@ class GraphGen #:nodoc: all
       return str
     else
       return predicate
-    end 
+    end
   end
 
   def process(depends)
@@ -92,11 +90,13 @@ class GraphGen #:nodoc: all
 
   def addonce(node, negcluster, inhead=false)
     if !@nodes[node]
-      @nodes[node] = @graph.add_node(node)
+      @nodes[node] = @graph.add_node("n_#{node}")
       if @cards and @cards[node]
-        @nodes[node].label = node +"\n (#{@cards[node].to_s})"
+        @nodes[node].label = "#{node}\n (#{@cards[node].to_s})"
+      else
+        @nodes[node].label = node
       end
-    end 
+    end
 
     if @budtime == -1
       @nodes[node].URL = "#{node}.html" if inhead
@@ -105,7 +105,7 @@ class GraphGen #:nodoc: all
     end
 
     if negcluster
-      # cleaning 
+      # cleaning
       res = node
       # pretty-printing issues
       node.split(", ").each_with_index do |p, i|
@@ -190,7 +190,7 @@ class GraphGen #:nodoc: all
       end
     end
 
-    unless depanalysis.nil? 
+    unless depanalysis.nil?
       depanalysis.source.each {|s| addedge("S", s.pred, false, false, false)}
       depanalysis.sink.each {|s| addedge(s.pred, "T", false, false, false)}
 
@@ -218,19 +218,19 @@ class GraphGen #:nodoc: all
   end
 end
 
-class SpaceTime    
+class SpaceTime
   def initialize(input, links = false)
-    @input = input 
+    @input = input
     @links = links
     processes = input.map {|i| i[1]}
     input.map{|i| processes << i[2]}
     processes.uniq!
 
-    @queues = {} 
-    
+    @queues = {}
+
     @g = GraphViz.new(:G, :type => :digraph, :rankdir => "LR", :outputorder => "nodesfirst", :splines => "line")#, :clusterrank => "none")
     @hdr = @g.subgraph("cluster_0")
-    
+
     @subs = {}
     @head = {}
     last = nil
@@ -238,7 +238,6 @@ class SpaceTime
       #@head[p] = @hdr.add_node("process #{p}(#{i})")#, :color => "white", :label => "")
       @subs[p] = @g.subgraph("buster_#{i+1}")
       @head[p] = @hdr.add_node("process #{p}(#{i})", :group => p)#, :color => "white", :label => "")
-      
     end
   end
 
@@ -251,13 +250,13 @@ class SpaceTime
       @edges[lbl] = [f, t, l, 1]
     end
   end
-  
+
   def process
     @edges = {}
     queues = {}
     @input.each do |i|
-      queues[i[1]] = [] unless queues[i[1]]
-      queues[i[2]] = [] unless queues[i[2]]
+      queues[i[1]] ||= []
+      queues[i[2]] ||= []
       queues[i[1]] << i[3]
       queues[i[2]] << i[4]
     end
@@ -275,7 +274,7 @@ class SpaceTime
           url = "DBM_#{k}_/tm_#{item}.svg"
           #puts "URL is #{url}"
         end
-        snd = @subs[k].add_node(label, {:label => item.to_s, :width => 0.1, :height => 0.1, :fontsize => 6, :pos => [1, i], :group => k, :URL => url})  
+        snd = @subs[k].add_node(label, {:label => item.to_s, :width => 0.1, :height => 0.1, :fontsize => 6, :pos => [1, i], :group => k, :URL => url})
 
         unless @head[k].id == snd.id
           @subs[k].add_edge(@head[k], snd, :weight => 2)
@@ -294,10 +293,10 @@ class SpaceTime
       msg_edge(snd_label, rcv_label, i[5])
     end
   end
-  
+
   def finish(file, fmt=nil)
     @edges.each_pair do |k, v|
-      lbl =  v[3] > 1 ? "#{v[2]}(#{v[3]})" : v[2]
+      lbl = v[3] > 1 ? "#{v[2]}(#{v[3]})" : v[2]
       @g.add_edge(v[0], v[1], :label => lbl, :color => "red", :weight => 1)
     end
     if fmt.nil?

data/lib/bud/joins.rb

@@ -9,6 +9,7 @@ module Bud
       @bud_instance = bud_instance
       @localpreds = nil
       @hashpreds = nil
+      @selfjoins = []
 
       # if any elements on rellist are BudJoins, suck up their contents
       tmprels = []
@@ -23,6 +24,18 @@ module Bud
       rellist = tmprels
       @origrels = rellist
 
+      # check for self-joins: we currently only handle 2 instances of the same table per rule
+      counts = @origrels.reduce({}) do |memo, r|
+        memo[r.tabname] ||= 0
+        memo[r.tabname] += 1
+        memo
+      end
+      counts.each do |name, cnt| 
+        raise Bud::CompileError, "#{cnt} instances of #{name} in rule; only one self-join currently allowed per rule" if cnt > 2
+        @selfjoins << name if cnt == 2
+      end
+      
+
       # recurse to form a tree of binary BudJoins
       @rels = [rellist[0]]
       @rels << (rellist.length == 2 ? rellist[1] : BudJoin.new(rellist[1..rellist.length-1], @bud_instance))
@@ -39,7 +52,7 @@ module Bud
         memo
       end
 
-      setup_preds(preds) unless preds.empty?
+      setup_preds(preds)
       setup_state
     end
 
@@ -76,7 +89,7 @@ module Bud
     # similar to <tt>SELECT * FROM ... WHERE...</tt> block in SQL.
     public
     def flatten(*preds)
-      setup_preds(preds) unless preds.empty?
+      setup_preds(preds)
       flat_schema = @rels.map{|r| r.schema}.flatten(1)
       dupfree_schema = []
       # while loop here (inefficiently) ensures no collisions
@@ -148,14 +161,14 @@ module Bud
     public
     def pairs(*preds, &blk)
       @origpreds = preds
-      setup_preds(preds) unless preds.empty?
+      setup_preds(preds)
       # given new preds, the state for the join will be different.  set it up again.
       setup_state if self.class <= Bud::BudJoin
       blk.nil? ? self : map(&blk)
     end
 
     alias combos pairs
-
+    
     # the natural join: given a * expression over n collections, form all
     # combinations of items that have the same values in matching fields
     public
@@ -169,7 +182,7 @@ module Bud
     # of the first collection
     public
     def lefts(*preds, &blk)
-      setup_preds(preds) unless preds.empty?
+      setup_preds(preds)
       # given new preds, the state for the join will be different.  set it up again.
       setup_state if self.class <= Bud::BudJoin
       map{ |l,r| blk.nil? ? l : blk.call(l) }
@@ -180,7 +193,7 @@ module Bud
     # of the second item
     public
     def rights(*preds, &blk)
-      setup_preds(preds) unless preds.empty?
+      setup_preds(preds)
       # given new preds, the state for the join will be different.  set it up again.
       setup_state if self.class <= Bud::BudJoin
       map{ |l,r| blk.nil? ? r : blk.call(r) }
@@ -190,16 +203,36 @@ module Bud
     # satisfy +preds+, and for any item from the 1st collection that has no
     # matches in the 2nd, nil-pad it and include it in the output.
     public
-    def outer(*preds)
+    def outer(*preds, &blk)
       @origpreds = preds
-      @localpreds = disambiguate_preds(preds)
+      setup_preds(preds)
       self.extend(Bud::BudOuterJoin)
-      map
+      blk.nil? ? self : map(&blk)
+    end
+    
+    # AntiJoin
+    # note: unlike other join methods (e.g. lefts) all we do with the return value
+    # of block is check whether it's nil.  Putting "projection" logic in the block
+    # has no effect on the output.
+    public
+    def anti(*preds, &blk)
+      @origpreds = preds
+      # no projection involved here, so we can propagate the schema
+      @schema = rels[0].schema
+      setup_preds(preds)
+      setup_state if self.class <= Bud::BudJoin
+      if @bud_instance.stratum_first_iter
+        @matches = map { |r, s| (blk.nil?) ? r : blk.call(r,s) }.compact
+        @rels[0].map {|r| (@matches.include? r) ? nil : r}.compact
+      else
+        []
+      end
     end
 
     # extract predicates on rellist[0] and recurse to right side with remainder
     protected
     def setup_preds(preds) # :nodoc: all
+      return if preds.empty?
       allpreds = disambiguate_preds(preds)
       allpreds = canonicalize_localpreds(@rels, allpreds)
       # check for refs to collections that aren't being joined, Issue 191
@@ -211,8 +244,16 @@ module Bud
           end
         end
       end
-      @hashpreds = allpreds.reject { |p| p[0][0] != @rels[0].tabname or p[1][0] == @rels[0].tabname }
+      @hashpreds = allpreds.reject {|p| p[0][0] != @rels[0].tabname}
       @localpreds = @hashpreds
+      
+      # only allow preds on the same table name if they're on a self-joined table
+      @localpreds.each do |p| 
+        if p[0][0] == p[1][0] and not @selfjoins.include? p[0][0]
+          raise Bud::CompileError, "single-table predicate on #{p[0][0]} disallowed in joins" 
+        end
+      end
+      
       @localpreds += allpreds.map do |p|
         p if p[0][0] == p[1][0] and (p[0][0] == @rels[0].tabname or p[0][0] == @rels[1].tabname)
       end.compact
@@ -284,7 +325,8 @@ module Bud
     protected
     def canonicalize_localpreds(rel_list, preds) # :nodoc:all
       retval = preds.map do |p|
-        p[1][0] == rel_list[0].tabname ? p.reverse : p
+        # reverse if rhs is rel_list[0], *unless* it's a self-join!
+        (p[1][0] == rel_list[0].tabname and p[1][0] != p[0][0]) ? p.reverse : p
       end
     end
 
@@ -303,7 +345,8 @@ module Bud
       if (@localpreds and skips and @localpreds.length > skips.length)
         # check remainder of the predicates
         @localpreds.each do |pred|
-          next if skips.include? pred
+          # skip skips, and self-join preds
+          next if (skips.include? pred or pred[0][0] == pred[1][0])
           vals = []
           (0..1).each do |i|
             if pred[i][0] == @rels[0].tabname
@@ -327,7 +370,9 @@ module Bud
       @rels[0].each_from_sym([left_rel]) do |r|
         @rels[1].each_from_sym([right_rel]) do |s|
           s = [s] if origrels.length == 2
-          yield([r] + s) if test_locals(r, s)
+          if test_locals(r, s)
+              yield([r] + s)
+          end
         end
       end
     end
@@ -340,12 +385,8 @@ module Bud
       name, offset = entry[0], entry[1]
 
       # determine which subtuple of the collection contains the table
-      # referenced in entry.  note that origrels[0] is a base table
-      # on the left of the join, hence we shouldn't be calling join_offset on it
+      # referenced in entry.  
       subtuple = 0
-      if origrels[0].tabname == entry[0]
-        raise BudError, "join_offset called on the result of a single collection #{entry[0]}"
-      end
       origrels[1..origrels.length].each_with_index do |t,i|
         if t.tabname == entry[0]
           subtuple = i
@@ -393,38 +434,42 @@ module Bud
           r = [r] unless probe_ix == 1 and origrels.length > 2
           attrval = (probe_ix == 0) ? r[0][left_offset] : r[right_subtuple][right_offset]
 
-          # insert into the prober's hashtable only if symmetric ...
+          # insert into the prober's hashtable only if symmetric
           if probe_sym == other_sym
             @hash_tables[probe_ix][probe_sym][attrval] ||= []
             @hash_tables[probe_ix][probe_sym][attrval] << r
           end
 
           # ...and probe the other hashtable
-          next if @hash_tables[other_ix][other_sym][attrval].nil?
-          @hash_tables[other_ix][other_sym][attrval].each do |s_tup|
-            if probe_ix == 0
-              left = r; right = s_tup
-            else
-              left = s_tup; right = r
+          if @hash_tables[other_ix][other_sym][attrval].nil?
+            next
+          else
+            @hash_tables[other_ix][other_sym][attrval].each do |s_tup|
+              if probe_ix == 0
+                left = r; right = s_tup
+              else
+                left = s_tup; right = r
+              end
+              retval = left + right
+              yield retval if test_locals(left[0], right, @hashpreds.first)
             end
-            retval = left + right
-            yield retval if test_locals(left[0], right, @hashpreds.first)
           end
         end
       end
     end
   end
 
+  # intended to be used to extend a BudJoin instance
   module BudOuterJoin
     public
     def each(&block) # :nodoc:all
       super(&block)
-      # previous line finds all the matches.
-      # now its time to ``preserve'' the outer tuples with no matches.
-      # this is totally inefficient: we should fold the identification of non-matches
-      # into the join algorithms.  Another day.
-      # our trick: for each tuple of the outer, generate a singleton relation
-      # and join with inner.  If result is empty, preserve tuple.
+      # Previous line finds all the matches.  Now its time to ``preserve'' the
+      # outer tuples with no matches.  Our trick: for each tuple of the outer,
+      # generate a singleton relation and join with inner.  If result is empty,
+      # preserve tuple.
+      # XXX: This is totally inefficient: we should fold the identification of
+      # non-matches into the join algorithms.  Another day.
       @rels[0].each do |r|
         t = @origrels[0].clone_empty
         # need to uniquify the tablename here to avoid sharing join state with original

data/lib/bud/metrics.rb

@@ -0,0 +1,43 @@
+require "bud/errors"
+require 'faster_csv'
+
+# metrics are reported in a nested hash representing a collection of relational tables.
+# The metrics hash has the following form:
+# - key of the metrics hash is the name of the metric (table), e.g. "tickstats", "collections", "rules", etc.
+# - value of the metrics is itself a hash holding the rows of the table, keyed by key columns.
+# - It has the following form:
+#   - key is a hash of key attributes, with the following form:
+#     - key is the name of an attribute
+#     - value is the attribute's value
+#   - value is a single dependent value, e.g. a statistic like a count
+
+def report_metrics 
+  metrics.each do |k,v|
+    if v.first
+      csvstr = FasterCSV.generate(:force_quotes=>true) do |csv|
+        csv << [k.to_s] + v.first[0].keys + [:val]
+        v.each do |row|
+          csv << [nil] + row[0].values + [row[1]]
+        end
+      end
+      puts csvstr
+    end
+  end
+end
+
+def initialize_stats
+  return {{:name=>:count} => 0, {:name=>:mean} => 0, {:name=>:Q} => 0, {:name=>:stddev} => 0}
+end
+  
+# see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
+def running_stats(stats, elapsed)
+  raise Bud::BudError, "running_stats called with negative elapsed time" if elapsed < 0
+  stats[{:name=>:count}] += 1
+  oldmean = stats[{:name=>:mean}]
+  stats[{:name=>:mean}] = stats[{:name=>:mean}] + \
+                         (elapsed - stats[{:name=>:mean}]) / stats[{:name=>:count}]
+  stats[{:name=>:Q}] = stats[{:name=>:Q}] + \
+                       (elapsed - oldmean) * (elapsed - stats[{:name=>:mean}])
+  stats[{:name=>:stddev}] = Math.sqrt(stats[{:name=>:Q}]/stats[{:name=>:count}])
+  stats
+end

data/lib/bud/monkeypatch.rb

@@ -12,7 +12,7 @@ class Module
     # module X points to X's own nested import table.
     @bud_import_tbl ||= {}
     child_tbl = mod.bud_import_table
-    raise Bud::CompileError if @bud_import_tbl.has_key? local_name
+    raise Bud::CompileError, "import symbol #{local_name} already in use" if @bud_import_tbl.has_key? local_name
     @bud_import_tbl[local_name] = child_tbl.clone # XXX: clone needed?
 
     rewritten_mod_name = ModuleRewriter.do_import(self, mod, local_name)

data/lib/bud/rebl.rb

@@ -4,7 +4,7 @@ require 'rubygems'
 require 'bud'
 require 'abbrev'
 
-TABLE_TYPES = ["table", "scratch", "channel"]
+TABLE_TYPES = ["table", "scratch", "channel", "loopback", "periodic", "sync", "store"]
 
 # The class to which rebl adds user-specified rules and declarations.
 class ReblClass
@@ -97,7 +97,8 @@ class ReblShell
       line = Readline::readline('rebl> ') unless noreadline
       line = gets if noreadline
       do_exit if line.nil?
-      line = line.lstrip.rstrip
+      line.strip!
+      return if line.empty?
       Readline::HISTORY.push(line) unless noreadline
       split_line = line.split(" ")
       if line[0..0] == @@escape_char then
@@ -180,7 +181,7 @@ class ReblShell
     rescue Exception
       puts "Error when saving permanent history: #{$!}"
     end
-    @rebl_class_inst.stop_bg if @rebl_class_inst
+    @rebl_class_inst.stop if @rebl_class_inst
     puts "\n" + @@exit_message
     exit!
   end
@@ -212,12 +213,12 @@ class LibRebl
 
   # Runs the bud instance (until a breakpoint, or stop() is called)
   def run
-    @rebl_class_inst.sync_do {@rebl_class_inst.lazy = false}
+    @rebl_class_inst.run_bg
   end
 
   # Stops the bud instance (and then performs another tick)
   def stop
-    @rebl_class_inst.sync_do {@rebl_class_inst.lazy = true}
+    @rebl_class_inst.pause
   end
 
   # Ticks the bud instance a specified integer number of times.
@@ -274,13 +275,20 @@ class LibRebl
       if not @state.empty?
         @rebl_class.class_eval("state do\n" + @state.values.join("\n") + "\nend")
       end
-    rescue
+    rescue Exception
       raise
     end
 
     @old_inst = @rebl_class_inst
-    @rebl_class_inst = @rebl_class.new(:no_signal_handlers => true, :ip => @ip,
-                                       :port => @port, :lazy => true)
+    @rebl_class_inst = @rebl_class.new(:signal_handling => :none, :ip => @ip,
+                                       :port => @port)
+
+    # Stop the old instance. We want to copy the old instance's state over to
+    # the new instance and then startup the new instance. Any network messages
+    # received before the new instance has been started will be lost, but that
+    # can't easily be avoided; the best we can do is ensure we get a consistent
+    # snapshot of the old instance's state.
+    @old_inst.stop if @old_inst
 
     # Copy the tables over.
     if @old_inst
@@ -302,17 +310,16 @@ class LibRebl
 
     # Run lazily in background, shutting down old instance.
     begin
-      @old_inst.stop_bg if @old_inst
       # Lazify the instance upon a breakpoint (no effect if instance is
       # already lazy)
       @rebl_class_inst.register_callback(:rebl_breakpoint) do
-        @rebl_class_inst.lazy = true
+        @rebl_class_inst.pause
       end
-      @rebl_class_inst.run_bg
+      @rebl_class_inst.start
       @ip = @rebl_class_inst.ip
       @port = @rebl_class_inst.port
-      puts "Listening on #{@rebl_class_inst.ip_port}" if not @old_inst
-    rescue
+      puts "Listening on #{@ip}:#{@port}" unless @old_inst
+    rescue Exception
       # The above two need to be atomic, or we're in trouble.
       puts "unrecoverable error, please file a bug: #{$!}"
       abort