bud 0.9.4 → 0.9.9

data/lib/bud/aggs.rb

@@ -1,5 +1,3 @@
-require 'set'
-
 module Bud
   ######## Agg definitions
   class Agg #:nodoc: all
@@ -38,7 +36,7 @@ module Bud
 
   class Min < ArgExemplary #:nodoc: all
     def trans(the_state, val)
-      if the_state < val
+      if (the_state <=> val) < 0
         return the_state, :ignore
       elsif the_state == val
         return the_state, :keep
@@ -55,8 +53,10 @@ module Bud
 
   class Max < ArgExemplary #:nodoc: all
     def trans(the_state, val)
-      if the_state > val
+      if (the_state <=> val) > 0
         return the_state, :ignore
+      elsif the_state == val
+        return the_state, :keep
       else
         return val, :replace
       end
@@ -207,4 +207,20 @@ module Bud
   def accum(x)
     [Accum.new, x]
   end
+
+  class AccumPair < Agg #:nodoc: all
+    def init(fst, snd)
+      [[fst, snd]].to_set
+    end
+    def trans(the_state, fst, snd)
+      the_state << [fst, snd]
+      return the_state, nil
+    end
+  end
+
+  # aggregate method to be used in Bud::BudCollection.group.
+  # accumulates x, y inputs into a set of pairs (two element arrays).
+  def accum_pair(x, y)
+    [AccumPair.new, x, y]
+  end
 end

data/lib/bud/bud_meta.rb

@@ -1,11 +1,14 @@
 require 'bud/rewrite'
 
-
 class BudMeta #:nodoc: all
-  def initialize(bud_instance, declarations)
-    @bud_instance = bud_instance
-    @declarations = declarations
-    @dependency_analysis = nil # the results of bud_meta are analyzed further using a helper bloom instance. See depanalysis())
+  def initialize(bud_i)
+    @bud_instance = bud_i
+    @declarations = bud_i.methods.select {|m| m =~ /^__bloom__.+$/}.map {|m| m.to_s}
+    @rule_idx = 0
+
+    # The results of bud_meta are analyzed further using a helper Bloom
+    # instance. See depanalysis().
+    @dependency_analysis = nil
   end
 
   def meta_rewrite
@@ -13,20 +16,30 @@ class BudMeta #:nodoc: all
 
     stratified_rules = []
     if @bud_instance.toplevel == @bud_instance
-      nodes, stratum_map, top_stratum = stratify_preds
+      nodes = compute_node_graph
+      analyze_dependencies(nodes)
+
+      stratum_map = stratify_preds(nodes)
+      top_stratum = stratum_map.values.max
+      top_stratum ||= -1
 
       # stratum_map = {fully qualified pred => stratum}. Copy stratum_map data
       # into t_stratum format.
       raise unless @bud_instance.t_stratum.to_a.empty?
-      @bud_instance.t_stratum <= stratum_map.to_a
+      @bud_instance.t_stratum.merge(stratum_map.to_a)
 
-      # slot each rule into the stratum corresponding to its lhs pred (from stratum_map)
       stratified_rules = Array.new(top_stratum + 2) { [] }  # stratum -> [ rules ]
       @bud_instance.t_rules.each do |rule|
         if rule.op == '<='
           # Deductive rules are assigned to strata based on the basic Datalog
-          # stratification algorithm
-          belongs_in = stratum_map[rule.lhs]
+          # stratification algorithm. Note that we don't place all the rules
+          # with a given lhs relation in the same strata; rather, we place a
+          # rule in the lowest strata we can, as determined by the rule's rhs
+          # relations (and whether the rel is used in a non-monotonic context).
+          body_rels = find_body_rels(rule)
+          body_strata = body_rels.map {|r,is_nm| stratum_map[r] + (is_nm ? 1 : 0) || 0}
+          belongs_in = body_strata.max
+
           # If the rule body doesn't reference any collections, it won't be
           # assigned a stratum, so just place it in stratum zero
           belongs_in ||= 0
@@ -36,28 +49,39 @@ class BudMeta #:nodoc: all
           stratified_rules[top_stratum + 1] << rule
         end
       end
-      # stratified_rules[0] may be empty if none of the nodes at stratum 0 are on the lhs
-      # stratified_rules[top_stratum+1] will be empty if there are no temporal rules.
-      # Cleanup
-      stratified_rules = stratified_rules.reject{|r| r.empty?}
+
+      # stratified_rules[0] may be empty if none of the nodes at stratum 0 are
+      # on the lhs stratified_rules[top_stratum+1] will be empty if there are no
+      # temporal rules.
+      stratified_rules.reject! {|r| r.empty?}
+
+      stratified_rules.each_with_index do |strat,strat_num|
+        strat.each do |rule|
+          @bud_instance.t_rule_stratum << [rule.bud_obj, rule.rule_id, strat_num]
+        end
+      end
+
       dump_rewrite(stratified_rules) if @bud_instance.options[:dump_rewrite]
     end
     return stratified_rules
   end
 
+  def find_body_rels(rule)
+    @bud_instance.t_depends.map do |d|
+      [d.body, d.nm] if d.rule_id == rule.rule_id and d.bud_obj == rule.bud_obj
+    end.compact
+  end
+
   def shred_rules
-    # to completely characterize the rules of a bud class we must extract
-    # from all parent classes/modules
-    # after making this pass, we no longer care about the names of methods.
-    # we are shredding down to the granularity of rule heads.
-    seed = 0
+    # After making this pass, we no longer care about the names of methods.  We
+    # are shredding down to the granularity of rule heads.
     rulebag = {}
     @bud_instance.class.ancestors.reverse.each do |anc|
       @declarations.each do |meth_name|
-        rw = rewrite_rule_block(anc, meth_name, seed)
+        rw = rewrite_rule_block(anc, meth_name)
         if rw
-          seed = rw.rule_indx
           rulebag[meth_name] = rw
+          @rule_idx = rw.rule_idx
         end
       end
     end
@@ -68,7 +92,7 @@ class BudMeta #:nodoc: all
     end
   end
 
-  def rewrite_rule_block(klass, block_name, seed)
+  def rewrite_rule_block(klass, block_name)
     return unless klass.respond_to? :__bloom_asts__
 
     pt = klass.__bloom_asts__[block_name]
@@ -104,18 +128,18 @@ class BudMeta #:nodoc: all
       end
       raise Bud::CompileError, "#{error_msg} in rule block \"#{block_name}\"#{src_msg}"
     end
-    rewriter = RuleRewriter.new(seed, @bud_instance)
+    rewriter = RuleRewriter.new(@bud_instance, @rule_idx)
     rewriter.process(pt)
     return rewriter
   end
 
   def get_qual_name(pt)
     # expect to see a parse tree corresponding to a dotted name
-    #    a.b.c == s(:call, s1,  :c, (:args))
-    # where s1 == s(:call, s2,  :b, (:args))
-    # where s2 == s(:call, nil, :a, (:args))
-    tag, recv, name, args = pt
-    return nil unless tag == :call and args.length == 1
+    #    a.b.c == s(:call, s1,  :c)
+    # where s1 == s(:call, s2,  :b)
+    # where s2 == s(:call, nil, :a)
+    tag, recv, name, *args = pt
+    return nil unless tag == :call and args.empty?
 
     if recv
       qn = get_qual_name(recv)
@@ -127,23 +151,16 @@ class BudMeta #:nodoc: all
   end
 
   # Perform some basic sanity checks on the AST of a rule block. We expect a
-  # rule block to consist of a :defn, a nested :scope, and then a sequence of
+  # rule block to consist of a :defn whose body consists of a sequence of
   # statements. Each statement is a :call node. Returns nil (no error found), a
   # Sexp (containing an error), or a pair of [Sexp, error message].
   def check_rule_ast(pt)
-    # :defn format: node tag, block name, args, nested scope
-    return pt if pt.sexp_type != :defn
-    scope = pt[3]
-    return pt if scope.sexp_type != :scope
-    block = scope[1]
-
-    block.each_with_index do |n,i|
-      if i == 0
-        return pt if n != :block
-        next
-      end
+    # :defn format: node tag, block name, args, body_0, ..., body_n
+    tag, name, args, *body = pt
+    return pt if tag != :defn
 
-      next if i == 1 and n.sexp_type == :nil # a block got rewritten to an empty block
+    body.each_with_index do |n,i|
+      next if i == 0 and n == s(:nil) # a block got rewritten to an empty block
 
       # Check for a common case
       if n.sexp_type == :lasgn
@@ -157,8 +174,9 @@ class BudMeta #:nodoc: all
       # Check that LHS references a named collection
       lhs_name = get_qual_name(lhs)
       return [n, "unexpected lhs format: #{lhs}"] if lhs_name.nil?
-      unless @bud_instance.tables.has_key? lhs_name.to_sym
-        return [n, "collection does not exist: '#{lhs_name}'"]
+      unless @bud_instance.tables.has_key? lhs_name.to_sym or
+             @bud_instance.lattices.has_key? lhs_name.to_sym
+        return [n, "Collection does not exist: '#{lhs_name}'"]
       end
 
       return [n, "illegal operator: '#{op}'"] unless [:<, :<=].include? op
@@ -171,13 +189,11 @@ class BudMeta #:nodoc: all
       # XXX: We don't check for illegal superators (e.g., "<--"). That would be
       # tricky, because they are encoded as a nested unary op in the rule body.
       if op == :<
-        return n unless rhs.sexp_type == :arglist
-        body = rhs[1]
-        return n unless body.sexp_type == :call
-        op_tail = body[2]
+        return n unless rhs.sexp_type == :call
+        op_tail = rhs[2]
         return n unless [:~, :-@, :+@].include? op_tail
-        rhs_args = body[3]
-        return n if rhs_args.sexp_type != :arglist or rhs_args.length != 1
+        rhs_args = rhs[3..-1]
+        return n unless rhs_args.empty?
       end
     end
 
@@ -185,64 +201,61 @@ class BudMeta #:nodoc: all
   end
 
 
-  Node = Struct.new :name, :status, :stratum, :edges, :in_lhs, :in_body, :in_cycle, :is_neg_head
   # Node.status is one of :init, :in_progress, :done
+  Node = Struct.new :name, :status, :stratum, :edges, :in_lhs, :in_body, :already_neg
   Edge = Struct.new :to, :op, :neg, :temporal
 
-  def stratify_preds
-    bud = @bud_instance.toplevel
+  def compute_node_graph
     nodes = {}
-    bud.t_depends.each do |d|
-      #t_depends [:bud_instance, :rule_id, :lhs, :op, :body] => [:nm]
-      lhs = (nodes[d.lhs] ||= Node.new(d.lhs, :init, 0, [], true, false, false, false))
+    @bud_instance.toplevel.t_depends.each do |d|
+      # t_depends [:bud_instance, :rule_id, :lhs, :op, :body] => [:nm, :in_body]
+      lhs = (nodes[d.lhs] ||= Node.new(d.lhs, :init, 0, [], true, false))
       lhs.in_lhs = true
-      body = (nodes[d.body] ||= Node.new(d.body, :init, 0, [], false, true, false, false))
-      temporal = d.op != "<="
-      lhs.edges << Edge.new(body, d.op, d.nm, temporal)
+      body = (nodes[d.body] ||= Node.new(d.body, :init, 0, [], false, true))
       body.in_body = true
+      temporal = (d.op != "<=")
+      lhs.edges << Edge.new(body, d.op, d.nm, temporal)
     end
 
+    return nodes
+  end
+
+  def stratify_preds(nodes)
     nodes.each_value {|n| calc_stratum(n, false, false, [n.name])}
+
     # Normalize stratum numbers because they may not be 0-based or consecutive
     remap = {}
     # if the nodes stratum numbers are [2, 3, 2, 4], remap = {2 => 0, 3 => 1, 4 => 2}
     nodes.values.map {|n| n.stratum}.uniq.sort.each_with_index do |num, i|
       remap[num] = i
     end
+
     stratum_map = {}
-    top_stratum = -1
     nodes.each_pair do |name, n|
       n.stratum = remap[n.stratum]
       stratum_map[n.name] = n.stratum
-      top_stratum = max(top_stratum, n.stratum)
     end
-    analyze_dependencies(nodes)
-    return nodes, stratum_map, top_stratum
+    return stratum_map
   end
 
-  def max(a, b) ; a > b ? a : b ; end
-
   def calc_stratum(node, neg, temporal, path)
     if node.status == :in_process
-      node.in_cycle = true
-      if neg and !temporal and node.is_neg_head
+      if neg and not temporal and not node.already_neg
         raise Bud::CompileError, "unstratifiable program: #{path.uniq.join(',')}"
       end
     elsif node.status == :init
       node.status = :in_process
       node.edges.each do |edge|
-        node.is_neg_head = edge.neg
         next unless edge.op == "<="
+        node.already_neg = neg
         body_stratum = calc_stratum(edge.to, (neg or edge.neg), (edge.temporal or temporal), path + [edge.to.name])
-        node.is_neg_head = false # reset for next edge
-        node.stratum = max(node.stratum, body_stratum + (edge.neg ? 1 : 0))
+        node.stratum = [node.stratum, body_stratum + (edge.neg ? 1 : 0)].max
       end
       node.status = :done
     end
     node.stratum
   end
 
-
   def analyze_dependencies(nodes)  # nodes = {node name => node}
     preds_in_lhs = nodes.select {|_, node| node.in_lhs}.map {|name, _| name}.to_set
     preds_in_body = nodes.select {|_, node| node.in_body}.map {|name, _| name}.to_set
@@ -274,9 +287,6 @@ class BudMeta #:nodoc: all
       da = ::DepAnalysis.new
       da.providing <+ @bud_instance.tables[:t_provides].to_a
       da.depends <+ @bud_instance.t_depends.map{|d| [d.lhs, d.op, d.body, d.nm]}
-
-      #@bud_instance.tables[:t_provides].each {|t| da.providing <+ t}
-      #@bud_instance.tables[:t_depends].each {|t| da.depends_tc <+ t}
       da.tick_internal
       @dependency_analysis = da
     end

data/lib/bud/collections.rb

@@ -1,6 +1,3 @@
-require 'msgpack'
-
-$struct_classes = {}
 module Bud
   ########
   #--
@@ -16,11 +13,11 @@ module Bud
   class BudCollection
     include Enumerable
 
-    attr_accessor :bud_instance, :tabname  # :nodoc: all
-    attr_reader :cols, :key_cols # :nodoc: all
+    attr_accessor :bud_instance  # :nodoc: all
+    attr_reader :tabname, :cols, :key_cols # :nodoc: all
     attr_reader :struct
-    attr_reader :storage, :delta, :new_delta, :pending, :tick_delta # :nodoc: all
-    attr_reader :wired_by, :to_delete
+    attr_reader :new_delta, :pending # :nodoc: all
+    attr_reader :wired_by, :scanner_cnt
     attr_accessor :invalidated, :rescan
     attr_accessor :is_source
     attr_accessor :accumulate_tick_deltas # updated in bud.do_wiring
@@ -30,6 +27,7 @@ module Bud
       @bud_instance = bud_instance
       @invalidated = true
       @is_source = true # unless it shows up on the lhs of some rule
+      @scanner_cnt = 0
       @wired_by = []
       @accumulate_tick_deltas = false
       init_schema(given_schema) unless given_schema.nil? and defer_schema
@@ -54,16 +52,17 @@ module Bud
       # user-specified schema.
       @cols.each do |s|
         if s.to_s.start_with? "@"
-          raise Bud::Error, "illegal use of location specifier (@) in column #{s} of non-channel collection #{tabname}"
+          raise Bud::CompileError, "illegal use of location specifier (@) in column #{s} of non-channel collection #{tabname}"
         end
       end
 
       @key_colnums = @key_cols.map {|k| @cols.index(k)}
+      @val_colnums = val_cols.map {|k| @cols.index(k)}
 
       if @cols.empty?
         @cols = nil
       else
-        @struct = ($struct_classes[@cols] ||= Struct.new(*@cols))
+        @struct = Bud::TupleStruct.new_struct(@cols)
         @structlen = @struct.members.length
       end
       setup_accessors
@@ -132,11 +131,19 @@ module Bud
         end
       end
 
-      # set up schema accessors, which are class methods
+      # Setup schema accessors, which are class methods. Note that the same
+      # table/column name might appear multiple times on the LHS of a single
+      # join (e.g., (foo * bar).combos(foo.x => bar.y, foo.x => bar.z)). Because
+      # the join predicates are represented as a hash, we need the two instances
+      # of foo.x to be distinct values (otherwise the resulting hash will only
+      # have a single key). Hence, we add a unique ID to the value returned by
+      # schema accessors.
       @cols_access = Module.new do
         sc.each_with_index do |c, i|
           define_method c do
-            [@tabname, i, c]
+            @counter ||= 0
+            @counter += 1
+            [qualified_tabname, i, c, @counter]
           end
         end
       end
@@ -189,13 +196,27 @@ module Bud
       if @bud_instance.wiring?
         pusher = to_push_elem(the_name, the_schema)
         # If there is no code block evaluate, use the scanner directly
-        return pusher if blk.nil?
-        pusher_pro = pusher.pro(&blk)
-        pusher_pro.elem_name = the_name
-        pusher_pro.tabname = the_name
-        pusher_pro
+        pusher = pusher.pro(&blk) unless blk.nil?
+        pusher
       else
-        @storage.map(&blk)
+        rv = []
+        self.each do |t|
+          t = blk.call(t)
+          rv << t unless t.nil?
+        end
+        rv
+      end
+    end
+
+    # XXX: Although we support each_with_index over Bud collections, using it is
+    # probably not a great idea: the index assigned to a given collection member
+    # is not defined by the language semantics.
+    def each_with_index(the_name=tabname, the_schema=schema, &blk)
+      if @bud_instance.wiring?
+        pusher = to_push_elem(the_name, the_schema)
+        pusher.each_with_index(&blk)
+      else
+        super(&blk)
       end
     end
 
@@ -218,7 +239,7 @@ module Bud
         end
         elem.set_block(&f)
         toplevel.push_elems[[self.object_id, :flatten]] = elem
-        return elem
+        elem
       else
         @storage.flat_map(&blk)
       end
@@ -228,16 +249,18 @@ module Bud
     def sort(&blk)
       if @bud_instance.wiring?
         pusher = self.pro
-        pusher.sort("sort#{object_id}", @bud_instance, @cols, &blk)
+        pusher.sort("sort#{object_id}".to_sym, @bud_instance, @cols, &blk)
       else
-        @storage.sort
+        @storage.values.sort(&blk)
       end
     end
 
     def rename(the_name, the_schema=nil, &blk)
-      raise unless @bud_instance.wiring?
+      raise Bud::Error unless @bud_instance.wiring?
       # a scratch with this name should have been defined during rewriting
-      raise Bud::Error, "rename failed to define a scratch named #{the_name}" unless @bud_instance.respond_to? the_name
+      unless @bud_instance.respond_to? the_name
+        raise Bud::Error, "rename failed to define a scratch named #{the_name}"
+      end
       pro(the_name, the_schema, &blk)
     end
 
@@ -251,7 +274,12 @@ module Bud
 
     public
     def each_raw(&block)
-      @storage.each_value(&block)
+      each_from([@storage], &block)
+    end
+
+    public
+    def each_delta(&block)
+      each_from([@delta], &block)
     end
 
     public
@@ -266,43 +294,37 @@ module Bud
 
     public
     def non_temporal_predecessors
-      @wired_by.select {|elem| elem.outputs.include? self}
+      @wired_by.select {|e| e.outputs.include? self}
+    end
+
+    public
+    def positive_predecessors
+      @wired_by.select {|e| e.outputs.include?(self) || e.pendings.include?(self)}
     end
 
     public
     def tick_metrics
       strat_num = bud_instance.this_stratum
-      rule_num = bud_instance.this_rule
-      addr = nil
       addr = bud_instance.ip_port unless bud_instance.port.nil?
+      key = { :addr=>addr, :tabname=>qualified_tabname,
+              :strat_num=>strat_num}
+
       bud_instance.metrics[:collections] ||= {}
-      bud_instance.metrics[:collections][{:addr=>addr, :tabname=>qualified_tabname, :strat_num=>strat_num, :rule_num=>rule_num}] ||= 0
-      bud_instance.metrics[:collections][{:addr=>addr, :tabname=>qualified_tabname, :strat_num=>strat_num, :rule_num=>rule_num}] += 1
+      bud_instance.metrics[:collections][key] ||= 0
+      bud_instance.metrics[:collections][key] += 1
     end
 
     private
     def each_from(bufs, &block) # :nodoc: all
+      do_metrics = bud_instance.options[:metrics]
       bufs.each do |b|
         b.each_value do |v|
-          tick_metrics if bud_instance and bud_instance.options[:metrics]
+          tick_metrics if do_metrics
           yield v
         end
       end
     end
 
-    public
-    def each_from_sym(buf_syms, &block) # :nodoc: all
-      bufs = buf_syms.map do |s|
-        case s
-        when :storage then @storage
-        when :delta then @delta
-        when :new_delta then @new_delta
-        else raise Bud::Error, "bad symbol passed into each_from_sym"
-        end
-      end
-      each_from(bufs, &block)
-    end
-
     private
     def init_storage
       @storage = {}
@@ -345,7 +367,7 @@ module Bud
     # checks for +item+ in the collection
     public
     def include?(item)
-      return true if key_cols.nil? or (key_cols.empty? and length > 0)
+      return true if key_cols.nil?
       return false if item.nil?
       key = get_key_vals(item)
       return (item == self[key])
@@ -374,13 +396,18 @@ module Bud
     private
     def raise_pk_error(new, old)
       key = get_key_vals(old)
-      raise Bud::KeyConstraintError, "key conflict inserting #{new.inspect} into \"#{tabname}\": existing tuple #{old.inspect}, key = #{key.inspect}"
+      raise Bud::KeyConstraintError, "key conflict inserting #{new.inspect} into \"#{qualified_tabname}\": existing tuple #{old.inspect}, key = #{key.inspect}"
+    end
+
+    private
+    def is_lattice_val(v)
+      v.kind_of? Bud::Lattice
     end
 
     private
     def prep_tuple(o)
       return o if o.class == @struct
-      if o.class == Array
+      if o.kind_of? Array
         if @struct.nil?
           sch = (1 .. o.length).map{|i| "c#{i}".to_sym}
           init_schema(sch)
@@ -391,9 +418,16 @@ module Bud
         raise Bud::TypeError, "array or struct type expected in \"#{qualified_tabname}\": #{o.inspect}"
       end
 
+      @key_colnums.each do |i|
+        next if i >= o.length
+        if is_lattice_val(o[i])
+          raise Bud::TypeError, "lattice value cannot be a key for #{qualified_tabname}: #{o[i].inspect}"
+        end
+      end
       if o.length > @structlen
         raise Bud::TypeError, "too many columns for \"#{qualified_tabname}\": #{o.inspect}"
       end
+
       return @struct.new(*o)
     end
 
@@ -411,7 +445,7 @@ module Bud
                       when @delta.object_id; "delta"
                       when @new_delta.object_id; "new_delta"
                     end
-        puts "#{qualified_tabname}.#{storetype} ==> #{t}"
+        puts "#{qualified_tabname}.#{storetype} ==> #{t.inspect}"
       end
       return if t.nil? # silently ignore nils resulting from map predicates failing
       t = prep_tuple(t)
@@ -420,13 +454,44 @@ module Bud
     end
 
     # Merge "tup" with key values "key" into "buf". "old" is an existing tuple
-    # with the same key columns as "tup" (if any such tuple exists).
+    # with the same key columns as "tup" (if any such tuple exists). If "old"
+    # exists and "tup" is not a duplicate, check whether the two tuples disagree
+    # on a non-key, non-lattice value; if so, raise a PK error. Otherwise,
+    # construct and return a merged tuple by using lattice merge functions.
     private
     def merge_to_buf(buf, key, tup, old)
-      if old.nil?       # no matching tuple found
+      if old.nil?               # no matching tuple found
         buf[key] = tup
-      elsif old != tup  # ignore duplicates
-        raise_pk_error(tup, old)
+        return
+      end
+      return if tup == old      # ignore duplicates
+
+      # Check for PK violation
+      @val_colnums.each do |i|
+        old_v = old[i]
+        new_v = tup[i]
+
+        unless old_v == new_v || (is_lattice_val(old_v) && is_lattice_val(new_v))
+          raise_pk_error(tup, old)
+        end
+      end
+
+      # Construct new tuple version. We discard the newly-constructed tuple if
+      # merging every lattice field doesn't yield a new value.
+      new_t = null_tuple
+      saw_change = false
+      @val_colnums.each do |i|
+        if old[i] == tup[i]
+          new_t[i] = old[i]
+        else
+          new_t[i] = old[i].merge(tup[i])
+          saw_change = true if new_t[i].reveal != old[i].reveal
+        end
+      end
+
+      if saw_change
+        @key_colnums.each {|k| new_t[k] = old[k]}
+        buf[key] = new_t
       end
     end
 
@@ -508,6 +573,12 @@ module Bud
         add_merge_target
         tbl = register_coll_expr(o)
         tbl.pro.wire_to self
+      elsif o.class <= Bud::LatticePushElement
+        add_merge_target
+        o.wire_to self
+      elsif o.class <= Bud::LatticeWrapper
+        add_merge_target
+        o.to_push_elem.wire_to self
       else
         unless o.nil?
           o = o.uniq.compact if o.respond_to?(:uniq)
@@ -528,6 +599,10 @@ module Bud
     public
     # instantaneously merge items from collection +o+ into +buf+
     def <=(collection)
+      unless bud_instance.toplevel.inside_tick
+        raise Bud::CompileError, "illegal use of <= outside of bloom block, use <+ instead"
+      end
+
       merge(collection)
     end
 
@@ -557,11 +632,22 @@ module Bud
         add_merge_target
         tbl = register_coll_expr(o)
         tbl.pro.wire_to(self, :pending)
+      elsif o.class <= Bud::LatticePushElement
+        add_merge_target
+        o.wire_to(self, :pending)
+      elsif o.class <= Bud::LatticeWrapper
+        add_merge_target
+        o.to_push_elem.wire_to(self, :pending)
       else
         pending_merge(o)
       end
     end
 
+    superator "<~" do |o|
+      # Overridden when <~ is defined (i.e., channels and terminals)
+      raise Bud::CompileError, "#{tabname} cannot appear on the lhs of a <~ operator"
+    end
+
     def tick
       raise Bud::Error, "tick must be overriden in #{self.class}"
     end
@@ -634,6 +720,7 @@ module Bud
                                           self, the_schema)
         toplevel.scanners[this_stratum][[oid, the_name]] = scanner
         toplevel.push_sources[this_stratum][[oid, the_name]] = scanner
+        @scanner_cnt += 1
       end
       return toplevel.scanners[this_stratum][[oid, the_name]]
     end
@@ -656,10 +743,10 @@ module Bud
     # for each distinct value of the grouping key columns, return the items in that group
     # that have the value of the exemplary aggregate +aggname+
     public
-    def argagg(aggname, gbkey_cols, collection)
+    def argagg(aggname, gbkey_cols, collection, &blk)
       elem = to_push_elem
       gbkey_cols = gbkey_cols.map{|k| canonicalize_col(k)} unless gbkey_cols.nil?
-      retval = elem.argagg(aggname, gbkey_cols, canonicalize_col(collection))
+      retval = elem.argagg(aggname, gbkey_cols, canonicalize_col(collection), &blk)
       # PushElement inherits the schema accessors from this Collection
       retval.extend @cols_access
       retval
@@ -669,37 +756,46 @@ module Bud
     # that group that have the minimum value of the attribute +col+. Note that
     # multiple tuples might be returned.
     public
-    def argmin(gbkey_cols, col)
-      argagg(:min, gbkey_cols, col)
+    def argmin(gbkey_cols, col, &blk)
+      argagg(:min, gbkey_cols, col, &blk)
     end
 
     # for each distinct value of the grouping key columns, return the items in
     # that group that have the maximum value of the attribute +col+. Note that
     # multiple tuples might be returned.
     public
-    def argmax(gbkey_cols, col)
-      argagg(:max, gbkey_cols, col)
+    def argmax(gbkey_cols, col, &blk)
+      argagg(:max, gbkey_cols, col, &blk)
     end
 
     # form a collection containing all pairs of items in +self+ and items in
     # +collection+
     public
     def *(collection)
-      elem1 = to_push_elem
-      return elem1.join(collection)
+      return to_push_elem.join(collection)
+    end
+
+    def prep_aggpairs(aggpairs)
+      aggpairs.map do |ap|
+        agg, *rest = ap
+        if rest.empty?
+          [agg]
+        else
+          [agg] + rest.map {|c| canonicalize_col(c)}
+        end
+      end
     end
 
     def group(key_cols, *aggpairs, &blk)
-      elem = to_push_elem
       key_cols = key_cols.map{|k| canonicalize_col(k)} unless key_cols.nil?
-      aggpairs = aggpairs.map{|ap| [ap[0], canonicalize_col(ap[1])].compact} unless aggpairs.nil?
-      return elem.group(key_cols, *aggpairs, &blk)
+      aggpairs = prep_aggpairs(aggpairs)
+      return to_push_elem.group(key_cols, *aggpairs, &blk)
     end
 
     def notin(collection, *preds, &blk)
       elem1 = to_push_elem
       elem2 = collection.to_push_elem
-      return elem1.notin(elem2, preds, &blk)
+      return elem1.notin(elem2, *preds, &blk)
     end
 
     def canonicalize_col(col)
@@ -725,6 +821,7 @@ module Bud
       false
     end
 
+    # tick_delta for scratches is @storage, so iterate over that instead
     public
     def each_tick_delta(&block)
       @storage.each_value(&block)
@@ -751,6 +848,11 @@ module Bud
     public
     def add_rescan_invalidate(rescan, invalidate)
       srcs = non_temporal_predecessors
+
+      # XXX: this seems wrong. We might rescan a node for many reasons (e.g.,
+      # because another one of the node's outputs needs to be refilled). We only
+      # need to invalidate + rescan this scratch if one of the inputs to this
+      # collection is *invalidated*.
       if srcs.any? {|e| rescan.member? e}
         invalidate << self
         rescan.merge(srcs)
@@ -786,6 +888,8 @@ module Bud
       given_schema ||= [:@address, :val]
       @is_loopback = loopback
       @locspec_idx = nil
+      @wire_buf = StringIO.new
+      @packer = MessagePack::Packer.new(@wire_buf)
 
       # We're going to mutate the caller's given_schema (to remove the location
       # specifier), so make a deep copy first. We also save a ref to the
@@ -870,28 +974,66 @@ module Bud
           raise Bud::Error, "'#{t[@locspec_idx]}', channel '#{@tabname}'" if the_locspec[0].nil? or the_locspec[1].nil? or the_locspec[0] == '' or the_locspec[1] == ''
         end
         puts "channel #{qualified_tabname}.send: #{t}" if $BUD_DEBUG
-        toplevel.dsock.send_datagram([qualified_tabname.to_s, t].to_msgpack,
+
+        # Convert the tuple into a suitable wire format. Because MsgPack cannot
+        # marshal arbitrary Ruby objects that we need to send via channels (in
+        # particular, lattice values and Class instances), we first encode such
+        # values using Marshal, and then encode the entire tuple with
+        # MsgPack. Obviously, this is gross. The wire format also includes an
+        # array of indices, indicating which fields hold Marshall'd values.
+        @packer.write_array_header(3)
+        @packer.write(qualified_tabname.to_s)
+        # The second element, wire_tuple, is an array.  We will write it one
+        # element at a time:
+        @packer.write_array_header(t.length)
+        @packer.flush
+        marshall_indexes = []
+        t.each_with_index do |f,i|
+          # Performance optimization for cases where we know that we can't
+          # marshal the field using MsgPack:
+          if [Bud::Lattice, Class].any?{|t| f.class <= t}
+            marshall_indexes << i
+            @wire_buf << Marshal.dump(f).to_msgpack
+          else
+            begin
+              @wire_buf << f.to_msgpack
+            rescue NoMethodError
+              # If MsgPack can't marshal the field, fall back to Marshal.
+              # This handles fields that contain nested non-MsgPack-able
+              # objects (in these cases, the entire field is Marshal'd.)
+              marshall_indexes << i
+              @wire_buf << Marshal.dump(f).to_msgpack
+            end
+          end
+        end
+        @packer.write(marshall_indexes)
+        @packer.flush
+        toplevel.dsock.send_datagram(@wire_buf.string,
                                      the_locspec[0], the_locspec[1])
+
+        # Reset output buffer
+        @wire_buf.rewind
+        @wire_buf.truncate(0)
       end
       @pending.clear
     end
 
     public
     # project to the non-address fields
-    def payloads
-      return self.pro if @is_loopback
-
-      if cols.size > 2
-        # bundle up each tuple's non-locspec fields into an array
-        retval = case @locspec_idx
-          when 0 then self.pro{|t| t.values_at(1..(t.size-1))}
-          when (schema.size - 1) then self.pro{|t| t.values_at(0..(t.size-2))}
-          else self.pro{|t| t.values_at(0..(@locspec_idx-1), @locspec_idx+1..(t.size-1))}
-        end
-      else
-        # just return each tuple's non-locspec field value
-        retval = self.pro{|t| t[(@locspec_idx == 0) ? 1 : 0]}
+    def payloads(&blk)
+      return self.pro(&blk) if @is_loopback
+
+      if @payload_struct.nil?
+        payload_cols = cols.dup
+        payload_cols.delete_at(@locspec_idx)
+        @payload_struct = Bud::TupleStruct.new(*payload_cols)
+        @payload_colnums = payload_cols.map {|k| cols.index(k)}
+      end
+
+      retval = self.pro do |t|
+        @payload_struct.new(*t.values_at(*@payload_colnums))
       end
+      retval = retval.pro(&blk) unless blk.nil?
       return retval
     end
 
@@ -903,51 +1045,40 @@ module Bud
       elsif o.class <= Proc
         tbl = register_coll_expr(o)
         tbl.pro.wire_to(self, :pending)
+      elsif o.class <= Bud::LatticePushElement
+        add_merge_target
+        o.wire_to(self, :pending)
+      elsif o.class <= Bud::LatticeWrapper
+        add_merge_target
+        o.to_push_elem.wire_to(self, :pending)
       else
         pending_merge(o)
       end
     end
 
     superator "<+" do |o|
-      raise Bud::Error, "illegal use of <+ with channel '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <+ with channel '#{@tabname}' on left"
     end
 
     undef merge
 
     def <=(o)
-      raise Bud::Error, "illegal use of <= with channel '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with channel '#{@tabname}' on left"
     end
   end
 
   class BudTerminal < BudScratch # :nodoc: all
-    def initialize(name, given_schema, bud_instance, prompt=false) # :nodoc: all
-      super(name, bud_instance, given_schema)
+    def initialize(name, bud_instance, prompt=false) # :nodoc: all
+      super(name, bud_instance, [:line])
       @prompt = prompt
     end
 
     public
     def start_stdin_reader # :nodoc: all
-      # XXX: Ugly hack. Rather than sending terminal data to EM via UDP,
-      # we should add the terminal file descriptor to the EM event loop.
-      @reader = Thread.new do
+      Thread.new do
         begin
-          toplevel = @bud_instance.toplevel
           while true
-            out_io = get_out_io
-            out_io.print("#{tabname} > ") if @prompt
-
-            in_io = toplevel.options[:stdin]
-            s = in_io.gets
-            break if s.nil? # Hit EOF
-            s = s.chomp if s
-            tup = [s]
-
-            ip = toplevel.ip
-            port = toplevel.port
-            EventMachine::schedule do
-              socket = EventMachine::open_datagram_socket("127.0.0.1", 0)
-              socket.send_datagram([tabname, tup].to_msgpack, ip, port)
-            end
+            break unless read_line
           end
         rescue Exception
           puts "terminal reader thread failed: #{$!}"
@@ -957,13 +1088,37 @@ module Bud
       end
     end
 
+    # XXX: Ugly hack. Rather than sending terminal data to EM via UDP, we should
+    # add the terminal file descriptor to the EM event loop.
+    private
+    def read_line
+      get_out_io.print("#{tabname} > ") if @prompt
+
+      toplevel = @bud_instance.toplevel
+      in_io = toplevel.options[:stdin]
+      input_str = in_io.gets
+      return false if input_str.nil? # Hit EOF
+      input_str.chomp!
+
+      EventMachine::schedule do
+        socket = EventMachine::open_datagram_socket("127.0.0.1", 0)
+        socket.send_datagram([tabname, [input_str], []].to_msgpack,
+                             toplevel.ip, toplevel.port)
+      end
+
+      return true
+    end
+
+    public
+    def bootstrap
+      # override BudCollection; pending should not be moved into delta.
+    end
+
     public
     def flush #:nodoc: all
       out_io = get_out_io
-      @pending.each_value do |p|
-        out_io.puts p[0]
-        out_io.flush
-      end
+      @pending.each_value {|p| out_io.puts p[0]}
+      out_io.flush
       @pending.clear
     end
 
@@ -975,7 +1130,7 @@ module Bud
     public
     def tick #:nodoc: all
       unless @pending.empty?
-        @delta = @pending #  pending used for input tuples in this case.
+        @delta = @pending       # pending used for input tuples in this case
         @tick_delta = @pending.values
         @pending.clear
       else
@@ -983,8 +1138,7 @@ module Bud
         @delta.clear
         @tick_delta.clear
       end
-      @invalidated = true # channels and terminals are always invalidated.
-      raise Bud::Error, "orphaned pending tuples in terminal" unless @pending.empty?
+      @invalidated = true       # channels and terminals are always invalidated
     end
 
     public
@@ -995,7 +1149,11 @@ module Bud
 
     public
     def <=(o) #:nodoc: all
-      raise Bud::Error, "illegal use of <= with terminal '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with terminal '#{@tabname}' on left"
+    end
+
+    superator "<+" do |o|
+      raise Bud::CompileError, "illegal use of <+ with terminal '#{@tabname}' on left"
     end
 
     superator "<~" do |o|
@@ -1015,26 +1173,28 @@ module Bud
     def get_out_io
       rv = @bud_instance.toplevel.options[:stdout]
       rv ||= $stdout
-      raise Bud::Error, "attempting to write to terminal #{tabname} that was already closed" if rv.closed?
+      if rv.closed?
+        raise Bud::Error, "attempt to write to closed terminal '#{tabname}'"
+      end
       rv
     end
   end
 
   class BudPeriodic < BudScratch # :nodoc: all
     def <=(o)
-      raise Bud::Error, "illegal use of <= with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with periodic '#{tabname}' on left"
     end
 
     superator "<~" do |o|
-      raise Bud::Error, "illegal use of <~ with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <~ with periodic '#{tabname}' on left"
     end
 
     superator "<-" do |o|
-      raise Bud::Error, "illegal use of <- with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <- with periodic '#{tabname}' on left"
     end
 
     superator "<+" do |o|
-      raise Bud::Error, "illegal use of <+ with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <+ with periodic '#{tabname}' on left"
     end
 
     def tick
@@ -1070,8 +1230,8 @@ module Bud
     public
     def tick #:nodoc: all
       if $BUD_DEBUG
-        puts "#{tabname}. storage -= pending deletes" unless @to_delete.empty? and @to_delete_by_key.empty?
-        puts "#{tabname}. delta += pending" unless @pending.empty?
+        puts "#{tabname}.storage -= pending deletes" unless @to_delete.empty? and @to_delete_by_key.empty?
+        puts "#{tabname}.delta += pending" unless @pending.empty?
       end
       @tick_delta.clear
       deleted = nil
@@ -1099,7 +1259,9 @@ module Bud
     end
 
     def invalidated=(val)
-      raise Bud::Error, "internal error: must not set invalidate on tables"
+      # Might be reset to false at end-of-tick, but shouldn't be set to true
+      raise Bud::Error, "cannot set invalidate on table '#{@tabname}'" if val
+      super
     end
 
     def pending_delete(o)
@@ -1113,6 +1275,12 @@ module Bud
         add_merge_target
         tbl = register_coll_expr(o)
         tbl.pro.wire_to(self, :delete)
+      elsif o.class <= Bud::LatticePushElement
+        add_merge_target
+        o.wire_to(self, :delete)
+      elsif o.class <= Bud::LatticeWrapper
+        add_merge_target
+        o.to_push_elem.wire_to(self, :delete)
       else
         unless o.nil?
           o = o.uniq.compact if o.respond_to?(:uniq)
@@ -1150,7 +1318,7 @@ module Bud
       # No cache to invalidate. Also, tables do not invalidate dependents,
       # because their own state is not considered invalidated; that happens only
       # if there were pending deletes at the beginning of a tick (see tick())
-      puts "******** invalidate_cache called on BudTable" if $BUD_DEBUG
+      puts "******** invalidate_cache called on table '#{@tabname}'" if $BUD_DEBUG
     end
 
     public
@@ -1166,11 +1334,11 @@ module Bud
 
   class BudReadOnly < BudCollection # :nodoc: all
     superator "<+" do |o|
-      raise CompileError, "illegal use of <+ with read-only collection '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <+ with read-only collection '#{@tabname}' on left"
     end
     public
     def merge(o)  #:nodoc: all
-      raise CompileError, "illegal use of <= with read-only collection '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with read-only collection '#{@tabname}' on left"
     end
     public
     def invalidate_cache
@@ -1195,7 +1363,20 @@ module Bud
 
     public
     def each(&block)
-      @expr.call.each(&block)
+      v = @expr.call
+      return if v.nil? or v == [nil]
+
+      # XXX: Gross hack. We want to support RHS expressions that do not
+      # necessarily return BudCollections (they might instead return lattice
+      # values or hashes). Since it isn't easy to distinguish between these two
+      # cases statically, instead we just always use CollExpr; at runtime, if
+      # the value doesn't look like a traditional Bloom collection, we don't try
+      # to break it up into tuples.
+      if v.class <= Array || v.class <= BudCollection
+        v.each(&block)
+      else
+        yield v
+      end
     end
 
     public
@@ -1205,13 +1386,18 @@ module Bud
   end
 
   class BudFileReader < BudReadOnly # :nodoc: all
-    def initialize(name, filename, delimiter, bud_instance) # :nodoc: all
+    def initialize(name, filename, bud_instance) # :nodoc: all
       super(name, bud_instance, {[:lineno] => [:text]})
       @filename = filename
       @storage = {}
       # NEEDS A TRY/RESCUE BLOCK
       @fd = File.open(@filename, "r")
       @linenum = 0
+      @invalidated = true
+    end
+    
+    def tick
+      @invalidated = true
     end
 
     public