bud 0.9.4 → 0.9.5

data/lib/bud/lattice-lib.rb

@@ -0,0 +1,367 @@
+require 'bud/lattice-core'
+
+class Bud::MaxLattice < Bud::Lattice
+  wrapper_name :lmax
+
+  def initialize(i=nil)
+    unless i.nil? || i.class <= Comparable
+      reject_input(i)
+    end
+    @v = i
+  end
+
+  def merge(i)
+    i_val = i.reveal
+    (@v.nil? || (i_val != nil && i_val > @v)) ? i : self
+  end
+
+  morph :gt do |k|
+    Bud::BoolLattice.new(!!(@v && @v > k))
+  end
+
+  morph :gt_eq do |k|
+    Bud::BoolLattice.new(!!(@v && @v >= k))
+  end
+
+  # XXX: support MaxLattice input?
+  morph :+ do |i|
+    # Since bottom of lmax is negative infinity, + is a no-op
+    return self if @v.nil?
+    reject_input(i, "+") unless i.class <= Numeric
+    self.class.new(@v + i)
+  end
+
+  morph :min_of do |i|
+    reject_input(i, "min_of") unless i.class <= Numeric
+    (@v.nil? || i < @v) ? self.class.new(i) : self
+  end
+
+  def lt_eq(k)
+    Bud::BoolLattice.new(!!(@v && @v <= k))
+  end
+end
+
+class Bud::MinLattice < Bud::Lattice
+  wrapper_name :lmin
+
+  def initialize(i=nil)
+    unless i.nil? || i.class <= Comparable
+      reject_input(i)
+    end
+    @v = i
+  end
+
+  def merge(i)
+    i_val = i.reveal
+    (@v.nil? || (i_val != nil && i_val < @v)) ? i : self
+  end
+
+  morph :lt do |k|
+    Bud::BoolLattice.new(!!(@v && @v < k))
+  end
+
+  # XXX: support MinLattice input
+  morph :+ do |i|
+    # Since bottom of lmin is infinity, + is a no-op
+    return self if @v.nil?
+    reject_input(i, "+") unless i.class <= Numeric
+    self.class.new(@v + i)
+  end
+end
+
+# XXX: consider creating two fixed ("interned") values for true and false.
+class Bud::BoolLattice < Bud::Lattice
+  wrapper_name :lbool
+
+  def initialize(i=false)
+    reject_input(i) unless [true, false].include? i
+    @v = i
+  end
+
+  def merge(i)
+    self.class.new(@v || i.reveal)
+  end
+
+  # XXX: ugly syntax
+  morph :when_true do |&blk|
+    blk.call if @v
+  end
+end
+
+class Bud::MapLattice < Bud::Lattice
+  wrapper_name :lmap
+
+  def initialize(i={})
+    reject_input(i) unless i.class == Hash
+    i.each_pair do |k,val|
+      reject_input(i) if k.class <= Bud::Lattice
+      reject_input(i) unless val.class <= Bud::Lattice
+    end
+    @v = i
+  end
+
+  def merge(i)
+    rv = @v.merge(i.reveal) do |k, lhs_v, rhs_v|
+      lhs_v.merge(rhs_v)
+    end
+    wrap_unsafe(rv)
+  end
+
+  def inspect
+    "<#{self.class.wrapper}: #{@v.inspect}>"
+  end
+
+  # XXX: If the key is not in the map, we would like to return some generic
+  # "bottom" value that is shared by all lattice types. Unfortunately, such a
+  # value does not exist, so we need the caller to tell us which class to use as
+  # an optional second argument (if omitted, fetching a non-existent key yields
+  # a runtime exception). Another alternative would be to specify the type of
+  # the map's values when the lmap is declared, but that hinders code reuse.
+  morph :at do |k, *args|
+    if @v.has_key? k
+      @v[k]
+    else
+      raise Bud::Error if args.empty?
+      args.first.new
+    end
+  end
+
+  morph :apply_morph do |sym, *args|
+    raise Bud::Error unless Bud::Lattice.global_morphs.include? sym
+    do_apply(sym, args)
+  end
+
+  monotone :apply_monotone do |sym, *args|
+    raise Bud::Error unless Bud::Lattice.global_mfuncs.include? sym
+    do_apply(sym, args)
+  end
+
+  def do_apply(sym, args)
+    rv = {}
+    @v.each_pair do |k, val|
+      res = val.send(sym, *args)
+      raise Bud::Error unless res.kind_of? Bud::Lattice
+      rv[k] = res
+    end
+    wrap_unsafe(rv)
+  end
+
+  morph :key? do |k|
+    Bud::BoolLattice.new(@v.has_key? k)
+  end
+
+  morph :key_set do
+    Bud::SetLattice.new(@v.keys)
+  end
+
+  monotone :size do
+    Bud::MaxLattice.new(@v.size)
+  end
+
+  morph :intersect do |i|
+    i_tbl = i.reveal
+    # Scan the smaller map, probe the larger one
+    scan, probe = (@v.size < i_tbl.size ? [@v, i_tbl] : [i_tbl, @v])
+    rv = {}
+    scan.each do |k,val|
+      rv[k] = val.merge(probe[k]) if probe.has_key? k
+    end
+    wrap_unsafe(rv)
+  end
+
+  # Produce a Bloom collection (array of tuples) from this lmap, optionally
+  # applying a user-provided code block to each (k,v) pair in turn. Note that
+  # this is slightly different from how projection over an lmap would work: we
+  # return an array, whereas projection would return an lmap.
+  morph :to_collection do |&blk|
+    @v.map(&blk)
+  end
+
+  # Return true if this map is strictly smaller than or equal to the given
+  # map. "x" is strictly smaller than or equal to "y" if:
+  #     (a) every key in "x"  also appears in "y"
+  #     (b) for every key k in "x", x[k] <= y[k]
+  #
+  # NB: For this to be a morphism, we require that (a) "self" is deflationary
+  # (or fixed) (b) the input lattice value is inflationary (or fixed). We
+  # currently don't have a way to express (a) in the type system.
+  def lt_eq(i)
+    reject_input(i, "lt_eq") unless i.class <= self.class
+
+    @v.each do |k, v|
+      unless i.key?(k).reveal == true
+        return Bud::BoolLattice.new(false)
+      end
+      unless v.lt_eq(i.at(k).reveal).reveal == true
+        return Bud::BoolLattice.new(false)
+      end
+    end
+
+    return Bud::BoolLattice.new(true)
+  end
+end
+
+# A set lattice contains zero or more primitive (non-lattice) values.
+class Bud::SetLattice < Bud::Lattice
+  wrapper_name :lset
+
+  def initialize(i=[])
+    reject_input(i) if i.any? {|e| e.kind_of? Bud::Lattice}
+
+    i = Set.new(i) unless i.kind_of? Set
+    @v = i
+  end
+
+  def merge(i)
+    wrap_unsafe(@v | i.reveal)
+  end
+
+  morph :intersect do |i|
+    wrap_unsafe(@v & i.reveal)
+  end
+
+  morph :product do |i, &blk|
+    rv = Set.new
+    @v.each do |a|
+      if blk.nil?
+        t = i.pro {|b| [a,b]}
+      else
+        t = i.pro {|b| blk.call(a, b)}
+      end
+      rv.merge(t.reveal)
+    end
+    wrap_unsafe(rv)
+  end
+
+  morph :contains? do |i|
+    Bud::BoolLattice.new(@v.member? i)
+  end
+
+  morph :pro do |&blk|
+    # We don't use Set#map, since it returns an Array (ugh).
+    rv = Set.new
+    @v.each do |t|
+      val = blk.call(t)
+      rv << val unless val.nil?
+    end
+    wrap_unsafe(rv)
+  end
+
+  monotone :size do
+    Bud::MaxLattice.new(@v.size)
+  end
+
+  # Assuming that this set contains tuples (arrays) as elements, this performs
+  # an equijoin between the current lattice and i. The join predicate is
+  # "self_t[lhs_idx] == i_t[rhs_idx]", for all tuples self_t and i_t in self and
+  # i, respectively. The return value is the result of passing pairs of join
+  # tuples to the user-supplied block.
+  morph :eqjoin do |i, lhs_idx, rhs_idx, &blk|
+    rv = Set.new
+    @v.each do |a|
+      i.probe(rhs_idx, a[lhs_idx]).each do |b|
+        rv << blk.call(a, b)
+      end
+    end
+    wrap_unsafe(rv)
+  end
+
+  # Assuming that this set contains tuples (arrays), this returns a list of
+  # tuples (possibly empty) whose idx'th column has the value "v".
+  # XXX: we assume probe(idx, v) will only be called for a single value of idx!
+  def probe(idx, v)
+    @ht ||= build_ht(idx)
+    return @ht[v] || []
+  end
+
+  private
+  def build_ht(idx)
+    rv = {}
+    @v.each do |i|
+      field = i[idx]
+      rv[field] ||= []
+      rv[field] << i
+    end
+    rv
+  end
+end
+
+# A set that admits only non-negative numbers. This allows "sum" to be an
+# order-preserving map.  Note that this does duplicate elimination on its input,
+# so it actually computes "SUM(DISTINCT ...)" in SQL.
+#
+# XXX: for methods that take a user-provided code block, we need to ensure that
+# the set continues to contain only positive numbers.
+class Bud::PositiveSetLattice < Bud::SetLattice
+  wrapper_name :lpset
+
+  def initialize(i=[])
+    super
+    @v.each do |n|
+      reject_input(i) unless n.class <= Numeric
+      reject_input(i) if n < 0
+    end
+  end
+
+  monotone :pos_sum do
+    @sum = @v.reduce(:+) if @sum.nil?
+    Bud::MaxLattice.new(@sum)
+  end
+end
+
+# XXX: Should this be just syntax sugar for a map lattice instead?
+class Bud::BagLattice < Bud::Lattice
+  wrapper_name :lbag
+
+  def initialize(i={})
+    reject_input(i) unless i.class <= Hash
+    i.each do |k, mult|
+      reject_input(i) if k.class <= Bud::Lattice
+      reject_input(i) unless (mult.class <= Integer && mult > 0)
+    end
+    @v = i
+  end
+
+  # Note that for merge to be idempotent, we need to use the traditional
+  # definition of multiset union (per-element max of multiplicities, rather than
+  # sum of multiplicities).
+  def merge(i)
+    rv = @v.merge(i.reveal) do |k, lhs_v, rhs_v|
+      [lhs_v, rhs_v].max
+    end
+    wrap_unsafe(rv)
+  end
+
+  morph :intersect do |i|
+    i_tbl = i.reveal
+    # Scan the smaller one, probe the larger one
+    scan, probe = (@v.size < i_tbl.size ? [@v, i_tbl] : [i_tbl, @v])
+    rv = {}
+    scan.each do |k,val|
+      rv[k] = [val, probe[k]].min if probe.has_key? k
+    end
+    wrap_unsafe(rv)
+  end
+
+  morph :multiplicity do |k|
+    rv = @v[k]
+    rv ||= 0
+    Bud::MaxLattice.new(rv)
+  end
+
+  morph :+ do |i|
+    rv = @v.merge(i.reveal) do |k, lhs_v, rhs_v|
+      lhs_v + rhs_v
+    end
+    self.class.new(rv)
+  end
+
+  morph :contains? do |i|
+    Bud::BoolLattice.new(@v.has_key? i)
+  end
+
+  monotone :size do
+    @size = @v.values.reduce(:+) if @size.nil?
+    Bud::MaxLattice.new(@size)
+  end
+end

data/lib/bud/monkeypatch.rb

@@ -11,6 +11,7 @@ class Class
 end
 
 # FIXME: Use a subclass of Struct.
+# FIXME: Should likely override eql? as well
 class Struct
   def <=>(o)
     if o.class == self.class
@@ -139,7 +140,7 @@ class Module
   end
 
   # bloom statements to be registered with Bud runtime.  optional +block_name+ 
-  # allows for multiple bloom blocks per module, and overriding
+  # allows for multiple bloom blocks per module and method overriding
   def bloom(block_name=nil, &block)
     # If no block name was specified, generate a unique name
     if block_name.nil?
@@ -148,7 +149,7 @@ class Module
       @block_id += 1
     else
       unless block_name.class <= Symbol
-        raise Bud::CompileError, "bloom block names must be a symbol: #{block_name}"
+        raise Bud::CompileError, "block name must be a symbol: #{block_name}"
       end
     end
 
@@ -161,15 +162,24 @@ class Module
     # module; this indicates a likely programmer error.
     if instance_methods(false).include?(meth_name) ||
        instance_methods(false).include?(meth_name.to_sym)
-      raise Bud::CompileError, "duplicate named bloom block: '#{block_name}' in #{self}"
+      raise Bud::CompileError, "duplicate block name: '#{block_name}' in #{self}"
     end
     ast = Source.read_block(caller[0]) # pass in caller's location via backtrace
+
     # ast corresponds only to the statements of the block. Wrap it in a method
     # definition for backward compatibility for now.
-    # First wrap ast in a block if it is only a single statement
-    ast = s(:block) if ast.nil?
-    ast = s(:block, ast) unless ast.sexp_type == :block
-    ast = s(:defn, meth_name.to_sym, s(:args), s(:scope, ast))
+
+    # If the block contained multiple statements, the AST will have a top-level
+    # :block node. Since ruby_parser ASTs for method definitions don't contain
+    # such a node, remove it.
+    if ast.nil?
+      ast = []
+    elsif ast.sexp_type == :block
+      ast = ast.sexp_body
+    else
+      ast = [ast]
+    end
+    ast = s(:defn, meth_name.to_sym, s(:args), *ast)
     unless self.respond_to? :__bloom_asts__
       def self.__bloom_asts__
         @__bloom_asts__ ||= {}
@@ -180,11 +190,11 @@ class Module
     define_method(meth_name.to_sym, &block)
   end
 
-  private
   # Return a string with a version of the class name appropriate for embedding
   # into a method name. Annoyingly, if you define class X nested inside
   # class/module Y, X's class name is the string "Y::X". We don't want to define
   # method names with semicolons in them, so just return "X" instead.
+  private
   def self.get_class_name(klass)
     (klass.name.nil? or klass.name == "") \
       ? "Anon#{klass.object_id}" \

data/lib/bud/rewrite.rb

@@ -1,6 +1,4 @@
 require 'rubygems'
-require 'ruby2ruby'
-require 'set'
 
 class RuleRewriter < Ruby2Ruby # :nodoc: all
   attr_accessor :rule_indx, :rules, :depends
@@ -8,10 +6,11 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   OP_LIST = Set.new([:<<, :<, :<=])
   TEMP_OP_LIST = Set.new([:-@, :~, :+@])
   MONOTONE_WHITELIST = Set.new([:==, :+, :<=, :-, :<, :>, :*, :~,
-                                :pairs, :matches, :combos, :flatten,
-                                :lefts, :rights, :map, :flat_map, :pro,
+                                :pairs, :matches, :combos, :flatten, :new,
+                                :lefts, :rights, :map, :flat_map, :pro, :merge,
                                 :cols, :key_cols, :val_cols, :payloads, :lambda,
-                                :tabname, :ip_port, :port, :ip, :int_ip_port])
+                                :tabname, :ip_port, :port, :ip, :int_ip_port,
+                                :current_value])
 
   def initialize(seed, bud_instance)
     @bud_instance = bud_instance
@@ -21,7 +20,8 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
     @collect = false
     @rules = []
     @depends = []
-    @nm_funcs_called = false
+    @iter_stack = []
+    @refs_in_body = Set.new
     super()
   end
 
@@ -29,6 +29,7 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   def resolve(obj, prefix, name)
     qn = prefix ? prefix + "." + name.to_s : name.to_s
     return [:collection, qn, obj.tables[name]] if obj.tables.has_key? name
+    return [:lattice, qn, obj.lattices[name]] if obj.lattices.has_key? name
 
     # does name refer to an import name?
     iobj = obj.import_instance name
@@ -38,12 +39,13 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   end
 
   def exp_id_type(recv, name, args) # call only if sexp type is :call
-    return $not_id unless args.size == 1
+    return $not_id unless args.empty?
     ty = $not_id
     if recv
       if recv.first == :call
-        # possibly nested reference.
-        rty, rqn, robj = exp_id_type(recv[1], recv[2], recv[3]) # rty, rqn, .. = receiver's type, qual name etc.
+        # possibly nested reference
+        # rty, rqn, .. = receiver's type, qual name etc.
+        rty, rqn, robj = exp_id_type(recv[1], recv[2], recv[3..-1])
         ty = resolve(robj, rqn, name) if rty == :import
       end
     else
@@ -56,26 +58,88 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   def call_to_id(exp)
     # convert a series of nested calls, a sexp of the form
     #   s(:call,
-    #       s(:call, s(:call, nil, :a, s(:arglist)), :b, s(:arglist)),
-    #         :bar ,
-    #         s(:arglist)))
+    #       s(:call, s(:call, nil, :a), :b),
+    #         :bar))
     # to the string "a.b.bar"
-    raise "Malformed exp: #{exp}" unless (exp[0] == :call)
-    _, recv, op, args = exp
+    raise Bud::CompileError, "malformed exp: #{exp}" unless exp.sexp_type == :call
+    _, recv, op = exp
     return recv.nil? ? op.to_s : call_to_id(recv) + "." + op.to_s
   end
 
+  # We want to distinguish between collection dependencies that occur in
+  # top-level expressions versus collections that are referenced inside rule
+  # bodies. We just want to set a flag when processing the :iter body, but
+  # annoyingly it seems that is hard to do without duplicating the
+  # implementation of process_iter().
+  #
+  # XXX: the whole RuleRewriter approach is wrong because it conflates
+  # converting ASTs to strings with doing analysis on ASTs. Those should be
+  # split into two separate passes.
+  def process_iter(exp)
+    iter = process exp.shift
+    args = exp.shift
+
+    @iter_stack.push(true)
+    body = exp.empty? ? nil : process(exp.shift)
+    @iter_stack.pop
+
+    do_process_iter(iter, args, body)
+  end
+
+  def do_process_iter(iter, args, body)
+    args = case args
+           when 0 then
+             " ||"
+           else
+             a = process(args)[1..-2]
+             a = " |#{a}|" unless a.empty?
+             a
+           end
+
+    b, e = if iter == "END" then
+             [ "{", "}" ]
+           else
+             [ "do", "end" ]
+           end
+
+    iter.sub!(/\(\)$/, '')
+
+    # REFACTOR: ugh
+    result = []
+    result << "#{iter} {"
+    result << args
+    if body then
+      result << " #{body.strip} "
+    else
+      result << ' '
+    end
+    result << "}"
+    result = result.join
+    return result if result !~ /\n/ and result.size < LINE_LENGTH
+
+    result = []
+    result << "#{iter} #{b}"
+    result << args
+    result << "\n"
+    if body then
+      result << indent(body.strip)
+      result << "\n"
+    end
+    result << e
+    result.join
+  end
+
   def process_call(exp)
-    recv, op, args = exp
-    if OP_LIST.include?(op) and @context[1] == :block and @context.length == 4
-      # NB: context.length is 4 when see a method call at the top-level of a
+    recv, op, *args = exp
+    if OP_LIST.include?(op) and @context[1] == :defn and @context.length == 2
+      # NB: context.length is 2 when see a method call at the top-level of a
       # :defn block -- this is where we expect Bloom statements to appear
       do_rule(exp)
     elsif op == :notin
       # Special case. In the rule "z <= x.notin(y)", z depends positively on x,
       # but negatively on y. See further explanation in the "else" section for
       # why this is a special case.
-      notintab = call_to_id(args[1])   # args expected to be of the form (:arglist (:call nil :y ...))
+      notintab = call_to_id(args[0])   # args expected to be of the form (:call nil :y ...)
       @tables[notintab.to_s] = true    # "true" denotes non-monotonic dependency
       super
     else
@@ -88,15 +152,16 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
       # a.b.c.notin(d.e.f), we register a non-monotonic dependency of lhs on
       # "d.e.f", not with "a.b.c"
       ty, qn, _ = exp_id_type(recv, op, args) # qn = qualified name
-      if ty == :collection
+      if ty == :collection or ty == :lattice
         (@tables[qn] = @nm if @collect) unless @tables[qn]
+        @refs_in_body << qn unless @iter_stack.empty?
       #elsif ty == :import .. do nothing
       elsif ty == :not_coll_id
         # Check if receiver is a collection, and further if the current exp
         # represents a field lookup
         op_is_field_name = false
         if recv and recv.first == :call
-          rty, _, robj = exp_id_type(recv[1], recv[2], recv[3])
+          rty, _, robj = exp_id_type(recv[1], recv[2], recv[3..-1])
           if rty == :collection
             cols = robj.cols
             op_is_field_name = true if cols and cols.include?(op)
@@ -104,18 +169,14 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
         end
         # For CALM analysis, mark deletion rules as non-monotonic
         @nm = true if op == :-@
+
+        # Don't worry about monotone ops, table names, table.attr calls, or
+        # accessors of iterator variables
         if recv
-          # Don't worry about monotone ops, table names, table.attr calls, or
-          # accessors of iterator variables
           unless RuleRewriter.is_monotone(op) or op_is_field_name or
                  recv.first == :lvar or op.to_s.start_with?("__")
             @nm = true
           end
-        else
-          # Function called (implicit receiver = Bud instance) in a user-defined
-          # code block. Check if it is non-monotonic (like budtime, that
-          # produces a new value every time it is called)
-          @nm_funcs_called = true unless RuleRewriter.is_monotone(op)
         end
       end
       if TEMP_OP_LIST.include? op
@@ -126,25 +187,35 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   end
 
   def self.is_monotone(op)
-    MONOTONE_WHITELIST.include?(op)
+    MONOTONE_WHITELIST.include?(op) ||
+      is_morphism(op) ||
+      Bud::Lattice.global_mfuncs.include?(op)
   end
 
-  # Rewrite top-level rhs array literals to lambdas. During wiring, these are
-  # turned into coll_expr collections.
+  def self.is_morphism(op)
+    Bud::Lattice.global_morphs.include?(op)
+  end
+
+  # Rewrite top-level rhs array and hash literals to lambdas. During wiring,
+  # these are turned into coll_expr collections.
   def lambda_rewrite(rhs)
     # the <= case
-    if rhs[0] == :array
-      return s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, rhs)
+    if is_coll_literal(rhs[0])
+      return s(:iter, s(:call, nil, :lambda), s(:args), rhs)
     # the superator case
     elsif rhs[0] == :call \
-      and rhs[1] and rhs[1][0] and rhs[1][0] == :array \
+      and rhs[1] and rhs[1][0] and is_coll_literal(rhs[1][0]) \
       and rhs[2] and (rhs[2] == :+@ or rhs[2] == :-@ or rhs[2] == :~@)
-      return s(rhs[0], s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, rhs[1]), rhs[2], rhs[3])
+      return s(rhs[0], s(:iter, s(:call, nil, :lambda), s(:args), rhs[1]), rhs[2], *rhs[3..-1])
     else
       return rhs
     end
   end
 
+  def is_coll_literal(e)
+    [:array, :hash].include? e
+  end
+
   def collect_rhs(exp)
     exp = lambda_rewrite(exp)
 
@@ -155,13 +226,13 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   end
 
   def reset_instance_vars
+    @refs_in_body = Set.new
     @tables = {}
     @nm = false
-    @nm_funcs_called = false
     @temp_op = nil
   end
 
-  def record_rule(lhs, op, rhs_pos, rhs)
+  def record_rule(lhs, op, rhs_pos, rhs, unsafe_funcs_called)
     rule_txt_orig = "#{lhs} #{op} (#{rhs})"
     rule_txt = "#{lhs} #{op} (#{rhs_pos})"
     if op == :<
@@ -170,9 +241,11 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
       op = op.to_s
     end
 
-    @rules << [@bud_instance, @rule_indx, lhs, op, rule_txt, rule_txt_orig, @nm_funcs_called]
+    @rules << [@bud_instance, @rule_indx, lhs, op, rule_txt,
+               rule_txt_orig, unsafe_funcs_called]
     @tables.each_pair do |t, nm|
-      @depends << [@bud_instance, @rule_indx, lhs, op, t, nm]
+      in_rule_body = @refs_in_body.include? t
+      @depends << [@bud_instance, @rule_indx, lhs, op, t, nm, in_rule_body]
     end
 
     reset_instance_vars
@@ -180,17 +253,14 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   end
 
   def do_rule(exp)
-    lhs = process exp[0]
-    op = exp[1]
-    rhs_ast = map2pro(exp[2])
-
-    # Remove the outer s(:arglist) from the rhs AST. An AST subtree rooted with
-    # s(:arglist) is not really sensible and it causes Ruby2Ruby < 1.3.1 to
-    # misbehave (for example, s(:arglist, s(:hash, ...)) is misparsed.
-    raise Bud::CompileError unless rhs_ast.sexp_type == :arglist
-    rhs_ast = rhs_ast[1]
+    lhs, op, rhs_ast = exp
+    lhs = process(lhs)
 
+    rhs_ast = MapRewriter.new.process(rhs_ast)
     rhs_ast = RenameRewriter.new(@bud_instance).process(rhs_ast)
+    rhs_ast = LatticeRefRewriter.new(@bud_instance).process(rhs_ast)
+    ufr = UnsafeFuncRewriter.new
+    rhs_ast = ufr.process(rhs_ast)
 
     if @bud_instance.options[:no_attr_rewrite]
       rhs = collect_rhs(rhs_ast)
@@ -203,35 +273,37 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
       reset_instance_vars
       rhs_pos = collect_rhs(AttrNameRewriter.new(@bud_instance).process(rhs_ast_dup))
     end
-    record_rule(lhs, op, rhs_pos, rhs)
+    record_rule(lhs, op, rhs_pos, rhs, ufr.unsafe_func_called)
     drain(exp)
   end
 
-  # We want to rewrite "map" calls on BudCollections to "pro" calls. It is hard
-  # to do this precisely (issue #225), so we just replace map calls liberally
-  # and define Enumerable#pro as an alias for "map".
-  def map2pro(exp)
-    # the non-superator case
-    if exp[1] and exp[1][0] and exp[1][0] == :iter \
-      and exp[1][1] and exp[1][1][1] and exp[1][1][1][0] == :call
-      if exp[1][1][2] == :map
-        exp[1][1][2] = :pro
-      end
-    # the superator case
-    elsif exp[1] and exp[1][0] == :call and (exp[1][2] == :~@ or exp[1][2] == :+@ or exp[1][2] == :-@)
-      if exp[1][1] and exp[1][1][1] and exp[1][1][1][2] == :map
-        exp[1][1][1][2] = :pro
-      end
-    end
-    exp
-  end
-
   def drain(exp)
     exp.shift until exp.empty?
     return ""
   end
 end
 
+# We want to rewrite "map" calls on BudCollections to "pro" calls. It is hard
+# to do this precisely (issue #225), so we just replace map calls liberally
+# and define Enumerable#pro as an alias for "map".
+class MapRewriter < SexpProcessor
+  def initialize
+    super
+    self.require_empty = false
+    self.expected = Sexp
+  end
+
+  def process_call(exp)
+    tag, recv, op, *args = exp
+
+    if op == :map and args.empty?
+      op = :pro
+    end
+
+    s(tag, process(recv), op, *(args.map{|a| process(a)}))
+  end
+end
+
 # Look for rename statements and define the necessary scratch collections
 class RenameRewriter < SexpProcessor
   def initialize(bud_instance)
@@ -252,14 +324,110 @@ class RenameRewriter < SexpProcessor
   end
 
   def process_call(exp)
-    call, recv, op, args = exp
+    tag, recv, op, *args = exp
 
     if op == :rename
-      arglist, namelit, schemahash = args
+      raise Bud::CompileError, "reduce takes two arguments" unless args.size == 2
+      namelit, schemahash = args
       register_scratch(namelit[1], schemahash)
     end
 
-    return s(call, process(recv), op, process(args))
+    return s(tag, process(recv), op, *(args.map{|a| process(a)}))
+  end
+end
+
+# Check for whether the rule invokes any "unsafe" functions (funcs that might
+# return a different value every time they are called, e.g., budtime). Note that
+# although we call this a rewriter, it doesn't modify the input AST.
+class UnsafeFuncRewriter < SexpProcessor
+  attr_reader :unsafe_func_called
+
+  def initialize
+    super()
+    self.require_empty = false
+    self.expected = Sexp
+    @unsafe_func_called = false
+    @elem_stack = []
+  end
+
+  def process_call(exp)
+    tag, recv, op, *args = exp
+
+    # We assume that unsafe funcs have a nil receiver (Bud instance is implicit
+    # receiver).
+    if recv.nil? and @elem_stack.size > 0
+      @unsafe_func_called = true unless RuleRewriter.is_monotone(op)
+    end
+
+    return s(tag, process(recv), op, *(args.map{|a| process(a)}))
+  end
+
+  def process_iter(exp)
+    tag, recv, iter_args, body = exp
+    new_body = push_and_process(body)
+    return s(tag, process(recv), process(iter_args), new_body)
+  end
+
+  def push_and_process(exp)
+    obj_id = exp.object_id
+    @elem_stack.push(obj_id)
+    rv = process(exp)
+    raise Bud::Error unless @elem_stack.pop == obj_id
+    return rv
+  end
+end
+
+# Rewrite references to lattice identifiers that appear in rule bodies. A
+# reference to a lattice identifier returns the associated lattice wrapper. When
+# the identifier appears at the top-level of the rule RHS, that is fine (since
+# we want the wrapper to do wiring). But for references that appear inside rule
+# bodies, we want to instead fetch the current value associated with the lattice
+# wrapper.
+class LatticeRefRewriter < SexpProcessor
+  def initialize(bud_instance)
+    super()
+    self.require_empty = false
+    self.expected = Sexp
+    @bud_instance = bud_instance
+    @elem_stack = []
+  end
+
+  def process_iter(exp)
+    tag, recv, iter_args, body = exp
+    new_body = push_and_process(body)
+    return s(tag, process(recv), process(iter_args), new_body)
+  end
+
+  def process_array(exp)
+    new_body = exp.sexp_body.map {|t| push_and_process(t)}
+    return s(:array, *new_body)
+  end
+
+  def process_hash(exp)
+    new_body = exp.sexp_body.map {|t| push_and_process(t)}
+    return s(:hash, *new_body)
+  end
+
+  def process_call(exp)
+    tag, recv, op, *args = exp
+
+    if recv.nil? and args.empty? and is_lattice?(op) and @elem_stack.size > 0
+      return s(:call, exp, :current_value)
+    else
+      return s(tag, process(recv), op, *(args.map{|a| process(a)}))
+    end
+  end
+
+  def push_and_process(exp)
+    obj_id = exp.object_id
+    @elem_stack.push(obj_id)
+    rv = process(exp)
+    raise Bud::Error unless @elem_stack.pop == obj_id
+    return rv
+  end
+
+  def is_lattice?(op)
+    @bud_instance.lattices.has_key? op.to_sym
   end
 end
 
@@ -278,32 +446,43 @@ class AttrNameRewriter < SexpProcessor # :nodoc: all
   # iter vars
   def process_iter(exp)
     if exp[1] and exp[1][0] == :call
+      return exp unless exp[2]
       gather_collection_names(exp[1])
+      meth_name = exp[1][2]
 
       # now find iter vars and match up
-      if exp[2] and exp[2][0] == :lasgn and @collnames.size == 1 #single-table iter
-        raise Bud::CompileError, "nested redefinition of block variable \"#{exp[2][1]}\" not allowed" if @iterhash[exp[2][1]]
-        @iterhash[exp[2][1]] = @collnames[0]
-      elsif exp[2] and exp[2][0] == :lasgn and @collnames.size > 1 and exp[1] # join iter with lefts/rights
-        case exp[1][2]
+      if exp[2][0] == :args and @collnames.size == 1 # single-table iter
+        if @iterhash[exp[2][1]]
+          raise Bud::CompileError, "redefinition of block variable \"#{exp[2][1]}\" not allowed"
+        end
+
+        # XXX: The BudChannel#payloads method assigns the correct schema to
+        # tuples that pass through it (i.e., it omits the location specifier);
+        # hence we don't want to apply the location rewrite to the code block
+        # that is passed to payloads(). This is a dirty hack.
+        unless meth_name == :payloads
+          @iterhash[exp[2][1]] = @collnames[0]
+        end
+      elsif exp[2][0] == :args and not @collnames.empty? # join iter with lefts/rights
+        case meth_name
         when :lefts
           @iterhash[exp[2][1]] = @collnames[0]
         when :rights
           @iterhash[exp[2][1]] = @collnames[1]
-        else
-          raise Bud::CompileError, "nested redefinition of block variable \"#{exp[2][1]}\" not allowed" if @iterhash[exp[2][1]]
-        end
-      elsif exp[2] and exp[2][0] == :masgn and not @collnames.empty? # join or reduce iter
-        return unless exp[2][1] and exp[2][1][0] == :array
-        if exp[1][2] == :reduce
+        when :reduce
           unless @collnames.length == 1
-            raise Bud::Error, "reduce should only have one associated collection, but has #{@collnames.inspect}"
+            raise Bud::CompileError, "reduce should only have one associated collection, but has #{@collnames.inspect}"
           end
-          @iterhash[exp[2][1][2][1]] = @collnames.first
-        else #join
-          @collnames.each_with_index do |c, i|
-            next unless exp[2][1][i+1] and exp[2][1][i+1][0] == :lasgn
-            @iterhash[exp[2][1][i+1][1]] = c
+          @iterhash[exp[2][1]] = @collnames[0]
+        else
+          # join
+          if @iterhash[exp[2][1]]
+            raise Bud::CompileError, "redefinition of block variable \"#{exp[2][1]}\" not allowed"
+          end
+
+          @collnames.each_with_index do |c,i|
+            next unless exp[2][i+1]
+            @iterhash[exp[2][i+1]] = c
           end
         end
       end
@@ -313,36 +492,43 @@ class AttrNameRewriter < SexpProcessor # :nodoc: all
   end
 
   def gather_collection_names(exp)
-    if exp[0] == :call and exp[1].nil?
+    # We expect a reference to a collection name to look like a function call
+    # (nil receiver) with no arguments.
+    if exp.sexp_type == :call and exp[1].nil? and exp.length == 3
       @collnames << exp[2]
-    elsif exp[2] and exp[2] == :rename
-      arglist, namelit, schemahash = exp[3]
+    elsif exp.sexp_type == :call and exp[2] == :rename
+      namelit = exp[3]
       @collnames << namelit[1]
+    elsif exp.sexp_type == :call and [:group, :argagg].include?(exp[2])
+      # For grouping and argagg expressions, only look at the receiver (the
+      # collection we're grouping on); otherwise, we might mistakenly think some
+      # of the arguments to the grouping operation are collection names.
+      gather_collection_names(exp[1])
     else
-      exp.each { |e| gather_collection_names(e) if e and e.class <= Sexp }
+      exp.each { |e| gather_collection_names(e) if e.class <= Sexp }
     end
   end
 
   def process_call(exp)
-    call, recv, op, args = exp
+    call, recv, op, *args = exp
 
-    if recv and recv.class == Sexp and recv.first == :lvar and recv[1] and @iterhash[recv[1]]
+    if recv.class == Sexp and recv.sexp_type == :lvar and @iterhash[recv[1]]
       if @bud_instance.respond_to?(@iterhash[recv[1]])
         if @bud_instance.send(@iterhash[recv[1]]).class <= Bud::BudCollection
           cols = @bud_instance.send(@iterhash[recv[1]]).cols
           if op != :[] and @bud_instance.send(@iterhash[recv[1]]).respond_to?(op)
-            # if the op is an attribute name in the schema, col is its index
-            col = cols.index(op) unless cols.nil?
-            unless col.nil?
+            # if the op is an attribute name in the schema, col_idx is its index
+            col_idx = cols.index(op) unless cols.nil?
+            unless col_idx.nil?
               op = :[]
-              args = s(:arglist, s(:lit, col))
+              args = [s(:lit, col_idx)]
             end
           end
         end
-        return s(call, recv, op, args)
+        return s(call, recv, op, *args)
       end
     end
-    return s(call, process(recv), op, process(args))
+    return s(call, process(recv), op, *(args.map{|a| process(a)}))
   end
 end
 
@@ -365,52 +551,41 @@ class TempExpander < SexpProcessor # :nodoc: all
   end
 
   def process_defn(exp)
-    tag, name, args, scope = exp
-    if name.to_s =~ /^__bloom__.+/
-      block = scope[1]
-
-      block.each_with_index do |n,i|
-        if i == 0
-          raise Bud::CompileError if n != :block
-          next
-        end
-
-        # temp declarations are misparsed if the RHS contains certain constructs
-        # (e.g., group, "do |f| ... end" rather than "{|f| ... }").  Rewrite to
-        # correct the misparsing.
-        if n.sexp_type == :iter
-          iter_body = n.sexp_body
-          new_n = fix_temp_decl(iter_body)
-          unless new_n.nil?
-            block[i] = n = new_n
-            @did_work = true
-          end
-        end
-
-        _, recv, meth, meth_args = n
-        if meth == KEYWORD and recv.nil?
-          block[i] = rewrite_me(n)
+    tag, name, args, *body = exp
+    return exp unless name.to_s =~ /^__bloom__.+/
+
+    body.each_with_index do |n,i|
+      # temp declarations are misparsed if the RHS contains certain constructs
+      # (e.g., group, "do |f| ... end" rather than "{|f| ... }").  Rewrite to
+      # correct the misparsing.
+      if n.sexp_type == :iter
+        iter_body = n.sexp_body
+        new_n = fix_temp_decl(iter_body)
+        unless new_n.nil?
+          body[i] = n = new_n
           @did_work = true
         end
       end
+
+      _, recv, meth, meth_args = n
+      if meth == KEYWORD and recv.nil?
+        body[i] = rewrite_me(n)
+        @did_work = true
+      end
     end
-    s(tag, name, args, scope)
+    s(tag, name, args, *body)
   end
 
   private
   def fix_temp_decl(iter_body)
     if iter_body.first.sexp_type == :call
       call_node = iter_body.first
+      _, recv, meth, *meth_args = call_node
 
-      _, recv, meth, meth_args = call_node
       if meth == KEYWORD and recv.nil?
-        _, lhs, op, rhs = meth_args.sexp_body.first
-
-        old_rhs_body = rhs.sexp_body
-        new_rhs_body = [:iter]
-        new_rhs_body += old_rhs_body
-        new_rhs_body += iter_body[1..-1]
-        rhs[1] = Sexp.from_array(new_rhs_body)
+        _, lhs, op, rhs = meth_args.first
+        new_rhs = s(:iter, rhs, *(iter_body[1..-1]))
+        meth_args.first[3] = new_rhs
         return call_node
       end
     end
@@ -418,18 +593,18 @@ class TempExpander < SexpProcessor # :nodoc: all
   end
 
   def rewrite_me(exp)
-    _, recv, meth, args = exp
+    _, recv, meth, *args = exp
 
-    raise Bud::CompileError unless recv == nil
-    nest_call = args.sexp_body.first
+    raise Bud::CompileError unless recv.nil?
+    nest_call = args.first
     raise Bud::CompileError unless nest_call.sexp_type == :call
 
-    nest_recv, nest_op, nest_args = nest_call.sexp_body
+    nest_recv, nest_op, *nest_args = nest_call.sexp_body
     raise Bud::CompileError unless nest_recv.sexp_type == :lit
 
     tmp_name = nest_recv.sexp_body.first
     @tmp_tables << tmp_name
-    new_recv = s(:call, nil, tmp_name, s(:arglist))
-    return s(:call, new_recv, nest_op, nest_args)
+    new_recv = s(:call, nil, tmp_name)
+    return s(:call, new_recv, nest_op, *nest_args)
   end
 end