bud 0.9.0 → 0.9.1

data/lib/bud/executor/elements.rb

@@ -13,9 +13,9 @@ module Bud
   # p.insert(1)
   # p.insert(nil)
   class PushElement < BudCollection
-    attr_accessor :elem_name
     attr_accessor :rescan, :invalidated
-    attr_reader :inputs, :found_delta, :refcount, :wired_by, :outputs
+    attr_accessor :elem_name
+    attr_reader :found_delta, :refcount, :wired_by, :outputs
 
     def initialize(name_in, bud_instance, collection_name=nil, given_schema=nil, defer_schema=false, &blk)
       super(name_in, bud_instance, given_schema, defer_schema)
@@ -33,10 +33,6 @@ module Bud
       @rescan = true
     end
 
-    def wiring?
-      @bud_instance.toplevel.done_wiring == false
-    end
-
     def wirings
       @wirings ||= @outputs + @pendings + @deletes + @delete_keys
     end
@@ -81,40 +77,27 @@ module Bud
     def set_block(&blk)
       @blk = blk
     end
-    def wire_to(element)
-      unless element.respond_to? :insert
-        raise Bud::Error, "attempt to wire_to element without insert method"
-      end
-      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
-      # puts "wiring #{self.elem_name} to #{elem_name}"
-      @outputs << element
-      element.wired_by << self if element.respond_to? :wired_by
-    end
-    def wire_to_pending(element)
-      unless element.respond_to? :pending_merge
-        raise Bud::Error, "attempt to wire_to_pending element without pending_merge method"
-      end
-      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
-      # puts "wiring #{self.elem_name} to #{elem_name}(pending)"
-      @pendings << element
-      element.wired_by << self if element.respond_to? :wired_by
-    end
-    def wire_to_delete(element)
-      unless element.respond_to? :pending_delete
-        raise Bud::Error, "attempt to wire_to_delete element without pending_delete method"
+
+    def wire_to(element, kind=:output)
+      unless @bud_instance.wiring?
+        raise Bud::Error, "wire_to called outside wiring phase"
       end
-      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
-      # puts "wiring #{self.elem_name} to #{elem_name}(delete)"
-      @deletes << element
-      element.wired_by << self if element.respond_to? :wired_by
-    end
-    def wire_to_delete_by_key(element)
-      unless element.respond_to? :pending_delete_keys
-        raise Bud::Error, "attempt to wire_to_delete_by_key element without pending_delete_keys method"
+
+      case kind
+      when :output
+        raise Bud::Error unless element.respond_to? :insert
+        @outputs << element
+      when :pending
+        raise Bud::Error unless element.respond_to? :pending_merge
+        @pendings << element
+      when :delete
+        raise Bud::Error unless element.respond_to? :pending_delete
+        @deletes << element
+      when :delete_by_key
+        raise Bud::Error unless element.respond_to? :pending_delete_keys
+        @delete_keys << element
       end
-      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
-      # puts "wiring #{self.elem_name} to #{elem_name}(delete)"
-      @delete_keys << element
+
       element.wired_by << self if element.respond_to? :wired_by
     end
 
@@ -144,12 +127,8 @@ module Bud
         unless item.nil?
           @outputs.each do |ou|
             if ou.class <= Bud::PushElement
-              #the_name = ou.elem_name
-              # puts "#{self.object_id%10000} (#{elem_name}) -> #{ou.object_id%10000} (#{the_name}): #{item.inspect}"
               ou.insert(item, self)
             elsif ou.class <= Bud::BudCollection
-              # the_name = ou.tabname
-              # puts "#{self.object_id%10000} (#{elem_name}) -> #{ou.object_id%10000} (#{the_name}): #{item.inspect}"
               ou.do_insert(item, ou.new_delta)
             else
               raise Bud::Error, "expected either a PushElement or a BudCollection"
@@ -210,19 +189,14 @@ module Bud
     public
     def stratum_end
     end
-    #public
-    #def set_schema(schema)
-    #  @schema=schema
-    #  setup_accessors
-    #end
-
 
     ####
     # and now, the Bloom-facing methods
+    # XXX: "the_name" parameter is unused
     public
-    def pro(the_name=@elem_name, the_schema=schema, &blk)
+    def pro(the_name=elem_name, the_schema=schema, &blk)
       toplevel = @bud_instance.toplevel
-      elem = Bud::PushElement.new('project' + object_id.to_s,
+      elem = Bud::PushElement.new("project#{object_id}",
                                   toplevel.this_rule_context,
                                   @collection_name, the_schema)
       self.wire_to(elem)
@@ -233,10 +207,11 @@ module Bud
 
     alias each pro
 
+    # XXX: "the_name" & "the_schema" parameters are unused
     public
     def each_with_index(the_name=elem_name, the_schema=schema, &blk)
       toplevel = @bud_instance.toplevel
-      elem = Bud::PushEachWithIndex.new('each_with_index' + object_id.to_s,
+      elem = Bud::PushEachWithIndex.new("each_with_index#{object_id}",
                                         toplevel.this_rule_context,
                                         @collection_name)
       elem.set_block(&blk)
@@ -273,7 +248,7 @@ module Bud
     end
 
     def merge(source)
-      if source.class <= PushElement and wiring?
+      if source.class <= PushElement and @bud_instance.wiring?
         source.wire_to(self)
       else
         source.each {|i| self << i}

data/lib/bud/executor/group.rb

@@ -46,7 +46,7 @@ module Bud
 
   class PushArgAgg < PushGroup
     def initialize(elem_name, bud_instance, collection_name, keys_in, aggpairs_in, schema_in, &blk)
-      raise "Multiple aggpairs #{aggpairs_in.map{|a| a.class.name}} in ArgAgg; only one allowed" if aggpairs_in.length > 1
+      raise Bud::Error, "multiple aggpairs #{aggpairs_in.map{|a| a.class.name}} in ArgAgg; only one allowed" if aggpairs_in.length > 1
       super(elem_name, bud_instance, collection_name, keys_in, aggpairs_in, schema_in, &blk)
       @agg = @aggpairs[0][0]
       @aggcol = @aggpairs[0][1]

data/lib/bud/executor/join.rb

@@ -16,7 +16,6 @@ module Bud
       @selfjoins = []
       @input_bufs = [[],[]]
       @missing_keys = Set.new
-      the_join = nil
 
       # if any elements on rellist are PushSHJoins, suck up their contents
       @all_rels_below = []
@@ -24,7 +23,6 @@ module Bud
         if r.class <= PushSHJoin
           @all_rels_below += r.all_rels_below
           preds += r.origpreds
-          the_join = r
         else
           @all_rels_below << r
         end
@@ -85,8 +83,7 @@ module Bud
     # extract predicates on rellist[1] and recurse to left side with remainder
     protected
     def setup_preds(preds) # :nodoc: all
-                           # print "setting up preds for #{@relnames.inspect}(#{self.object_id}): "
-                           # print "setting up preds for #{@relnames.inspect}(#{self.object_id}): "
+      # print "setting up preds for #{@relnames.inspect}(#{self.object_id}): "
       allpreds = disambiguate_preds(preds)
       allpreds = canonicalize_localpreds(@rels, allpreds)
       # check for refs to collections that aren't being joined, Issue 191
@@ -135,7 +132,7 @@ module Bud
       else
         @keys = []
       end
-                           # puts "@keys = #{@keys.inspect}"
+      # puts "@keys = #{@keys.inspect}"
     end
 
     public
@@ -301,7 +298,7 @@ module Bud
       offsets.each do |offset|
         buf = @input_bufs[offset]
         buf << item
-        if (buf.length >= ELEMENT_BUFSIZE)
+        if buf.length >= ELEMENT_BUFSIZE
           flush_buf(buf, offset)
         end
       end
@@ -309,7 +306,7 @@ module Bud
 
     protected
     def insert_item(item, offset)
-      if (@keys.nil? or @keys.empty?)
+      if @keys.nil? or @keys.empty?
         the_key = nil
       else
         # assumes left-deep trees
@@ -337,7 +334,6 @@ module Bud
 
     public
     def add_rescan_invalidate(rescan, invalidate)
-
       if non_temporal_predecessors.any? {|e| rescan.member? e}
         rescan << self
         invalidate << self
@@ -450,7 +446,6 @@ module Bud
       self.extend(Bud::PushSHOuterJoin)
     end
 
-
     public
     def rights(*preds, &blk)
       @cols = blk.nil? ? @bud_instance.tables[@rels[1].tabname].cols : nil
@@ -535,7 +530,7 @@ module Bud
 
     private
     def insert_item(item, offset)
-      if (@keys.nil? or @keys.empty?)
+      if @keys.nil? or @keys.empty?
         the_key = nil
       else
         if all_rels_below.length > 2 and offset == 1
@@ -571,7 +566,7 @@ module Bud
       if @missing_keys
         @missing_keys.each do |key|
           @hash_tables[0][key].each do |t|
-            push_out([t, []])
+            push_out([t, @rels[1].null_tuple])
           end
         end
       end
@@ -580,7 +575,7 @@ module Bud
 
   class PushNotIn < PushSHJoin
     def initialize(rellist, bud_instance, preds=nil, &blk) # :nodoc: all
-      if (preds.nil? or preds.empty?)
+      if preds.nil? or preds.empty?
         preds = positionwise_preds(bud_instance, rellist)
       end
       super(rellist, bud_instance, preds)

data/lib/bud/rewrite.rb

@@ -1,21 +1,20 @@
 require 'rubygems'
 require 'ruby2ruby'
+require 'set'
 
 class RuleRewriter < Ruby2Ruby # :nodoc: all
   attr_accessor :rule_indx, :rules, :depends
 
+  OP_LIST = Set.new([:<<, :<, :<=])
+  TEMP_OP_LIST = Set.new([:-@, :~, :+@])
+  MONOTONE_WHITELIST = Set.new([:==, :+, :<=, :-, :<, :>, :*, :~,
+                                :pairs, :matches, :combos, :flatten,
+                                :lefts, :rights, :map, :flat_map, :pro,
+                                :cols, :key_cols, :val_cols, :payloads, :lambda,
+                                :tabname, :ip_port, :port, :ip, :int_ip_port])
+
   def initialize(seed, bud_instance)
     @bud_instance = bud_instance
-    @ops = {:<< => 1, :< => 1, :<= => 1}
-    @monotonic_whitelist = {
-          :== => 1, :+ => 1, :<= => 1, :- => 1, :< => 1, :> => 1,
-          :* => 1, :pairs => 1, :matches => 1, :combos => 1, :flatten => 1,
-          :lefts => 1, :rights => 1, :map => 1, :flat_map => 1, :pro => 1,
-          :cols => 1,  :key_cols => 1, :val_cols => 1, :payloads => 1, :~ => 1,
-          :lambda => 1, :tabname => 1,
-          :ip_port => 1, :port => 1, :ip => 1
-    }
-    @temp_ops = {:-@ => 1, :~ => 1, :+@ => 1}
     @tables = {}
     @nm = false
     @rule_indx = seed
@@ -29,7 +28,7 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   $not_id = [:not_coll_id]
   def resolve(obj, prefix, name)
     qn = prefix ? prefix + "." + name.to_s : name.to_s
-    return [:collection, qn, obj.tables[name]]  if obj.tables.has_key? name
+    return [:collection, qn, obj.tables[name]] if obj.tables.has_key? name
 
     # does name refer to an import name?
     iobj = obj.import_instance name
@@ -56,60 +55,72 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
 
   def process_call(exp)
     recv, op, args = exp
-    if @ops[op] and @context[1] == :block and @context.length == 4
+    if OP_LIST.include?(op) and @context[1] == :block and @context.length == 4
       # NB: context.length is 4 when see a method call at the top-level of a
       # :defn block -- this is where we expect Bloom statements to appear
       do_rule(exp)
     else
-      ty = :not_coll_id
-      ty, qn, obj = exp_id_type(recv, op, args) # qn = qualified name, obj is the corresponding object
+      ty, qn, _ = exp_id_type(recv, op, args) # qn = qualified name
       if ty == :collection
         @tables[qn] = @nm if @collect
       #elsif ty == :import .. do nothing
       elsif ty == :not_coll_id
-        # check if receiver is a collection, and further if the current exp represents a field lookup
+        # check if receiver is a collection, and further if the current exp
+        # represents a field lookup
         op_is_field_name = false
         if recv and recv.first == :call
           rty, _, robj = exp_id_type(recv[1], recv[2], recv[3])
           if rty == :collection
             cols = robj.cols
-            op_is_field_name =  true if cols and cols.include?(op)
+            op_is_field_name = true if cols and cols.include?(op)
           end
         end
         # for CALM analysis, mark deletion rules as non-monotonic
         @nm = true if op == :-@
         if recv
-          # don't worry about monotone ops, table names, table.attr calls, or accessors of iterator variables
-          unless @monotonic_whitelist[op] or op_is_field_name or recv.first == :lvar or op.to_s.start_with?("__")
-            @nm = true if recv
+          # don't worry about monotone ops, table names, table.attr calls, or
+          # accessors of iterator variables
+          unless RuleRewriter.is_monotone(op) or op_is_field_name or
+                 recv.first == :lvar or op.to_s.start_with?("__")
+            @nm = true
           end
         else
-          # function called (implicit receiver = Bud instance) in a user-defined code block. Check if it is
-          # non-monotonic (like budtime, that produces a new answer every time it is called)
-          @nm_funcs_called = true unless @monotonic_whitelist[op]
+          # function called (implicit receiver = Bud instance) in a user-defined
+          # code block. Check if it is non-monotonic (like budtime, that
+          # produces a new value every time it is called)
+          @nm_funcs_called = true unless RuleRewriter.is_monotone(op)
         end
       end
-      if @temp_ops[op]
+      if TEMP_OP_LIST.include? op
         @temp_op = op.to_s.gsub("@", "")
       end
       super
     end
   end
 
+  def self.is_monotone(op)
+    MONOTONE_WHITELIST.include?(op)
+  end
+
+  # rewrite constant array expressions to lambdas
+  def lambda_rewrite(rhs)
+    # the <= case
+    if rhs[0] == :array
+      return s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, rhs)
+    # the superator case
+    elsif rhs[0] == :call \
+      and rhs[1] and rhs[1][0] and rhs[1][0] == :array \
+      and rhs[2] and (rhs[2] == :+@ or rhs[2] == :-@ or rhs[2] == :~@)
+      return s(rhs[0], s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, rhs[1]), rhs[2], rhs[3])
+    else
+      return rhs
+    end
+  end
+
   def collect_rhs(exp)
+    exp = lambda_rewrite(exp)
+
     @collect = true
-    # rewrite constant array expressions to lambdas
-    if exp[0] and exp[0] == :arglist
-      # the <= case
-      if exp[1] and exp[1][0] == :array
-        exp = s(exp[0], s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, exp[1]))
-      # the superator case
-      elsif exp[1] and exp[1][0] == :call \
-        and exp[1][1] and exp[1][1][0] and exp[1][1][0] == :array \
-        and exp[1][2] and (exp[1][2] == :+@ or exp[1][2] == :-@ or exp[1][2] == :~@)
-        exp = s(exp[0], s(exp[1][0], s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, exp[1][1]), exp[1][2], exp[1][3]))
-      end
-    end
     rhs = process exp
     @collect = false
     return rhs
@@ -132,8 +143,8 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
     end
 
     @rules << [@bud_instance, @rule_indx, lhs, op, rule_txt, rule_txt_orig, @nm_funcs_called]
-    @tables.each_pair do |t, non_monotonic|
-      @depends << [@bud_instance, @rule_indx, lhs, op, t, non_monotonic]
+    @tables.each_pair do |t, nm|
+      @depends << [@bud_instance, @rule_indx, lhs, op, t, nm]
     end
 
     reset_instance_vars
@@ -149,7 +160,9 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
     # s(:arglist) is not really sensible and it causes Ruby2Ruby < 1.3.1 to
     # misbehave (for example, s(:arglist, s(:hash, ...)) is misparsed.
     raise Bud::CompileError unless rhs_ast.sexp_type == :arglist
-    #rhs_ast = rhs_ast[1]
+    rhs_ast = rhs_ast[1]
+
+    rhs_ast = RenameRewriter.new(@bud_instance).process(rhs_ast)
 
     if @bud_instance.options[:no_attr_rewrite]
       rhs = collect_rhs(rhs_ast)
@@ -191,6 +204,37 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
   end
 end
 
+# Look for rename statements and define the necessary scratch collections
+class RenameRewriter < SexpProcessor
+  def initialize(bud_instance)
+    super()
+    self.require_empty = false
+    self.expected = Sexp
+    @bud_instance = bud_instance
+  end
+
+  def register_scratch(name, schemahash)
+    # define a scratch with the name and schema in this rename block
+    hash, key_array, val_array = schemahash
+    key_array ||= []
+    val_array ||= []
+    key_cols = key_array.map{|i| i[1] if i.class <= Sexp}.compact
+    val_cols = val_array.map{|i| i[1] if i.class <= Sexp}.compact
+    @bud_instance.scratch(name, key_cols=>val_cols)
+  end
+
+  def process_call(exp)
+    call, recv, op, args = exp
+
+    if op == :rename
+      arglist, namelit, schemahash = args
+      register_scratch(namelit[1], schemahash)
+    end
+
+    return s(call, process(recv), op, process(args))
+  end
+end
+
 # Rewrite named-column refs to positional refs
 class AttrNameRewriter < SexpProcessor # :nodoc: all
   def initialize(bud_instance)
@@ -225,7 +269,7 @@ class AttrNameRewriter < SexpProcessor # :nodoc: all
         return unless exp[2][1] and exp[2][1][0] == :array
         if exp[1][2] == :reduce
           unless @collnames.length == 1
-            raise Bud::Error, "reduce should only one associated collection, but has #{@collnames.inspect}"
+            raise Bud::Error, "reduce should only have one associated collection, but has #{@collnames.inspect}"
           end
           @iterhash[exp[2][1][2][1]] = @collnames.first
         else #join
@@ -240,16 +284,6 @@ class AttrNameRewriter < SexpProcessor # :nodoc: all
     exp
   end
 
-  def register_scratch(name, schemahash)
-    # define a scratch with the name and schema in this rename block
-    hash, key_array, val_array = schemahash
-    key_array ||= []
-    val_array ||= []
-    key_cols = key_array.map{|i| i[1] if i.class <= Sexp}.compact
-    val_cols = val_array.map{|i| i[1] if i.class <= Sexp}.compact
-    @bud_instance.scratch(name, key_cols=>val_cols)
-  end
-
   def gather_collection_names(exp)
     if exp[0] == :call and exp[1].nil?
       @collnames << exp[2]
@@ -264,10 +298,6 @@ class AttrNameRewriter < SexpProcessor # :nodoc: all
   def process_call(exp)
     call, recv, op, args = exp
 
-    if op == :rename
-      arglist, namelit, schemahash = args
-      register_scratch(namelit[1], schemahash)
-    end
     if recv and recv.class == Sexp and recv.first == :lvar and recv[1] and @iterhash[recv[1]]
       if @bud_instance.respond_to?(@iterhash[recv[1]])
         if @bud_instance.send(@iterhash[recv[1]]).class <= Bud::BudCollection

data/lib/bud/storage/dbm.rb

@@ -166,9 +166,9 @@ module Bud
     public
     def pending_delete(o)
       if o.class <= Bud::PushElement
-         o.wire_to_delete self
+         o.wire_to(self, :delete)
       elsif o.class <= Bud::BudCollection
-        o.pro.wire_to_delete self
+        o.pro.wire_to(self, :delete)
       else
         @to_delete = @to_delete + o.map{|t| prep_tuple(t) unless t.nil?}
       end

data/lib/bud/viz_util.rb

@@ -146,9 +146,11 @@ module VizUtil #:nodoc: all
   end
 
   def write_graphs(tabinf, builtin_tables, cycle, depends, rules, viz_name,
-                   output_base, fmt, collapse, depanalysis=nil, budtime=-1, card_info=nil, pathsto={}, begins = {})
+                   output_base, fmt, collapse, depanalysis=nil, budtime=-1,
+                   card_info=nil, pathsto={}, begins={})
     staging = "#{viz_name}.staging"
-    gv = GraphGen.new(tabinf, builtin_tables, cycle, staging, budtime, collapse, card_info, pathsto, begins)
+    gv = GraphGen.new(tabinf, builtin_tables, cycle, staging, budtime,
+                      collapse, card_info, pathsto, begins)
     gv.process(depends)
     dump(rules, output_base, gv)
     gv.finish(depanalysis, fmt)

data/lib/bud.rb

@@ -62,7 +62,7 @@ $bud_instances = {}        # Map from instance id => Bud instance
 module Bud
   attr_reader :budtime, :inbound, :options, :meta_parser, :viz, :rtracer, :dsock
   attr_reader :tables, :builtin_tables, :channels, :zk_tables, :dbm_tables, :app_tables
-  attr_reader :push_sources, :push_elems, :push_joins, :scanners, :merge_targets, :done_wiring
+  attr_reader :push_sources, :push_elems, :push_joins, :scanners, :merge_targets
   attr_reader :this_stratum, :this_rule, :rule_orig_src, :done_bootstrap
   attr_accessor :stratified_rules
   attr_accessor :metrics, :periodics
@@ -75,7 +75,6 @@ module Bud
   #   * <tt>:port</tt>   port number for this instance
   #   * <tt>:ext_ip</tt>  IP address at which external nodes can contact this instance
   #   * <tt>:ext_port</tt>   port number to go with <tt>:ext_ip</tt>
-  #   * <tt>:bust_port</tt>  port number for the restful HTTP messages
   # * operating system interaction
   #   * <tt>:stdin</tt>  if non-nil, reading from the +stdio+ collection results in reading from this +IO+ handle
   #   * <tt>:stdout</tt> writing to the +stdio+ collection results in writing to this +IO+ handle; defaults to <tt>$stdout</tt>
@@ -199,6 +198,11 @@ module Bud
     toplevel? ? @budtime : toplevel.budtime
   end
 
+  # Are we currently in the process of wiring together the dataflow?
+  def wiring?
+    toplevel? ? (@done_bootstrap && !@done_wiring) : toplevel.wiring?
+  end
+
   private
   def import_defs
     @imported_defs = {} # mod name -> Module map
@@ -999,20 +1003,20 @@ module Bud
         fixpoint = false
         first_iter = true
         until fixpoint
-          fixpoint = true
           @scanners[stratum].each_value {|s| s.scan(first_iter)}
+          fixpoint = true
           first_iter = false
           # flush any tuples in the pipes
           @push_sorted_elems[stratum].each {|p| p.flush}
           # tick deltas on any merge targets and look for more deltas
           # check to see if any joins saw a delta
-          push_joins[stratum].each do |p|
+          @push_joins[stratum].each do |p|
             if p.found_delta
               fixpoint = false
               p.tick_deltas
             end
           end
-          merge_targets[stratum].each do |t|
+          @merge_targets[stratum].each do |t|
             fixpoint = false if t.tick_deltas
           end
         end
@@ -1020,7 +1024,7 @@ module Bud
         @push_sorted_elems[stratum].each do |p|
           p.stratum_end
         end
-        merge_targets[stratum].each do |t|
+        @merge_targets[stratum].each do |t|
           t.flush_deltas
         end
       end
@@ -1054,6 +1058,17 @@ module Bud
     @tick_clock_time
   end
 
+  # Return the stratum number of the given collection.
+  # NB: if a collection is not referenced by any rules, it is not currently
+  # assigned to a strata.
+  def collection_stratum(collection)
+    t_stratum.each do |t|
+      return t.stratum if t.predicate == collection
+    end
+
+    raise Bud::Error, "no such collection: #{collection}"
+  end
+
   private
 
   # Builtin Bud state (predefined collections). We could define this using the
@@ -1125,7 +1140,6 @@ module Bud
     end
   end
 
-  private
   ######## ids and timers
   def gen_id
     Time.new.to_i.to_s << rand.to_s