bud 0.1.0.pre1 → 0.9.0

data/lib/bud/executor/elements.rb

@@ -1,5 +1,6 @@
-require "set"
+require 'set'
 require 'bud/collections'
+
 ELEMENT_BUFSIZE = 1
 
 module Bud
@@ -14,8 +15,8 @@ module Bud
   class PushElement < BudCollection
     attr_accessor :elem_name
     attr_accessor :rescan, :invalidated
-    attr_reader :arity, :inputs, :found_delta, :refcount, :wired_by, :outputs
-    
+    attr_reader :inputs, :found_delta, :refcount, :wired_by, :outputs
+
     def initialize(name_in, bud_instance, collection_name=nil, given_schema=nil, defer_schema=false, &blk)
       super(name_in, bud_instance, given_schema, defer_schema)
       @blk = blk
@@ -27,7 +28,6 @@ module Bud
       @elem_name = name_in
       @found_delta = false
       @refcount = 1
-      @each_index = 0
       @collection_name = collection_name
       @invalidated = true
       @rescan = true
@@ -42,7 +42,7 @@ module Bud
     end
 
     public
-    def print_wiring(depth=0, accum = "")
+    def print_wiring(depth=0, accum="")
       depth.times {print "  "}
       puts "#{accum} #{(self.object_id*2).to_s(16)}: #{qualified_tabname} (#{self.class})"
 
@@ -54,11 +54,11 @@ module Bud
           next_accum = "+> "
         when @deletes.object_id, @delete_keys.object_id
           next_accum = "-> "
-        end  
-        
+        end
+
         kind.each do |o|
-          if o.respond_to?(:print_wiring) 
-            o.print_wiring(depth+1, next_accum) 
+          if o.respond_to?(:print_wiring)
+            o.print_wiring(depth+1, next_accum)
           else
             (depth+1).times {print "  "}
             print "#{next_accum} "
@@ -74,16 +74,16 @@ module Bud
 
     def check_wiring
       if @blk.nil? and @outputs.empty? and @pendings.empty? and @deletes.empty? and @delete_keys.empty?
-        raise "no output specified for PushElement #{@qualified_tabname}"
+        raise Bud::Error, "no output specified for PushElement #{@qualified_tabname}"
       end
     end
-      
+
     def set_block(&blk)
       @blk = blk
     end
     def wire_to(element)
-      unless element.methods.include? :insert or element.methods.include? "insert"
-        raise Bud::Error, "attempt to wire_to element without insert method" 
+      unless element.respond_to? :insert
+        raise Bud::Error, "attempt to wire_to element without insert method"
       end
       # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
       # puts "wiring #{self.elem_name} to #{elem_name}"
@@ -91,22 +91,28 @@ module Bud
       element.wired_by << self if element.respond_to? :wired_by
     end
     def wire_to_pending(element)
-      raise Bud::Error, "attempt to wire_to_pending element without pending_merge method" unless element.methods.include? "pending_merge" or element.methods.include? :pending_merge
-      elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
+      unless element.respond_to? :pending_merge
+        raise Bud::Error, "attempt to wire_to_pending element without pending_merge method"
+      end
+      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
       # puts "wiring #{self.elem_name} to #{elem_name}(pending)"
       @pendings << element
       element.wired_by << self if element.respond_to? :wired_by
     end
     def wire_to_delete(element)
-      raise Bud::Error, "attempt to wire_to_delete element without pending_delete method" unless element.methods.include? "pending_delete" or element.methods.include? :pending_delete
-      elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
+      unless element.respond_to? :pending_delete
+        raise Bud::Error, "attempt to wire_to_delete element without pending_delete method"
+      end
+      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
       # puts "wiring #{self.elem_name} to #{elem_name}(delete)"
       @deletes << element
       element.wired_by << self if element.respond_to? :wired_by
     end
     def wire_to_delete_by_key(element)
-      raise Bud::Error, "attempt to wire_to_delete_by_key element without pending_delete_keys method" unless element.methods.include? "pending_delete_keys" or element.methods.include? :pending_delete_keys
-      elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
+      unless element.respond_to? :pending_delete_keys
+        raise Bud::Error, "attempt to wire_to_delete_by_key element without pending_delete_keys method"
+      end
+      # elem_name = element.respond_to?(:tabname) ? element.tabname : element.elem_name
       # puts "wiring #{self.elem_name} to #{elem_name}(delete)"
       @delete_keys << element
       element.wired_by << self if element.respond_to? :wired_by
@@ -116,7 +122,6 @@ module Bud
       false
     end
 
-
     def insert(item, source=nil)
       push_out(item)
     end
@@ -131,36 +136,34 @@ module Bud
 
     def push_out(item, do_block=true)
       if item
-        blk = @blk if do_block
-        if blk
-          item = item.to_a if blk.arity > 1
-          begin
-          item = blk.call item
-          rescue Exception
-            raise
-          end
+        if do_block && @blk
+          item = item.to_a if @blk.arity > 1
+          item = @blk.call item
         end
-        @outputs.each do |ou|
-          if ou.class <= Bud::PushElement
-            #the_name = ou.elem_name
-            # puts "#{self.object_id%10000} (#{elem_name}) -> #{ou.object_id%10000} (#{the_name}): #{item.inspect}"
-            ou.insert(item,self)
-          elsif ou.class <= Bud::BudCollection
-            # the_name = ou.tabname
-            # puts "#{self.object_id%10000} (#{elem_name}) -> #{ou.object_id%10000} (#{the_name}): #{item.inspect}"
-            ou.do_insert(item,ou.new_delta)
-          else
-            raise "Expected either a PushElement or a BudCollection"
+
+        unless item.nil?
+          @outputs.each do |ou|
+            if ou.class <= Bud::PushElement
+              #the_name = ou.elem_name
+              # puts "#{self.object_id%10000} (#{elem_name}) -> #{ou.object_id%10000} (#{the_name}): #{item.inspect}"
+              ou.insert(item, self)
+            elsif ou.class <= Bud::BudCollection
+              # the_name = ou.tabname
+              # puts "#{self.object_id%10000} (#{elem_name}) -> #{ou.object_id%10000} (#{the_name}): #{item.inspect}"
+              ou.do_insert(item, ou.new_delta)
+            else
+              raise Bud::Error, "expected either a PushElement or a BudCollection"
+            end
           end
-        end unless item.nil?
-        # for all the following, o is a BudCollection
-        @deletes.each{|o| o.pending_delete([item])} unless item.nil?
-        @delete_keys.each{|o| o.pending_delete_keys([item])} unless item.nil?
-        @pendings.each{|o| o.pending_merge([item])} unless item.nil?
+
+          # for all the following, o is a BudCollection
+          @deletes.each{|o| o.pending_delete([item])}
+          @delete_keys.each{|o| o.pending_delete_keys([item])}
+          @pendings.each{|o| o.pending_merge([item])}
+        end
       end
     end
 
-
     # default for stateless elements
     public
     def add_rescan_invalidate(rescan, invalidate)
@@ -170,23 +173,25 @@ module Bud
         rescan << self
       end
 
-      # pass the current state to the non-element outputs, and see if they end up marking this node for rescan
+      # pass the current state to the non-element outputs, and see if they end
+      # up marking this node for rescan
       invalidate_tables(rescan, invalidate)
 
-      # finally, if this node is in rescan, pass the request on to all source elements
+      # finally, if this node is in rescan, pass the request on to all source
+      # elements
       if rescan.member? self
-        srcs.each{|e| rescan << e} # propagate a rescan request to all sources.
+        rescan += srcs
       end
     end
 
     def invalidate_tables(rescan, invalidate)
-      # exchange rescan and invalidate information with tables. If this node is in rescan, it may invalidate a target
-      # table (if it is a scratch). And if the target node is invalidated, this node marks itself for rescan to
+      # exchange rescan and invalidate information with tables. If this node is
+      # in rescan, it may invalidate a target table (if it is a scratch). And if
+      # the target node is invalidated, this node marks itself for rescan to
       # enable a refill of that table at run-time
-
       @outputs.each do |o|
         unless o.class <= PushElement
-          o.add_rescan_invalidate(rescan, invalidate) unless o.class <= PushElement
+          o.add_rescan_invalidate(rescan, invalidate)
           rescan << self if invalidate.member? o
         end
       end
@@ -211,45 +216,48 @@ module Bud
     #  setup_accessors
     #end
 
-    
+
     ####
     # and now, the Bloom-facing methods
     public
-    def pro(the_name = @elem_name, the_schema = schema, &blk)
+    def pro(the_name=@elem_name, the_schema=schema, &blk)
       toplevel = @bud_instance.toplevel
-      elem = Bud::PushElement.new('project' + object_id.to_s, toplevel.this_rule_context, @collection_name, the_schema)
-      #elem.init_schema(the_schema) unless the_schema.nil?
+      elem = Bud::PushElement.new('project' + object_id.to_s,
+                                  toplevel.this_rule_context,
+                                  @collection_name, the_schema)
       self.wire_to(elem)
       elem.set_block(&blk)
-      toplevel.push_elems[[self.object_id,:pro,blk]] = elem
+      toplevel.push_elems[[self.object_id, :pro, blk]] = elem
       return elem
     end
-    
+
     alias each pro
-    
+
     public
-    def each_with_index(the_name = elem_name, the_schema = schema, &blk)
+    def each_with_index(the_name=elem_name, the_schema=schema, &blk)
       toplevel = @bud_instance.toplevel
-      elem = Bud::PushEachWithIndex.new('each_with_index' + object_id.to_s, toplevel.this_rule_context, @collection_name)
+      elem = Bud::PushEachWithIndex.new('each_with_index' + object_id.to_s,
+                                        toplevel.this_rule_context,
+                                        @collection_name)
       elem.set_block(&blk)
       self.wire_to(elem)
-      toplevel.push_elems[[self.object_id,:each,blk]] = elem
+      toplevel.push_elems[[self.object_id, :each, blk]] = elem
     end
-    
+
     def join(elem2, &blk)
       # cached = @bud_instance.push_elems[[self.object_id,:join,[self,elem2], @bud_instance, blk]]
       # if cached.nil?
         elem2  = elem2.to_push_elem unless elem2.class <= PushElement
         toplevel = @bud_instance.toplevel
-        join = Bud::PushSHJoin.new([self,elem2], toplevel.this_rule_context, [])
+        join = Bud::PushSHJoin.new([self, elem2], toplevel.this_rule_context, [])
         self.wire_to(join)
         elem2.wire_to(join)
-        toplevel.push_elems[[self.object_id,:join,[self,elem2], toplevel, blk]] = join
+        toplevel.push_elems[[self.object_id, :join, [self, elem2], toplevel, blk]] = join
         toplevel.push_joins[toplevel.this_stratum] << join
       # else
       #   cached.refcount += 1
       # end
-      return toplevel.push_elems[[self.object_id,:join,[self,elem2], toplevel, blk]]
+      return toplevel.push_elems[[self.object_id, :join, [self, elem2], toplevel, blk]]
     end
     def *(elem2, &blk)
       join(elem2, &blk)
@@ -260,7 +268,7 @@ module Bud
       notin_elem = Bud::PushNotIn.new([self, elem2], toplevel.this_rule_context, preds, &blk)
       self.wire_to(notin_elem)
       elem2.wire_to(notin_elem)
-      toplevel.push_elems[[self.object_id, :notin, collection, toplevel, blk]] == notin_elem
+      toplevel.push_elems[[self.object_id, :notin, collection, toplevel, blk]] = notin_elem
       return notin_elem
     end
 
@@ -268,20 +276,20 @@ module Bud
       if source.class <= PushElement and wiring?
         source.wire_to(self)
       else
-        source.each{|i| self << i}
+        source.each {|i| self << i}
       end
     end
     alias <= merge
     superator "<~" do |o|
-      raise Bud::Error, "Illegal use of <~ with pusher '#{tabname}' on left"
+      raise Bud::Error, "illegal use of <~ with pusher '#{tabname}' on left"
     end
 
     superator "<-" do |o|
-      raise Bud::Error, "Illegal use of <- with pusher '#{tabname}' on left"
+      raise Bud::Error, "illegal use of <- with pusher '#{tabname}' on left"
     end
 
     superator "<+" do |o|
-      raise Bud::Error, "Illegal use of <+ with pusher '#{tabname}' on left"
+      raise Bud::Error, "illegal use of <+ with pusher '#{tabname}' on left"
     end
 
     def group(keycols, *aggpairs, &blk)
@@ -293,13 +301,13 @@ module Bud
       aggcols = []
       aggcolsdups.each_with_index do |n, i|
         aggcols << "#{n.downcase}_#{i}".to_sym
-      end      
+      end
       if aggcols.empty?
         the_schema = keynames
       else
         the_schema = { keynames => aggcols }
       end
-      
+
       aggpairs = aggpairs.map{|ap| ap[1].nil? ? [ap[0]] : [ap[0], canonicalize_col(ap[1])]}
       toplevel = @bud_instance.toplevel
       # if @bud_instance.push_elems[[self.object_id, :group, keycols, aggpairs, blk]].nil?
@@ -325,7 +333,7 @@ module Bud
           k[2]
         end
       end
-      aggpairs = [[agg,collection]]
+      aggpairs = [[agg, collection]]
       # if toplevel.push_elems[[self.object_id,:argagg, gbkey_cols, aggpairs, blk]].nil?
         aa = Bud::PushArgAgg.new('argagg'+Time.new.tv_usec.to_s, toplevel.this_rule_context, @collection_name, gbkey_cols, aggpairs, schema, &blk)
         self.wire_to(aa)
@@ -345,8 +353,10 @@ module Bud
       wire_to(elem)
       elem
     end
-    def push_predicate(pred_symbol, name=nil, bud_instance=nil, the_schema=nil, &blk)
-      elem = Bud::PushPredicate.new(pred_symbol, name, bud_instance, the_schema, &blk)
+    def push_predicate(pred_symbol, name=nil, bud_instance=nil,
+                       the_schema=nil, &blk)
+      elem = Bud::PushPredicate.new(pred_symbol, name, bud_instance,
+                                    the_schema, &blk)
       wire_to(elem)
       elem
     end
@@ -368,44 +378,45 @@ module Bud
     def one?(name=nil, bud_instance=nil, the_schema=nil, &blk)
       push_predicate(:one?, name, bud_instance, the_schema, &blk)
     end
-    
+
     def reduce(initial, &blk)
       @memo = initial
-      retval = Bud::PushReduce.new('reduce'+Time.new.tv_usec.to_s, @bud_instance, @collection_name, schema, initial, &blk)
+      retval = Bud::PushReduce.new("reduce#{Time.new.tv_usec}",
+                                   @bud_instance, @collection_name,
+                                   schema, initial, &blk)
       self.wire_to(retval)
       retval
     end
-    
+
     alias on_exists? pro
     def on_include?(item, &blk)
       toplevel = @bud_instance.toplevel
-      if toplevel.push_elems[[self.object_id,:on_include?, item, blk]].nil?
+      if toplevel.push_elems[[self.object_id, :on_include?, item, blk]].nil?
         inc = pro{|i| blk.call(item) if i == item and not blk.nil?}
         wire_to(inc)
-        toplevel.push_elems[[self.object_id,:on_include?, item, blk]] = inc
+        toplevel.push_elems[[self.object_id, :on_include?, item, blk]] = inc
       end
-      toplevel.push_elems[[self.object_id,:on_include?, item, blk]]
+      toplevel.push_elems[[self.object_id, :on_include?, item, blk]]
     end
     def inspected
       toplevel = @bud_instance.toplevel
-      if toplevel.push_elems[[self.object_id,:inspected]].nil?
+      if toplevel.push_elems[[self.object_id, :inspected]].nil?
         ins = pro{|i| [i.inspect]}
         self.wire_to(ins)
-        toplevel.push_elems[[self.object_id,:inspected]] = ins
+        toplevel.push_elems[[self.object_id, :inspected]] = ins
       end
-      toplevel.push_elems[[self.object_id,:inspected]]
+      toplevel.push_elems[[self.object_id, :inspected]]
     end
-    
+
     def to_enum
-        # scr = @bud_instance.scratch(("scratch_" + Process.pid.to_s + "_" + object_id.to_s + "_" + rand(10000).to_s).to_sym, schema)
-        scr = []
-        self.wire_to(scr)
-        scr
+      # scr = @bud_instance.scratch(("scratch_" + Process.pid.to_s + "_" + object_id.to_s + "_" + rand(10000).to_s).to_sym, schema)
+      scr = []
+      self.wire_to(scr)
+      scr
     end
-  end  
+  end
 
   class PushStatefulElement < PushElement
-
     def rescan_at_tick
       true
     end
@@ -415,10 +426,9 @@ module Bud
     end
 
     def add_rescan_invalidate(rescan, invalidate)
-      # If an upstream node is set to rescan, a stateful node invalidates its cache
-      # In addition, a stateful node always rescans its own contents (doesn't need to pass a rescan request to its
-      # its source nodes
-
+      # If an upstream node is set to rescan, a stateful node invalidates its
+      # cache.  In addition, a stateful node always rescans its own contents
+      # (doesn't need to pass a rescan request to its its source nodes).
       rescan << self
       srcs = non_temporal_predecessors
       if srcs.any? {|p| rescan.member? p}
@@ -435,7 +445,7 @@ module Bud
       @in_buf = []
       super(elem_name, bud_instance, collection_name, schema_in, &blk)
     end
-  
+
     def insert(item, source)
       @in_buf << item
     end
@@ -450,13 +460,14 @@ module Bud
       @in_buf.clear
     end
   end
-  
+
   class PushSort < PushStatefulElement
-    def initialize(elem_name=nil, bud_instance=nil, collection_name=nil, schema_in=nil, &blk)
+    def initialize(elem_name=nil, bud_instance=nil, collection_name=nil,
+                   schema_in=nil, &blk)
       @sortbuf = []
       super(elem_name, bud_instance, collection_name, schema_in, &blk)
     end
-  
+
     def insert(item, source)
       @sortbuf << item
     end
@@ -476,12 +487,13 @@ module Bud
       @sortbuf = []
     end
   end
-  
+
   class ScannerElement < PushElement
     attr_reader :collection
     attr_reader :rescan_set, :invalidate_set
-    def initialize(elem_name, bud_instance, collection_in, the_schema=collection_in.schema, &blk)
-      # puts self.class
+
+    def initialize(elem_name, bud_instance, collection_in,
+                   the_schema=collection_in.schema, &blk)
       super(elem_name, bud_instance, collection_in.qualified_tabname, the_schema)
       @collection = collection_in
       @rescan_set = []
@@ -496,8 +508,9 @@ module Bud
       @collection.invalidate_at_tick # need to scan afresh if collection invalidated.
     end
 
+    # collection of others to rescan/invalidate if this scanner's collection
+    # were to be invalidated.
     def invalidate_at_tick(rescan, invalidate)
-      # collection of others to rescan/invalidate if this scanner's collection were to be invalidated.
       @rescan_set = rescan
       @invalidate_set = invalidate
     end
@@ -507,19 +520,19 @@ module Bud
       # scanner elements are never directly connected to tables.
       rescan << self if invalidate.member? @collection
 
-      # Note also that this node can be nominated for rescan by a target node; in other words, a scanner element
-      # can be set to rescan even if the collection is not invalidated.
+      # Note also that this node can be nominated for rescan by a target node;
+      # in other words, a scanner element can be set to rescan even if the
+      # collection is not invalidated.
     end
 
     def scan(first_iter)
-      if (first_iter)
+      if first_iter
         if rescan
-          # scan entire storage
-          @collection.each_raw {|item|
-            push_out(item)
-          }
+          # Scan entire storage
+          @collection.each_raw {|item| push_out(item)}
         else
-          # In the first iteration, tick_delta would be non-null IFF the collection has grown in an earlier stratum
+          # In the first iteration, tick_delta would be non-null IFF the
+          # collection has grown in an earlier stratum
           @collection.tick_delta.each {|item| push_out(item)}
         end
       end
@@ -530,14 +543,15 @@ module Bud
   end
 
   class PushReduce < PushStatefulElement
-    def initialize(elem_name, bud_instance, collection_name, schema_in, initial, &blk)
+    def initialize(elem_name, bud_instance, collection_name,
+                   schema_in, initial, &blk)
       @memo = initial
       @blk = blk
       super(elem_name, bud_instance, collection_name, schema)
     end
 
     def insert(i, source=nil)
-      @memo = @blk.call(@memo,i)
+      @memo = @blk.call(@memo, i)
     end
 
     def invalidate_cache
@@ -567,11 +581,12 @@ module Bud
 
       invalidate_tables(rescan, invalidate)
 
-      # This node has some state (@each_index), but not the tuples. If it is in rescan mode, then it must ask its
-      # sources to rescan, and restart its index.
+      # This node has some state (@each_index), but not the tuples. If it is in
+      # rescan mode, then it must ask its sources to rescan, and restart its
+      # index.
       if rescan.member? self
         invalidate << self
-        srcs.each {|e| rescan << e}
+        rescan += srcs
       end
     end
 
@@ -585,7 +600,7 @@ module Bud
 
     def insert(item, source=nil)
       ix = @each_index
-      @each_index = ix + 1
+      @each_index += 1
       push_out([item, ix])
     end
   end

data/lib/bud/executor/group.rb

@@ -5,7 +5,7 @@ module Bud
     def initialize(elem_name, bud_instance, collection_name, keys_in, aggpairs_in, schema_in, &blk)
       @groups = {}
       if keys_in.nil?
-        @keys = [] 
+        @keys = []
       else
         @keys = keys_in.map{|k| k[1]}
       end
@@ -13,9 +13,9 @@ module Bud
       @aggpairs = aggpairs_in.map{|ap| ap[1].nil? ? [ap[0]] : [ap[0], ap[1][1]]}
       super(elem_name, bud_instance, collection_name, schema_in, &blk)
     end
-  
+
     def insert(item, source)
-      key = @keys.map{|k| item[k]}   
+      key = @keys.map{|k| item[k]}
       @aggpairs.each_with_index do |ap, agg_ix|
         agg_input = ap[1].nil? ? item : item[ap[1]]
         agg = (@groups[key].nil? or @groups[key][agg_ix].nil?) ? ap[0].send(:init, agg_input) : ap[0].send(:trans, @groups[key][agg_ix], agg_input)[0]
@@ -43,7 +43,7 @@ module Bud
       #@groups = {}
     end
   end
-  
+
   class PushArgAgg < PushGroup
     def initialize(elem_name, bud_instance, collection_name, keys_in, aggpairs_in, schema_in, &blk)
       raise "Multiple aggpairs #{aggpairs_in.map{|a| a.class.name}} in ArgAgg; only one allowed" if aggpairs_in.length > 1
@@ -63,7 +63,7 @@ module Bud
     def insert(item, source)
       key = @keys.map{|k| item[k]}
       @aggpairs.each_with_index do |ap, agg_ix|
-                agg_input = item[ap[1]]
+        agg_input = item[ap[1]]
         if @groups[key].nil?
           agg = ap[0].send(:init, agg_input)
           @winners[key] = [item]
@@ -88,7 +88,7 @@ module Bud
         @groups[key] ||= Array.new(@aggpairs.length)
         @groups[key][agg_ix] = agg
         #push_out(nil)
-      end      
+      end
     end
 
     def flush