bud 0.0.3 → 0.0.4

data/lib/bud/joins.rb

@@ -3,12 +3,12 @@ module Bud
     attr_accessor :rels, :origrels, :origpreds # :nodoc: all
     attr_reader :hash_tables # :nodoc: all
 
-    def initialize(rellist, bud_instance, preds=nil) # :nodoc: all
+    def initialize(rellist, bud_instance, preds=[]) # :nodoc: all
       @schema = []
-      otherpreds = nil
       @origpreds = preds
       @bud_instance = bud_instance
       @localpreds = nil
+      @hashpreds = nil
 
       # if any elements on rellist are BudJoins, suck up their contents
       tmprels = []
@@ -25,7 +25,7 @@ module Bud
 
       # recurse to form a tree of binary BudJoins
       @rels = [rellist[0]]
-      @rels << (rellist.length == 2 ? rellist[1] : BudJoin.new(rellist[1..rellist.length-1], @bud_instance, nil))
+      @rels << (rellist.length == 2 ? rellist[1] : BudJoin.new(rellist[1..rellist.length-1], @bud_instance))
       # derive schema: one column for each table.
       # duplicated inputs get distinguishing numeral
       @schema = []
@@ -39,7 +39,7 @@ module Bud
         memo
       end
 
-      setup_preds(preds) unless preds.nil? or preds.empty?
+      setup_preds(preds) unless preds.empty?
       setup_state
     end
 
@@ -76,7 +76,7 @@ module Bud
     # similar to <tt>SELECT * FROM ... WHERE...</tt> block in SQL.
     public
     def flatten(*preds)
-      setup_preds(preds) unless preds.nil? or preds.size == 0
+      setup_preds(preds) unless preds.empty?
       flat_schema = @rels.map{|r| r.schema}.flatten(1)
       dupfree_schema = []
       # while loop here (inefficiently) ensures no collisions
@@ -127,8 +127,8 @@ module Bud
 
       methods.each do |left_rel|
         methods.each do |right_rel|
-          next if (mode == :delta and left_rel == :storage and right_rel == :storage)
-          if @localpreds.nil? or @localpreds.empty?
+          next if (mode == :both and left_rel == :storage and right_rel == :storage)
+          if @hashpreds.nil? or @hashpreds.empty?
             nestloop_join(left_rel, right_rel, &block)
           else
             hash_join(left_rel, right_rel, &block)
@@ -138,6 +138,156 @@ module Bud
       tick_hash_deltas
     end
 
+    # given a * expression over n collections, form all combinations of items
+    # subject to an array of predicates, pred
+    # currently supports two options for equijoin predicates:
+    #    general form: an array of arrays capturing a conjunction of equiv. classes
+    #          [[table1.col1, table2.col2, table3.col3], [table1.col2, table2.col3]]
+    #    common form: a hash capturing equality of a column on left with one on right.
+    #          :col1 => :col2  (same as  lefttable.col1 => righttable.col2)
+    public
+    def pairs(*preds, &blk)
+      @origpreds = preds
+      setup_preds(preds) unless preds.empty?
+      # given new preds, the state for the join will be different.  set it up again.
+      setup_state if self.class <= Bud::BudJoin
+      blk.nil? ? self : map(&blk)
+    end
+
+    alias combos pairs
+
+    # the natural join: given a * expression over n collections, form all
+    # combinations of items that have the same values in matching fields
+    public
+    def matches(&blk)
+      preds = BudJoin::natural_preds(@bud_instance, @origrels)
+      pairs(*preds, &blk)
+    end
+
+    # given a * expression over 2 collections, form all combinations of items
+    # that satisfy the predicates +preds+, and project only onto the attributes
+    # of the first collection
+    public
+    def lefts(*preds, &blk)
+      setup_preds(preds) unless preds.empty?
+      # given new preds, the state for the join will be different.  set it up again.
+      setup_state if self.class <= Bud::BudJoin
+      map{ |l,r| blk.nil? ? l : blk.call(l) }
+    end
+
+    # given a * expression over 2 collections, form all combinations of items
+    # that satisfy the predicates +preds+, and project only onto the attributes
+    # of the second item
+    public
+    def rights(*preds, &blk)
+      setup_preds(preds) unless preds.empty?
+      # given new preds, the state for the join will be different.  set it up again.
+      setup_state if self.class <= Bud::BudJoin
+      map{ |l,r| blk.nil? ? r : blk.call(r) }
+    end
+
+    # given a * expression over 2 collections, form all combos of items that
+    # satisfy +preds+, and for any item from the 1st collection that has no
+    # matches in the 2nd, nil-pad it and include it in the output.
+    public
+    def outer(*preds)
+      @origpreds = preds
+      @localpreds = disambiguate_preds(preds)
+      self.extend(Bud::BudOuterJoin)
+      map
+    end
+
+    # extract predicates on rellist[0] and recurse to right side with remainder
+    protected
+    def setup_preds(preds) # :nodoc: all
+      allpreds = disambiguate_preds(preds)
+      allpreds = canonicalize_localpreds(@rels, allpreds)
+      # check for refs to collections that aren't being joined, Issue 191
+      unless @rels[1].class <= Bud::BudJoin
+        tabnames = @rels.map{ |r| r.tabname }
+        allpreds.each do |p|
+          unless tabnames.include? p[0][0] and tabnames.include? p[1][0]
+            raise Bud::CompileError, "illegal predicate: collection #{} is not being joined"
+          end
+        end
+      end
+      @hashpreds = allpreds.reject { |p| p[0][0] != @rels[0].tabname or p[1][0] == @rels[0].tabname }
+      @localpreds = @hashpreds
+      @localpreds += allpreds.map do |p|
+        p if p[0][0] == p[1][0] and (p[0][0] == @rels[0].tabname or p[0][0] == @rels[1].tabname)
+      end.compact
+      otherpreds = allpreds - @localpreds
+      unless otherpreds.empty?
+        unless @rels[1].class <= Bud::BudJoin
+          raise Bud::CompileError, "join predicates don't match tables being joined: #{otherpreds.inspect}"
+        end
+        @rels[1].setup_preds(otherpreds)
+      end
+    end
+
+    protected
+    def disambiguate_preds(preds) # :nodoc: all
+      if preds.size == 1 and preds[0].class <= Hash
+        predarray = preds[0].map do |k,v|
+          if k.class != v.class
+            raise Bud::CompileError, "inconsistent attribute ref style #{k.inspect} => #{v.inspect}"
+          elsif k.class <= Array
+            [k,v]
+          elsif k.class <= Symbol
+            if @origrels and @origrels.length == 2
+              [find_attr_match(k, @origrels[0]), find_attr_match(v, @origrels[1])]
+            else
+              [find_attr_match(k), find_attr_match(v)]
+            end
+          else
+            raise Bud::CompileError, "invalid attribute ref in #{k.inspect} => #{v.inspect}"
+          end
+        end
+        return decomp_preds(*predarray)
+      else
+        return decomp_preds(*preds)
+      end
+    end
+
+    # find element in @origrels that contains this +aname+ method
+    # if +rel+ is non-nil, only check that collection.
+    # after found, return the result of invoking +aname+ from chosen collection
+    protected
+    def find_attr_match(aname, rel=nil) # :nodoc: all
+      dorels = (rel.nil? ? @origrels : [rel])
+      match = nil
+      dorels.each do |r|
+        match ||= r if r.respond_to?(aname)
+        if r.respond_to?(aname) and match != r
+          raise Bud::CompileError, "ambiguous attribute :#{aname} in both #{match.tabname} and #{r.tabname}"
+        end
+      end
+      if match.nil?
+        raise Bud::CompileError, "attribute :#{aname} not found in any of #{dorels.map{|t| t.tabname}.inspect}"
+      end
+      match.send(aname)
+    end
+
+    protected
+    def decomp_preds(*preds) # :nodoc:all
+      # decompose each pred into a binary pred
+      return nil if preds.empty? or preds == [nil]
+      newpreds = []
+      preds.each do |p|
+        p.each_with_index do |c, i|
+          newpreds << [p[i], p[i+1]] unless p[i+1].nil?
+        end
+      end
+      newpreds
+    end
+
+    protected
+    def canonicalize_localpreds(rel_list, preds) # :nodoc:all
+      retval = preds.map do |p|
+        p[1][0] == rel_list[0].tabname ? p.reverse : p
+      end
+    end
+
     public
     def each_from_sym(buf_syms, &block) # :nodoc: all
       buf_syms.each do |s|
@@ -154,8 +304,16 @@ module Bud
         # check remainder of the predicates
         @localpreds.each do |pred|
           next if skips.include? pred
-          r_offset, s_index, s_offset = join_offsets(pred)
-          if r[r_offset] != s[s_index][s_offset]
+          vals = []
+          (0..1).each do |i|
+            if pred[i][0] == @rels[0].tabname
+              vals[i] = r[pred[i][1] ]
+            else
+              ix, off = join_offset(pred[i])
+              vals[i] = s[ix][off]
+            end
+          end
+          if vals[0] != vals[1]
             retval = false
             break
           end
@@ -175,28 +333,27 @@ module Bud
     end
 
     private
-    # calculate the attribute position for the left table in the join ("left_offset")
-    # the right table may itself be a nested tuple from a join, so calculate
-    # the tuple offset ("right_subtuple") and the attribute position within it
-    # ("right_offset")
-    def join_offsets(pred)
-      right_entry = pred[1]
-      right_name, right_offset = right_entry[0], right_entry[1]
-      left_entry = pred[0]
-      left_name, left_offset = left_entry[0], left_entry[1]
-
-      # determine which subtuple of right collection contains the table
-      # referenced in RHS of pred.  note that right collection doesn't contain the
-      # first entry in rels, which is the left collection
-      right_subtuple = 0
+    # calculate the position for a field in the result of a join:
+    # the tuple offset ("subtuple") and the attribute position within it
+    # ("offset")
+    def join_offset(entry)
+      name, offset = entry[0], entry[1]
+
+      # determine which subtuple of the collection contains the table
+      # referenced in entry.  note that origrels[0] is a base table
+      # on the left of the join, hence we shouldn't be calling join_offset on it
+      subtuple = 0
+      if origrels[0].tabname == entry[0]
+        raise BudError, "join_offset called on the result of a single collection #{entry[0]}"
+      end
       origrels[1..origrels.length].each_with_index do |t,i|
-        if t.tabname == pred[1][0]
-          right_subtuple = i
+        if t.tabname == entry[0]
+          subtuple = i
           break
         end
       end
 
-      return left_offset, right_subtuple, right_offset
+      return subtuple, offset
     end
 
     def tick_hash_deltas
@@ -204,7 +361,7 @@ module Bud
       return if @hash_tables.nil?
       (0..1).each do |i|
         @hash_tables[i][:storage].merge!(@hash_tables[i][:delta]) do |k,l,r|
-            l+r
+          l+r
         end
         @hash_tables[i][:delta] = {}
       end
@@ -213,7 +370,9 @@ module Bud
     # semi-naive symmetric hash join on first predicate
     private
     def hash_join(left_sym, right_sym, &block)
-      left_offset, right_subtuple, right_offset = join_offsets(@localpreds.first)
+      # we know that a hashpred has been canonicalized with @rels[0] in left offset
+      left_offset = @hashpreds.first[0][1]
+      right_subtuple, right_offset = join_offset(@hashpreds.first[1])
 
       syms = [left_sym, right_sym]
 
@@ -249,25 +408,14 @@ module Bud
               left = s_tup; right = r
             end
             retval = left + right
-            yield retval if test_locals(left[0], right, @localpreds.first)
+            yield retval if test_locals(left[0], right, @hashpreds.first)
           end
         end
       end
     end
   end
 
-  class BudLeftJoin < BudJoin # :nodoc: all
-    def initialize(rellist, bud_instance, preds=nil)
-      raise(BudError, "Left Join only defined for two relations") unless rellist.length == 2
-      super(rellist, bud_instance, preds)
-      @origpreds = preds
-      preds.each do |k,v|
-        if k.class <= Array
-          raise Bud::CompileError, "in leftjoin, attribute refs must have style :col1 => :col2"
-        end
-      end
-    end
-
+  module BudOuterJoin
     public
     def each(&block) # :nodoc:all
       super(&block)
@@ -298,3 +446,4 @@ module Bud
     end
   end
 end
+

data/lib/bud/monkeypatch.rb

@@ -0,0 +1,84 @@
+# We monkeypatch Module to add support for Bloom state and code declarations.
+class Module
+  # import another module and assign to a qualifier symbol: <tt>import MyModule => :m</tt>
+  def import(spec)
+    raise Bud::CompileError unless (spec.class <= Hash and spec.length == 1)
+    mod, local_name = spec.first
+    raise Bud::CompileError unless (mod.class <= Module and local_name.class <= Symbol)
+
+    # To correctly expand qualified references to an imported module, we keep a
+    # table with the local bind names of all the modules imported by this
+    # module. To handle nested references (a.b.c.d etc.), the import table for
+    # module X points to X's own nested import table.
+    @bud_import_tbl ||= {}
+    child_tbl = mod.bud_import_table
+    raise Bud::CompileError if @bud_import_tbl.has_key? local_name
+    @bud_import_tbl[local_name] = child_tbl.clone # XXX: clone needed?
+
+    rewritten_mod_name = ModuleRewriter.do_import(self, mod, local_name)
+    self.module_eval "include #{rewritten_mod_name}"
+  end
+
+  # the block of Bloom collection declarations.  one per module.
+  def state(&block)
+    meth_name = Module.make_state_meth_name(self)
+    define_method(meth_name, &block)
+  end
+
+  # a ruby block to be run before timestep 1.  one per module.
+  def bootstrap(&block)
+    meth_name = "__bootstrap__#{Module.get_class_name(self)}".to_sym
+    define_method(meth_name, &block)
+  end
+
+  # bloom statements to be registered with Bud runtime.  optional +block_name+ 
+  # allows for multiple bloom blocks per module, and overriding
+  def bloom(block_name=nil, &block)
+    # If no block name was specified, generate a unique name
+    if block_name.nil?
+      @block_id ||= 0
+      block_name = "#{Module.get_class_name(self)}__#{@block_id.to_s}"
+      @block_id += 1
+    else
+      unless block_name.class <= Symbol
+        raise Bud::CompileError, "Bloom block names must be a symbol: #{block_name}"
+      end
+    end
+
+    # Note that we don't encode the module name ("self") into the name of the
+    # method. This allows named blocks to be overridden (via inheritance or
+    # mixin) in the same way as normal Ruby methods.
+    meth_name = "__bloom__#{block_name}"
+
+    # Don't allow duplicate named bloom blocks to be defined within a single
+    # module; this indicates a likely programmer error.
+    if instance_methods(false).include? meth_name
+      raise Bud::CompileError, "Duplicate named bloom block: '#{block_name}' in #{self}"
+    end
+    define_method(meth_name.to_sym, &block)
+  end
+
+  def bud_import_table() #:nodoc: all
+    @bud_import_tbl ||= {}
+    @bud_import_tbl
+  end
+
+  private
+  # Return a string with a version of the class name appropriate for embedding
+  # into a method name. Annoyingly, if you define class X nested inside
+  # class/module Y, X's class name is the string "Y::X". We don't want to define
+  # method names with semicolons in them, so just return "X" instead.
+  def self.get_class_name(klass)
+    klass.name.split("::").last
+  end
+
+  # State method blocks are named using an auto-incrementing counter. This is to
+  # ensure that we can rediscover the possible dependencies between these blocks
+  # after module import (see Bud#call_state_methods).
+  def self.make_state_meth_name(klass)
+    @state_meth_id ||= 0
+    r = "__state#{@state_meth_id}__#{Module.get_class_name(klass)}".to_sym
+    @state_meth_id += 1
+    return r
+  end
+end

data/lib/bud/rebl.rb

@@ -72,7 +72,7 @@ class ReblShell
   # LibRebl that is created; testcases call this directly.
   def self.setup
     Signal.trap("INT") {do_exit}
-    Signal.trap("TRAP") {do_exit}
+    Signal.trap("TERM") {do_exit}
 
     ipport = ARGV[0] ? ARGV[0].split(":") : []
     lib = LibRebl.new(*[(ipport[0] or "localhost"), (ipport[1] or 0)])
@@ -287,6 +287,13 @@ class LibRebl
       @rebl_class_inst.tables.merge!(@old_inst.tables.reject do |k,v|
                                        @@builtin_tables.include? k
                                      end)
+      @rebl_class_inst.channels.merge!(@old_inst.channels.reject do |k,v|
+                                         @@builtin_tables.include? k
+                                       end)
+      @rebl_class_inst.tc_tables.merge! @old_inst.tc_tables
+      @rebl_class_inst.dbm_tables.merge! @old_inst.dbm_tables
+      @rebl_class_inst.zk_tables.merge! @old_inst.zk_tables
+
       # Fix the bud instance pointers from copied tables.
       @rebl_class_inst.tables.values.each do |v|
         v.bud_instance = @rebl_class_inst