bud 0.9.4 → 0.9.9

data/lib/bud/executor/join.rb

@@ -1,24 +1,24 @@
 require 'bud/executor/elements'
-require 'set'
 
-$EMPTY = []
 module Bud
   class PushSHJoin < PushStatefulElement
     attr_reader :all_rels_below, :origpreds, :relnames, :keys, :localpreds
 
     def initialize(rellist, bud_instance, preds=nil) # :nodoc: all
       @rels = rellist
-      @relnames = @rels.map{|r| r.elem_name}
+      @relnames = @rels.map{|r| r.qualified_tabname}
       @cols = []
       @bud_instance = bud_instance
       @origpreds = preds
-      @localpreds = nil
+      @localpreds = []
       @selfjoins = []
+      @keys = []
+      @key_attnos = [[], []]
       @missing_keys = Set.new
 
       # if any elements on rellist are PushSHJoins, suck up their contents
       @all_rels_below = []
-      rellist.each do |r|
+      @rels.each do |r|
         if r.class <= PushSHJoin
           @all_rels_below += r.all_rels_below
           preds += r.origpreds
@@ -26,12 +26,13 @@ module Bud
           @all_rels_below << r
         end
       end
+      @left_is_array = @all_rels_below.length > 2
 
       # check for self-joins: we currently only handle 2 instances of the same
       # table per rule
       counts = @all_rels_below.reduce({}) do |memo, r|
-        memo[r.elem_name] ||= 0
-        memo[r.elem_name] += 1
+        memo[r.qualified_tabname] ||= 0
+        memo[r.qualified_tabname] += 1
         memo
       end
       counts.each do |name, cnt|
@@ -39,31 +40,12 @@ module Bud
         @selfjoins << name if cnt == 2
       end
 
-      # derive schema: one column for each table.
-      # duplicated inputs get distinguishing numeral
-      @cols = []
-      index = 0
-      retval = @all_rels_below.reduce({}) do |memo, r|
-        index += 1
-        memo[r.tabname.to_s] ||= 0
-        newstr = r.tabname.to_s + ((memo[r.tabname.to_s] > 0) ? ("_" + memo[r.tabname.to_s].to_s) : "")
-        @cols << newstr.to_sym
-        memo[r.tabname.to_s] += 1
-        memo
-      end
-
       setup_preds(preds) unless preds.empty?
       setup_state
 
       super(@tabname, @bud_instance, nil, @cols)
     end
 
-    public
-    def copy_on_write
-      @refcount -= 1
-      return Bud::PushSHJoin.new(@all_rels_below, @bud_instance, [])
-    end
-
     public
     def state_id # :nodoc: all
       object_id
@@ -77,7 +59,7 @@ module Bud
     private
     def setup_state
       sid = state_id
-      @tabname = ("(" + @all_rels_below.map{|r| r.tabname}.join('*') +"):"+sid.to_s).to_sym
+      @tabname = ("(" + @all_rels_below.map{|r| r.qualified_tabname}.join('*') +"):"+sid.to_s).to_sym
       @hash_tables = [{}, {}]
     end
 
@@ -87,25 +69,26 @@ module Bud
       # print "setting up preds for #{@relnames.inspect}(#{self.object_id}): "
       allpreds = disambiguate_preds(preds)
       allpreds = canonicalize_localpreds(@rels, allpreds)
-      # check for refs to collections that aren't being joined, Issue 191
+
+      # check for refs to collections that aren't being joined
       unless @rels[0].class <= Bud::PushSHJoin
-        tabnames = @rels.map{ |r| r.tabname }
         allpreds.each do |p|
-          unless tabnames.include? p[0][0]
+          unless @relnames.include? p[0][0]
             raise Bud::CompileError, "illegal predicate: collection #{p[0][0]} is not being joined"
           end
-          unless tabnames.include? p[1][0]
+          unless @relnames.include? p[1][0]
             raise Bud::CompileError, "illegal predicate: collection #{p[1][0]} is not being joined"
           end
         end
       end
+
       @localpreds = allpreds.reject do |p|
         # reject if it doesn't match the right (leaf node) of the join
         # or reject if it does match, but it can be evaluated by a lower join
-        # i.e. one that also has this table on the right (lead node)
-        p[1][0] != @rels[1].tabname \
-        or (p[0][0] != @rels[1].tabname \
-            and p[1][0] == @rels[1].tabname and @selfjoins.include? @rels[1].tabname)
+        # i.e. one that also has this table on the right (leaf node)
+        p[1][0] != @rels[1].qualified_tabname \
+        or (p[0][0] != @rels[1].qualified_tabname \
+            and p[1][0] == @rels[1].qualified_tabname and @selfjoins.include? @rels[1].qualified_tabname)
       end
 
       # only allow preds on the same table name if they're on a self-joined table
@@ -115,9 +98,9 @@ module Bud
         end
       end
 
-      @localpreds += allpreds.map do |p|
-        p if p[0][0] == p[1][0] and (p[1][0] == @rels[0].tabname or p[1][0] == @rels[1].tabname)
-      end.compact
+      @localpreds += allpreds.select do |p|
+        p[0][0] == p[1][0] and (p[1][0] == @rels[0].qualified_tabname or p[1][0] == @rels[1].qualified_tabname)
+      end
       otherpreds = allpreds - @localpreds
       unless otherpreds.empty?
         unless @rels[0].class <= Bud::PushSHJoin
@@ -126,29 +109,27 @@ module Bud
         @rels[0].setup_preds(otherpreds)
       end
 
-      if @localpreds.length > 0
-        @right_offset = @localpreds.first[1][1]
-        @left_subtuple, @left_offset = join_offset(@localpreds.first[0])
-        @keys = [[@left_subtuple, @left_offset], [1, @right_offset]]
-      else
-        @keys = []
+      @localpreds.each do |lp|
+        right_offset = lp[1][1]
+        left_subtuple, left_offset = join_offset(lp[0])
+        @keys << [[left_subtuple, left_offset], [1, right_offset]]
       end
+
+      # Optimize for a common case. When we're just fetching key values from
+      # an input tuple, lookup the column offsets we need to fetch for each
+      # input. This doesn't apply when we're computing the key for the left
+      # input and @left_is_array is true.
+      @key_attnos = []
+      @key_attnos[0] = @keys.map {|k| k[0][1]}
+      @key_attnos[1] = @keys.map {|k| k[1][1]}
     end
 
     public
     def invalidate_cache
       @rels.each_with_index do |source_elem, i|
         if source_elem.rescan
-          puts "#{tabname} rel:#{i}(#{source_elem.tabname}) invalidated" if $BUD_DEBUG
+          puts "#{qualified_tabname} rel:#{i}(#{source_elem.qualified_tabname}) invalidated" if $BUD_DEBUG
           @hash_tables[i] = {}
-          if i == 0
-            # Only if i == 0 because outer joins in Bloom are left outer joins.
-            # If i == 1, missing_keys will be corrected when items are populated
-            # in the rhs fork.
-            # XXX This is not modular. We are doing invalidation work for outer
-            # joins, which is part of a separate module PushSHOuterJoin.
-            @missing_keys.clear
-          end
         end
       end
     end
@@ -163,7 +144,7 @@ module Bud
       # referenced in entry.
       subtuple = 0
       all_rels_below[0..all_rels_below.length-1].each_with_index do |t,i|
-        if t.tabname == entry[0]
+        if t.qualified_tabname == name
           subtuple = i
           break
         end
@@ -181,7 +162,7 @@ module Bud
           elsif k.class <= Array
             [k,v]
           elsif k.class <= Symbol
-            if @all_rels_below and @all_rels_below.length == 2
+            if @all_rels_below.length == 2
               [find_attr_match(k, @all_rels_below[0]), find_attr_match(v, @all_rels_below[1])]
             else
               [find_attr_match(k), find_attr_match(v)]
@@ -204,20 +185,23 @@ module Bud
       dorels = (rel.nil? ? @all_rels_below : [rel])
       match = nil
       dorels.each do |r|
-        match ||= r if bud_instance.tables[r.elem_name].respond_to?(aname)
-        if bud_instance.tables[r.elem_name].respond_to?(aname) and match != r
-          raise Bud::CompileError, "ambiguous attribute :#{aname} in both #{match.tabname} and #{r.tabname}"
+        r_name = r.qualified_tabname
+        tbl = bud_instance.toplevel.tables[r_name]
+        match ||= r if tbl.respond_to?(aname)
+        if tbl.respond_to?(aname) and match != r
+          raise Bud::CompileError, "ambiguous attribute :#{aname} in both #{match.qualified_tabname} and #{r_name}"
         end
       end
       if match.nil?
-        raise Bud::CompileError, "attribute :#{aname} not found in any of #{dorels.map{|t| t.tabname}.inspect}"
+        rel_names = dorels.map{|t| t.qualified_tabname.to_s}.to_s
+        raise Bud::CompileError, "attribute :#{aname} not found in any of #{rel_names}"
       end
-      bud_instance.tables[match.elem_name].send(aname)
+      match.send(aname)
     end
 
+    # decompose each pred into a binary pred
     protected
     def decomp_preds(*preds) # :nodoc:all
-                             # decompose each pred into a binary pred
       return nil if preds.empty? or preds == [nil]
       newpreds = []
       preds.each do |p|
@@ -230,40 +214,11 @@ module Bud
 
     protected
     def canonicalize_localpreds(rel_list, preds) # :nodoc:all
-      retval = preds.map do |p|
-        # reverse if lhs is rel_list[1], *unless* it's a self-join!
-        (p[0][0] == rel_list[1].tabname and p[0][0] != p[1][0]) ? p.reverse : p
-      end
-    end
-
-    private
-    # right is a tuple
-    # left is a tuple or an array (combo) of joined tuples.
-    def test_locals(left, left_is_array, right, *skips)
-      retval = true
-      if (skips and @localpreds.length > skips.length)
-        # check remainder of the predicates
-        @localpreds.each do |pred|
-          # skip skips
-          next if (skips.include? pred)
-          # assumption of left-deep joins here
-          if pred[1][0] != @rels[1].tabname
-            raise Bud::Error, "expected rhs table to be #{@rels[1].tabname}, not #{pred[1][0]}"
-          end
-          rfield = right[pred[1][1]]
-          if left_is_array
-            ix, off = join_offset(pred[0])
-            lfield = left[ix][off]
-          else
-            lfield = left[pred[0][1]]
-          end
-          if lfield != rfield
-            retval = false
-            break
-          end
-        end
+      second_rel = rel_list[1].qualified_tabname
+      preds.map do |p|
+        # reverse if lhs is second_rel *unless* it's a self-join!
+        (p[0][0] == second_rel and p[0][0] != p[1][0]) ? p.reverse : p
       end
-      return retval
     end
 
     undef do_insert
@@ -276,30 +231,29 @@ module Bud
       # again if we didn't rescan now.
       replay_join if @rescan
 
-      if @selfjoins.include? source.elem_name
+      source_tbl = source.qualified_tabname
+      if @selfjoins.include? source_tbl
         offsets = []
-        @relnames.each_with_index{|r,i| offsets << i if r == source.elem_name}
+        @relnames.each_with_index{|r,i| offsets << i if r == source_tbl}
       else
-        offsets = [@relnames.index(source.elem_name)]
-      end
-      raise Bud::Error, "item #{item.inspect} inserted into join from unknown source #{source.elem_name}" if offsets == $EMPTY
-      offsets.each do |offset|
-        insert_item(item, offset)
+        offsets = [@relnames.index(source_tbl)]
       end
+
+      offsets.each {|offset| insert_item(item, offset)}
     end
 
     protected
     def insert_item(item, offset)
-      if @keys.nil? or @keys.empty?
-        the_key = nil
-      else
-        # assumes left-deep trees
-        if all_rels_below.length > 2 and offset == 0
-          the_key = item[@keys[0][0]][@keys[0][1]]
-        else
-          the_key = item[@keys[offset][1]]
+      # assumes left-deep trees
+      if @left_is_array and offset == 0
+        the_key = @keys.map do |k|
+          left_subtuple, left_offset = k.first
+          item[left_subtuple][left_offset]
         end
+      else
+        the_key = item.values_at(*@key_attnos[offset])
       end
+
       #build
       # puts "building #{item.inspect} into @source[#{offset}] on key #{the_key.inspect}"
       if (@hash_tables[offset][the_key] ||= Set.new).add? item
@@ -347,29 +301,42 @@ module Bud
           left = m
           right = item
         end
-        left_is_array = all_rels_below.length > 2
-        if @localpreds.nil? or @localpreds.length == 1 or test_locals(left, left_is_array, right, @localpreds.first)
-          result = left_is_array ? left + [right] : [left, right] # FIX: reduce arrays being created.
-          push_out(result)
-        end
+
+        # FIX: reduce arrays being created
+        result = @left_is_array ? left + [right] : [left, right]
+        push_out(result)
       end
     end
 
     ####
     # and now, the Bloom-facing methods
     # given a * expression over n collections, form all combinations of items
-    # subject to an array of predicates, pred
-    # currently supports two options for equijoin predicates:
+    # subject to an array of predicates, +preds+.
+    # currently supports two syntax options for equijoin predicates:
     #    general form: an array of arrays capturing a conjunction of equiv. classes
     #          [[table1.col1, table2.col2, table3.col3], [table1.col2, table2.col3]]
     #    common form: a hash capturing equality of a column on left with one on right.
     #          :col1 => :col2  (same as  lefttable.col1 => righttable.col2)
     public
     def pairs(*preds, &blk)
-      ## XXX Need to do this for all the join modifiers
-      unless @refcount == 1
-        return self.copy_on_write.pairs(preds, blk)
+      if @cols.nil?
+        # derive schema if needed: one column for each table.  duplicated inputs
+        # get distinguishing numeral.
+        #
+        # XXX: actually, this seems completely bogus. The schema for the output
+        # of the join should depend on the join's *targetlist*.
+        @cols = []
+        retval = @all_rels_below.reduce({}) do |memo, r|
+          r_name = r.qualified_tabname.to_s
+          memo[r_name] ||= 0
+          newstr = r_name + (memo[r_name] > 0 ? "_#{memo[r_name]}" : "")
+          @cols << newstr.to_sym
+          memo[r_name] += 1
+          memo
+        end
+        setup_accessors
       end
+
       @origpreds = preds
       setup_preds(preds) unless preds.empty?
       # given new preds, the state for the join will be different.  set it up again.
@@ -383,25 +350,32 @@ module Bud
     # matches in the 2nd, nil-pad it and include it in the output.
     public
     def outer(*preds, &blk)
+      if @all_rels_below.length > 2
+        raise Bud::Error, "outer joins cannot be used with more than 2 join relations"
+      end
       pairs(*preds, &blk)
       self.extend(Bud::PushSHOuterJoin)
     end
 
     public
-    def rights(*preds, &blk)
-      @cols = blk.nil? ? @bud_instance.tables[@rels[1].tabname].cols : nil
-      setup_accessors if blk.nil?
+    def lefts(*preds, &blk)
+      if blk.nil?
+        @cols = @bud_instance.toplevel.tables[@rels[0].qualified_tabname].cols
+        setup_accessors
+      end
       pairs(*preds) do |x,y|
-        blk.nil? ? y : blk.call(y)
+        blk.nil? ? x : blk.call(x)
       end
     end
 
     public
-    def lefts(*preds, &blk)
-      @cols = blk.nil? ? @bud_instance.tables[@rels[0].tabname].cols : nil
-      setup_accessors if blk.nil?
+    def rights(*preds, &blk)
+      if blk.nil?
+        @cols = @bud_instance.toplevel.tables[@rels[1].qualified_tabname].cols
+        setup_accessors
+      end
       pairs(*preds) do |x,y|
-        blk.nil? ? x : blk.call(x)
+        blk.nil? ? y : blk.call(y)
       end
     end
 
@@ -432,13 +406,13 @@ module Bud
     public
     def flatten(*preds, &blk)
       if blk.nil?
-        @cols = dupfree_schema(@bud_instance.tables[@cols[0]].cols + @bud_instance.tables[@cols[1]].cols)
+        @cols = dupfree_schema(@rels[0].cols + @rels[1].cols)
       else
         @cols = []
       end
       setup_accessors
       pairs(*preds) do |x,y|
-        blk.nil? ? x.to_a + y.to_a : blk.call(x.to_a + y.to_a)
+        blk.nil? ? x + y : blk.call(x + y)
       end
     end
 
@@ -468,17 +442,11 @@ module Bud
   end
 
   module PushSHOuterJoin
+    # XXX: duplicates code from PushSHJoin
     private
     def insert_item(item, offset)
-      if @keys.nil? or @keys.empty?
-        the_key = nil
-      else
-        if all_rels_below.length > 2 and offset == 1
-          the_key = item[@keys[1][0]][@keys[1][1]]
-        else
-          the_key = item[@keys[offset][1]]
-        end
-      end
+      the_key = item.values_at(*@key_attnos[offset])
+
       #build
       # puts "building #{item.inspect} into @source[#{offset}] on key #{the_key.inspect}"
       if (@hash_tables[offset][the_key] ||= Set.new).add? item
@@ -489,7 +457,8 @@ module Bud
         if the_matches.nil? and offset == 0 # only doing Left Outer Join right now
           @missing_keys << the_key
         else
-          @missing_keys.delete(the_key) # no longer missing no matter which side this tuple is
+          # no longer missing no matter which side this tuple is
+          @missing_keys.delete(the_key)
           process_matches(item, the_matches, offset) unless the_matches.nil?
         end
       end
@@ -508,12 +477,22 @@ module Bud
 
     private
     def push_missing
+      left_hash = @hash_tables[0]
+      null_tuple = @rels[1].null_tuple
       @missing_keys.each do |key|
-        @hash_tables[0][key].each do |t|
-          push_out([t, @rels[1].null_tuple])
+        left_hash[key].each do |t|
+          push_out([t, null_tuple])
         end
       end
     end
+
+    public
+    def invalidate_cache
+      super
+      # Only if need to check left join rel because outer joins in Bloom are
+      # left outer joins.
+      @missing_keys.clear if @rels.first.rescan
+    end
   end
 
 
@@ -527,17 +506,18 @@ module Bud
   # first flush, at which point we are sure to have seen all the t-side tuples
   # in this tick.
   class PushNotIn < PushStatefulElement
-    def initialize(rellist, bud_instance, preds=nil, &blk) # :nodoc: all
+    def initialize(rellist, bud_instance, preds, &blk) # :nodoc: all
       @lhs, @rhs = rellist
       @lhs_keycols = nil
       @rhs_keycols = nil
-      name_in = "#{@lhs.tabname}_notin_#{@rhs.tabname}"
-      super(name_in, bud_instance)
+      name_in = "#{@lhs.qualified_tabname}_notin_#{@rhs.qualified_tabname}".to_sym
+      super(name_in, bud_instance, nil, @lhs.schema)
       setup_preds(preds) unless preds.empty?
       @rhs_rcvd = false
       @hash_tables = [{},{}]
       if @lhs_keycols.nil? and blk.nil?
-        # pointwise comparison. Could use zip, but it creates an array for each field pair
+        # Pointwise comparison. Could use zip, but it creates an array for each
+        # field pair.
         blk = lambda {|lhs, rhs|
           lhs.to_a == rhs.to_a
         }
@@ -547,7 +527,7 @@ module Bud
 
     def setup_preds(preds)
       # This is simpler than PushSHJoin's setup_preds, because notin is a binary
-      # operator where both lhs and rhs are collections.  preds an array of
+      # operator where both lhs and rhs are collections. preds is an array of
       # hash_pairs. For now assume that the attributes are in the same order as
       # the tables.
       @lhs_keycols, @rhs_keycols = preds.reduce([[], []]) do |memo, item|
@@ -559,21 +539,25 @@ module Bud
         memo
       end
     end
+
     def find_col(colspec, rel)
-      if colspec.is_a? Symbol
+      case colspec
+      when Symbol
+        unless rel.respond_to? colspec
+          raise Bud::Error, "attribute :#{colspec} not found in #{rel.qualified_tabname}"
+        end
         col_desc = rel.send(colspec)
-        raise Bud::Error, "unknown column #{colspec} in #{@rel.tabname}" if col_desc.nil?
-      elsif colspec.is_a? Array
+      when Array
         col_desc = colspec
       else
         raise Bud::Error, "symbol or column spec expected. Got #{colspec}"
       end
-      col_desc[1] # col_desc is of the form [tabname, colnum, colname]
+      col_desc[1] # col_desc is of the form [tabname, colnum, colname, seqno]
     end
 
     def get_key(item, offset)
-      keycols = offset == 0 ? @lhs_keycols : @rhs_keycols
-      keycols.nil? ? $EMPTY : keycols.map{|col| item[col]}
+      keycols = (offset == 0 ? @lhs_keycols : @rhs_keycols)
+      keycols.nil? ? [] : item.values_at(*keycols)
     end
 
     public
@@ -582,11 +566,21 @@ module Bud
     end
 
     def insert(item, source)
-      offset = source == @lhs ? 0 : 1
+      if source == @lhs && source == @rhs       # Self join
+        do_insert(item, 0)
+        do_insert(item, 1)
+      else
+        offset = source == @lhs ? 0 : 1
+        do_insert(item, offset)
+      end
+    end
+
+    def do_insert(item, offset)
       key = get_key(item, offset)
       (@hash_tables[offset][key] ||= Set.new).add item
       if @rhs_rcvd and offset == 0
-        push_lhs(key, item)
+        rhs_values = @hash_tables[1][key]
+        process_match(item, rhs_values)
       end
     end
 
@@ -596,19 +590,15 @@ module Bud
       # growing any more, until the next tick.
       unless @rhs_rcvd
         @rhs_rcvd = true
+        rhs_hash = @hash_tables[1]
         @hash_tables[0].each do |key,values|
-          values.each {|item| push_lhs(key, item)}
+          rhs_values = rhs_hash[key]
+          values.each {|item| process_match(item, rhs_values)}
         end
       end
     end
 
-    def push_lhs(key, lhs_item)
-      rhs_values = @hash_tables[1][key]
-      process_match(lhs_item, rhs_values)
-    end
-
     def process_match(lhs_item, rhs_values)
-      exclude = true
       if rhs_values.nil?
         # no corresponding rhs. Include in output
         exclude = false
@@ -616,33 +606,24 @@ module Bud
         # for any lhs * rhs pair, if block returns true, do not push lhs. lhs is pushed
         # only if there is no match (anti-join)
         exclude = rhs_values.any?{|rhs_item| @blk.call(lhs_item, rhs_item)}
+      else
+        exclude = true
       end
-      unless exclude
-        push_out(lhs_item)
-      end
-    end
 
-    public
-    def push_out(item)
-      @outputs.each do |ou|
-        if ou.class <= Bud::PushElement
-          ou.insert(item, self)
-        elsif ou.class <= Bud::BudCollection
-          ou.do_insert(item, ou.new_delta)
-        else
-          raise Bud::Error, "expected either a PushElement or a BudCollection"
-        end
-      end
-      # for all the following, o is a BudCollection
-      @deletes.each{|o| o.pending_delete([item])}
-      @delete_keys.each{|o| o.pending_delete_keys([item])}
-      @pendings.each{|o| o.pending_merge([item])}
+      push_out(lhs_item, false) unless exclude
     end
 
     def invalidate_cache
-      puts "#{self.class}/#{self.tabname} invalidated" if $BUD_DEBUG
-      @hash_tables = [{},{}]
-      @rhs_rcvd = false
+      raise Bud::Error if @rhs_rcvd     # sanity check; should already be reset
+
+      if @lhs.rescan
+        puts "#{tabname} rel:#{@lhs.qualified_tabname} invalidated" if $BUD_DEBUG
+        @hash_tables[0] = {}
+      end
+      if @rhs.rescan
+        puts "#{tabname} rel:#{@rhs.qualified_tabname} invalidated" if $BUD_DEBUG
+        @hash_tables[1] = {}
+      end
     end
 
     def stratum_end