bud 0.0.3 → 0.0.4

data/lib/bud/collections.rb

@@ -1,7 +1,7 @@
 require 'msgpack'
 
 module Bud
-  ######## 
+  ########
   #--
   # the collection types
   # each collection is partitioned into 4:
@@ -16,7 +16,7 @@ module Bud
 
     attr_accessor :bud_instance, :locspec_idx # :nodoc: all
     attr_reader :schema, :tabname # :nodoc: all
-    attr_reader :storage, :delta, :new_delta # :nodoc: all
+    attr_reader :storage, :delta, :new_delta, :pending # :nodoc: all
 
     def initialize(name, bud_instance, given_schema=nil, defer_schema=false) # :nodoc: all
       @tabname = name
@@ -27,7 +27,6 @@ module Bud
 
     private
     def init_buffers
-      @sealed = false
       init_storage
       init_pending
       init_deltas
@@ -70,7 +69,7 @@ module Bud
       return [schema, key_cols]
     end
 
-    public 
+    public
     def clone_empty #:nodoc: all
       self.class.new(tabname, bud_instance, @given_schema)
     end
@@ -80,7 +79,7 @@ module Bud
     def key_cols
       @key_cols
     end
-    
+
     # subset of the schema (i.e. an array of attribute names) that is not in the key
     public
     def val_cols # :nodoc: all
@@ -155,25 +154,21 @@ module Bud
     public
     def pro(&blk)
       if @bud_instance.stratum_first_iter
-        return map(&blk) 
+        return map(&blk)
       else
-        if @delta.empty?
-          return []
-        else
-          retval = []
-          each_from([@delta]) do |t|
-            newitem = blk.call(t)
-            retval << newitem unless newitem.nil?
-          end
-          return retval
+        retval = []
+        each_from([@delta]) do |t|
+          newitem = blk.call(t)
+          retval << newitem unless newitem.nil?
         end
-      end    
+        return retval
+      end
     end
 
     # By default, all tuples in any rhs are in storage or delta. Tuples in
     # new_delta will get transitioned to delta in the next iteration of the
     # evaluator (but within the current time tick).
-    public 
+    public
     def each(&block) # :nodoc: all
       each_from([@storage, @delta], &block)
     end
@@ -223,7 +218,8 @@ module Bud
     # checks for key +k+ in the key columns
     public
     def has_key?(k)
-      return false if k.nil? or k.empty? or self[k].nil?
+      check_enumerable(k)
+      return false if k.nil? or self[k].nil?
       return true
     end
 
@@ -232,7 +228,8 @@ module Bud
     def [](k)
       # assumes that key is in storage or delta, but not both
       # is this enforced in do_insert?
-      return @storage[k].nil? ? @delta[k] : @storage[k]
+      t = @storage[k]
+      return t.nil? ? @delta[k] : t
     end
 
     # checks for +item+ in the collection
@@ -257,15 +254,18 @@ module Bud
     end
 
     private
-    def raise_pk_error(new, old)
+    def raise_pk_error(new_guy, old)
       keycols = key_cols.map{|k| old[schema.index(k)]}
-      raise KeyConstraintError, "Key conflict inserting #{old.inspect} into \"#{tabname}\": existing tuple #{new.inspect}, key_cols = #{keycols.inspect}"
+      raise KeyConstraintError, "Key conflict inserting #{new_guy.inspect} into \"#{tabname}\": existing tuple #{old.inspect}, key_cols = #{keycols.inspect}"
     end
 
     private
     def prep_tuple(o)
       unless o.respond_to?(:length) and o.respond_to?(:[])
-        raise BudTypeError, "non-indexable type inserted into BudCollection #{self.tabname}: #{o.inspect}"
+        raise BudTypeError, "non-indexable type inserted into \"#{tabname}\": #{o.inspect}"
+      end
+      if o.class <= String
+        raise BudTypeError, "String value used as a fact inserted into \"#{tabname}\": #{o.inspect}"
       end
 
       if o.length < schema.length then
@@ -308,8 +308,8 @@ module Bud
 
     private
     def check_enumerable(o)
-      unless (o.nil? or o.class < Enumerable) and o.respond_to? 'each'
-        raise BudTypeError, "Attempt to merge non-enumerable type into BudCollection"
+      unless o.nil? or o.class < Enumerable
+        raise BudTypeError, "Collection #{tabname} expected Enumerable value, not #{o.inspect} (class = #{o.class})"
       end
     end
 
@@ -318,10 +318,15 @@ module Bud
     private
     def establish_schema(o)
       # use o's schema if available
-      deduce_schema(o) if @schema.nil?
-      # else use arity of first tuple of o
-      fit_schema(o.first.size) if @schema.nil? and not o.first.nil?
-      return @schema
+      deduce_schema(o)
+      # else use arity of first non-nil tuple of o
+      if @schema.nil?
+        o.each do |t|
+          next if t.nil?
+          fit_schema(t.size)
+          break
+        end
+      end
     end
 
     # Copy over the schema from +o+ if available
@@ -331,8 +336,7 @@ module Bud
         # must have been initialized with defer_schema==true.  take schema from rhs
         init_schema(o.schema)
       end
-      # returns old state of @schema (nil) if nothing available
-      return @schema
+      # if nothing available, leave @schema unchanged
     end
 
     # manufacture schema of the form [:c0, :c1, ...] with width = +arity+
@@ -340,24 +344,36 @@ module Bud
     def fit_schema(arity)
       # rhs is schemaless.  create schema from first tuple merged
       init_schema((0..arity-1).map{|indx| ("c"+indx.to_s).to_sym})
-      return @schema
+    end
+
+    private
+    def include_any_buf?(t, key_vals)
+      bufs = [self, @delta, @new_delta]
+      bufs.each do |b|
+        old = b[key_vals]
+        next if old.nil?
+        if old != t
+          raise_pk_error(t, old)
+        else
+          return true
+        end
+      end
+      return false
     end
 
     public
     def merge(o, buf=@new_delta) # :nodoc: all
-      check_enumerable(o)
-      establish_schema(o) if @schema.nil?
-
-      delta = o.map do |i|
-        next if i.nil? or i == []
-        i = prep_tuple(i)
-        key_vals = @key_colnums.map{|k| i[k]}
-        if (old = self[key_vals])
-          raise_pk_error(i, old) if old != i
-        elsif (oldnew = self.new_delta[key_vals])
-          raise_pk_error(i, oldnew) if oldnew != i
-        else
-          buf[key_vals] = tuple_accessors(i)
+      unless o.nil?
+        o = o.uniq.compact if o.respond_to?(:uniq)
+        check_enumerable(o)
+        establish_schema(o) if @schema.nil?
+
+        # it's a pity that we are massaging the tuples that already exist in the head
+        o.each do |t|
+          next if t.nil? or t == []
+          t = prep_tuple(t)
+          key_vals = @key_colnums.map{|k| t[k]}
+          buf[key_vals] = tuple_accessors(t) unless include_any_buf?(t, key_vals)
         end
       end
       return self
@@ -365,7 +381,7 @@ module Bud
 
     public
     # instantaneously merge items from collection +o+ into +buf+
-    def <=(collection) 
+    def <=(collection)
       merge(collection)
     end
 
@@ -373,7 +389,7 @@ module Bud
     public
     def pending_merge(o) # :nodoc: all
       check_enumerable(o)
-      deduce_schema(o)
+      establish_schema(o) if @schema.nil?
 
       o.each {|i| do_insert(i, @pending)}
       return self
@@ -395,7 +411,7 @@ module Bud
     end
 
     # move deltas to storage, and new_deltas to deltas.
-    public 
+    public
     def tick_deltas # :nodoc: all
       # assertion: intersect(@storage, @delta) == nil
       @storage.merge!(@delta)
@@ -421,7 +437,7 @@ module Bud
         return []
       end
     end
-      
+
 
     # a generalization of argmin/argmax to arbitrary exemplary aggregates.
     # for each distinct value in the grouping key columns, return the item in that group
@@ -447,30 +463,42 @@ module Bud
         if memo[pkey_cols].nil?
           memo[pkey_cols] = {:agg=>agg.send(:init, p[colnum]), :tups => [p]}
         else
-          newval = agg.send(:trans, memo[pkey_cols][:agg], p[colnum])
-          if memo[pkey_cols][:agg] == newval
-            if agg.send(:tie, memo[pkey_cols][:agg], p[colnum])
-              memo[pkey_cols][:tups] << p
-            end
-          else
-            memo[pkey_cols] = {:agg=>newval, :tups=>[p]}
+          memo[pkey_cols][:agg], argflag = \
+             agg.send(:trans, memo[pkey_cols][:agg], p[colnum])
+          if argflag == :keep or agg.send(:tie, memo[pkey_cols][:agg], p[colnum])
+            memo[pkey_cols][:tups] << p 
+          elsif argflag == :replace
+            memo[pkey_cols][:tups] = [p]
+          elsif argflag.class <= Array and argflag[0] == :delete
+            memo[pkey_cols][:tups] -= argflag[1..-1] 
           end
         end
         memo
       end
 
+      # now we need to finalize the agg per group
+      finalaggs = {}
       finals = []
-      outs = tups.each_value do |t|
-        ties = t[:tups].map do |tie|
-          finals << tie
+      tups.each do |k,v|
+        finalaggs[k] = agg.send(:final, v[:agg])
+      end
+      
+      # and winnow the tups to match
+      finalaggs.each do |k,v|
+        tups[k][:tups].each do |t|
+          finals << t if (t[colnum] == v)
         end
       end
 
-      # merge directly into retval.storage, so that the temp tuples get picked up
-      # by the lhs of the rule
-      retval = BudScratch.new('argagg_temp', bud_instance, @given_schema)
-      retval.uniquify_tabname
-      retval.merge(finals, retval.storage)
+      if block_given?
+        finals.map{|r| yield r}
+      else
+        # merge directly into retval.storage, so that the temp tuples get picked up
+        # by the lhs of the rule
+        retval = BudScratch.new('argagg_temp', bud_instance, @given_schema)
+        retval.uniquify_tabname
+        retval.merge(finals, retval.storage)
+      end
     end
 
     # for each distinct value in the grouping key columns, return the item in that group
@@ -487,14 +515,29 @@ module Bud
       argagg(:max, gbkey_cols, col)
     end
 
+    private
+    def wrap_map(j, &blk)
+      if blk.nil?
+        return j
+      else
+        return j.map(&blk)
+      end
+    end
+
+    def join(collections, *preds, &blk)
+      # since joins are stateful, we want to allocate them once and store in this Bud instance
+      # we ID them on their tablenames, preds, and block
+      return wrap_map(BudJoin.new(collections, @bud_instance, preds), &blk)
+    end
+
     # form a collection containing all pairs of items in +self+ and items in
     # +collection+
     public
     def *(collection)
-      bud_instance.join([self, collection])
+      join([self, collection])
     end
 
-    # SQL-style grouping.  first argument is an array of attributes to group by.  
+    # SQL-style grouping.  first argument is an array of attributes to group by.
     # Followed by a variable-length list of aggregates over attributes (e.g. +min(:x)+)
     # Attributes can be referenced as symbols, or as +collection_name.attribute_name+
     public
@@ -514,21 +557,24 @@ module Bud
       aggcolsdups.each_with_index do |n, i|
         aggcols << "#{n.downcase}_#{i}".to_sym
       end
+      aggpairs = aggpairs.map do |ap|
+        if ap[1].class == Symbol
+          colnum = ap[1].nil? ? nil : self.send(ap[1].to_s)[1]
+        else
+          colnum = ap[1].nil? ? nil : ap[1][1]
+        end
+        [ap[0], colnum]
+      end
       tups = agg_in.inject({}) do |memo, p|
         pkey_cols = keynames.map{|n| p.send(n)}
         memo[pkey_cols] = [] if memo[pkey_cols].nil?
         aggpairs.each_with_index do |ap, i|
           agg = ap[0]
-          if ap[1].class == Symbol
-            colnum = ap[1].nil? ? nil : self.send(ap[1].to_s)[1]
-          else
-            colnum = ap[1].nil? ? nil : ap[1][1]
-          end
-          colval = colnum.nil? ? nil : p[colnum]
+          colval = ap[1].nil? ? nil : p[ap[1]]
           if memo[pkey_cols][i].nil?
             memo[pkey_cols][i] = agg.send(:init, colval)
           else
-            memo[pkey_cols][i] = agg.send(:trans, memo[pkey_cols][i], colval)
+            memo[pkey_cols][i], ignore = agg.send(:trans, memo[pkey_cols][i], colval)
           end
         end
         memo
@@ -559,132 +605,6 @@ module Bud
 
     alias reduce inject
 
-    # methods that work on nested collections (resulting from joins)
-
-
-    # given a * expression over n collections, form all combinations of items
-    # subject to an array of predicates, pred
-    # currently supports two options for equijoin predicates:
-        #    general form: an array of arrays capturing a conjunction of equiv. classes
-        #          [[table1.col1, table2.col2, table3.col3], [table1.col2, table2.col3]]
-        #    common form: a hash capturing equality of a column on left with one on right.
-        #          :col1 => :col2  (same as  lefttable.col1 => righttable.col2)
-    public
-    def pairs(*preds, &blk)
-      setup_preds(preds) unless (preds.nil? or preds.empty?)
-      # given new preds, the state for the join will be different.  set it up again.
-      setup_state if self.class <= Bud::BudJoin
-      blk.nil? ? self : map(&blk)
-    end
-
-    alias combos pairs
-
-    # the natural join: given a * expression over 2 collections, form all
-    # combinations of items that have the same values in matching fiels
-    public
-    def matches(&blk)
-      preds = BudJoin::natural_preds(@bud_instance, @rels)
-      pairs(*preds, &blk)
-    end
-
-    # given a * expression over 2 collections, form all
-    # combinations of items that satisfy the predicates +preds+,
-    # and project only onto the attributes of the first collection
-    public
-    def lefts(*preds)
-      @localpreds = disambiguate_preds(preds)
-      map{ |l,r| l }
-    end
-
-    # given a * expression over 2 collections, form all
-    # combinations of items that satisfy the predicates +preds+,
-    # and project only onto the attributes of the second item
-    public
-    def rights(*preds)
-      @localpreds = disambiguate_preds(preds)
-      map{ |l,r| r }
-    end
-
-    # extract predicates on rellist[0] and recurse to right side with remainder
-    protected
-    def setup_preds(preds) # :nodoc: all
-      allpreds = disambiguate_preds(preds)
-      allpreds = canonicalize_localpreds(@rels, allpreds)
-      @localpreds = allpreds.reject { |p|  p[0][0] != @rels[0].tabname }
-      otherpreds = allpreds - @localpreds
-      otherpreds = nil if otherpreds.empty?
-      unless otherpreds.nil?
-        unless @rels[1].class <= Bud::BudJoin
-          raise BudError, "join predicates don't match tables being joined: #{otherpreds.inspect}"
-        end
-        @rels[1].setup_preds(otherpreds)
-      end
-    end
-
-    protected
-    def disambiguate_preds(preds) # :nodoc: all
-      if preds.size == 1 and preds[0].class <= Hash
-        predarray = preds[0].map do |k,v|
-          if k.class != v.class
-            raise Bud::CompileError, "inconsistent attribute ref style #{k.inspect} => #{v.inspect}"
-          elsif k.class <= Array
-            [k,v]
-          elsif k.class <= Symbol
-            if @origrels and @origrels.length == 2
-              [find_attr_match(k,@origrels[0]), find_attr_match(v,@origrels[1])]
-            else
-              [find_attr_match(k), find_attr_match(v)]
-            end
-          else
-            raise Bud::CompileError, "invalid attribute ref in #{k.inspect} => #{v.inspect}"
-          end
-        end
-        return decomp_preds(*predarray)
-      else
-        return decomp_preds(*preds)
-      end
-    end
-
-    # find element in @origrels that contains this aname method
-    # if 2nd arg is non-nil, only check that collection.
-    # after found, return the result of invoking aname from chosen collection
-    protected
-    def find_attr_match(aname, rel=nil) # :nodoc: all
-      dorels = (rel.nil? ? @origrels : [rel])
-      match = nil
-      dorels.each do |r|
-        match ||= r if r.respond_to?(aname)
-        if r.respond_to?(aname) and match != r
-          raise Bud::CompileError, "ambiguous attribute :#{aname} in both #{match.tabname} and #{r.tabname}"
-        end
-      end
-      if match.nil?
-        raise Bud::CompileError, "attribute :#{aname} not found in any of #{dorels.map{|t| t.tabname}.inspect}"
-      end
-      match.send(aname)
-    end
-
-    protected
-    def decomp_preds(*preds) # :nodoc:all
-      # decompose each pred into a binary pred
-      return nil if preds.nil? or preds.empty? or preds == [nil]
-      newpreds = []
-      preds.each do |p|
-        p.each_with_index do |c, i|
-          newpreds << [p[i], p[i+1]] unless p[i+1].nil?
-        end
-      end
-      newpreds
-    end
-
-    protected
-    def canonicalize_localpreds(rel_list, preds) # :nodoc:all
-      return if preds.nil?
-      retval = preds.map do |p|
-        p[1][0] == rel_list[0].tabname ? p.reverse : p
-      end
-    end
-    
     public
     def uniquify_tabname # :nodoc: all
       # just append current number of microseconds
@@ -701,18 +621,25 @@ module Bud
   class BudChannel < BudCollection
     attr_reader :locspec_idx # :nodoc: all
 
-    def initialize(name, bud_instance, given_schema=nil) # :nodoc: all
+    def initialize(name, bud_instance, given_schema=nil, loopback=false) # :nodoc: all
       given_schema ||= [:@address, :val]
-      the_schema, the_key_cols = parse_schema(given_schema)
-      the_val_cols = the_schema - the_key_cols
-      @locspec_idx = remove_at_sign!(the_key_cols)
-      @locspec_idx = remove_at_sign!(the_schema) if @locspec_idx.nil?
-      # If @locspec_idx is still nil, this is a loopback channel
-
-      # We mutate the hash key above, so we need to recreate the hash
-      # XXX: ugh, hacky
-      if given_schema.respond_to? :keys
-        given_schema = {the_key_cols => the_val_cols}
+      @is_loopback = loopback
+      @locspec_idx = nil
+
+      unless @is_loopback
+        the_schema, the_key_cols = parse_schema(given_schema)
+        the_val_cols = the_schema - the_key_cols
+        @locspec_idx = remove_at_sign!(the_key_cols)
+        @locspec_idx = remove_at_sign!(the_schema) if @locspec_idx.nil?
+        if @locspec_idx.nil?
+          raise BudError, "Missing location specifier for channel '#{name}'"
+        end
+
+        # We mutate the hash key above, so we need to recreate the hash
+        # XXX: ugh, hacky
+        if given_schema.respond_to? :keys
+          given_schema = {the_key_cols => the_val_cols}
+        end
       end
 
       super(name, bud_instance, given_schema)
@@ -728,37 +655,38 @@ module Bud
     end
 
     private
-    def split_locspec(l)
-      lsplit = l.split(':')
-      lsplit[1] = lsplit[1].to_i
-      return lsplit
+    def split_locspec(t, idx)
+      begin
+        lsplit = t[idx].split(':')
+        lsplit[1] = lsplit[1].to_i
+        return lsplit
+      rescue Exception => e
+        raise BudError, "Illegal location specifier in tuple #{t.inspect} for channel \"#{tabname}\": #{e.to_s}"
+      end
     end
 
     public
     def clone_empty
-      retval = super
-      retval.locspec_idx = @locspec_idx
-      retval
+      self.class.new(tabname, bud_instance, @given_schema, @is_loopback)
     end
 
-    public 
+    public
     def tick # :nodoc: all
-      @sealed = false
       @storage = {}
       # Note that we do not clear @pending here: if the user inserted into the
       # channel manually (e.g., via <~ from inside a sync_do block), we send the
       # message at the end of the current tick.
     end
 
-    public 
+    public
     def flush # :nodoc: all
       ip = @bud_instance.ip
       port = @bud_instance.port
       each_from([@pending]) do |t|
-        if @locspec_idx.nil?
+        if @is_loopback
           the_locspec = [ip, port]
         else
-          the_locspec = split_locspec(t[@locspec_idx])
+          the_locspec = split_locspec(t, @locspec_idx)
           raise BudError, "'#{t[@locspec_idx]}', channel '#{@tabname}'" if the_locspec[0].nil? or the_locspec[1].nil? or the_locspec[0] == '' or the_locspec[1] == ''
         end
         @bud_instance.dsock.send_datagram([@tabname, t].to_msgpack, the_locspec[0], the_locspec[1])
@@ -769,6 +697,8 @@ module Bud
     public
     # project to the non-address fields
     def payloads
+      return self.pro if @is_loopback
+
       if schema.size > 2
         # bundle up each tuple's non-locspec fields into an array
         retval = case @locspec_idx
@@ -792,7 +722,7 @@ module Bud
     end
 
     undef merge
-    
+
     def <=(o)
       raise BudError, "Illegal use of <= with channel '#{@tabname}' on left"
     end
@@ -804,15 +734,18 @@ module Bud
       @prompt = prompt
     end
 
-    public 
+    public
     def start_stdin_reader # :nodoc: all
       # XXX: Ugly hack. Rather than sending terminal data to EM via UDP,
       # we should add the terminal file descriptor to the EM event loop.
       @reader = Thread.new do
         begin
           while true
-            $stdout.print("#{tabname} > ") if @prompt
-            s = $stdin.gets
+            out_io = get_out_io
+            out_io.print("#{tabname} > ") if @prompt
+
+            in_io = @bud_instance.options[:stdin]
+            s = in_io.gets
             break if s.nil? # Hit EOF
             s = s.chomp if s
             tup = [s]
@@ -834,9 +767,10 @@ module Bud
 
     public
     def flush #:nodoc: all
+      out_io = get_out_io
       @pending.each do |p|
-        $stdout.puts p[0]
-        $stdout.flush
+        out_io.puts p[0]
+        out_io.flush
       end
       @pending = {}
     end
@@ -857,9 +791,35 @@ module Bud
     superator "<~" do |o|
       pending_merge(o)
     end
+
+    private
+    def get_out_io
+      rv = @bud_instance.options[:stdout]
+      rv ||= $stdout
+      rv
+    end
   end
 
   class BudPeriodic < BudCollection # :nodoc: all
+    def <=(o)
+      raise BudError, "Illegal use of <= with periodic '#{tabname}' on left"
+    end
+
+    superator "<~" do |o|
+      raise BudError, "Illegal use of <~ with periodic '#{tabname}' on left"
+    end
+
+    superator "<-" do |o|
+      raise BudError, "Illegal use of <- with periodic '#{tabname}' on left"
+    end
+
+    superator "<+" do |o|
+      raise BudError, "Illegal use of <+ with periodic '#{tabname}' on left"
+    end
+
+    def add_periodic_tuple(id)
+      pending_merge([[id, Time.now]])
+    end
   end
 
   class BudTable < BudCollection # :nodoc: all
@@ -876,16 +836,22 @@ module Bud
           @storage.delete keycols
         end
       end
-      @storage.merge! @pending
+      @pending.each do |keycols, tuple|
+        old = @storage[keycols]
+        if old.nil?
+          @storage[keycols] = tuple
+        else
+          raise_pk_error(tuple, old) unless tuple == old
+        end
+      end
       @to_delete = []
       @pending = {}
     end
 
     superator "<-" do |o|
-      o.each do |tuple|
-        next if tuple.nil?
-        tuple = prep_tuple(tuple)
-        @to_delete << tuple
+      o.each do |t|
+        next if t.nil?
+        @to_delete << prep_tuple(t)
       end
     end
   end
@@ -910,6 +876,15 @@ module Bud
       @linenum = 0
     end
 
+    public
+    def pro(&blk)
+      if @bud_instance.stratum_first_iter
+        return map(&blk)
+      else
+        return []
+      end
+    end
+
     public
     def each(&block) # :nodoc: all
       while (l = @fd.gets)