bud 0.0.2

data/lib/bud/graphs.rb

@@ -0,0 +1,405 @@
+# -*- coding: utf-8 -*-
+require 'rubygems'
+require 'graphviz'
+
+class GraphGen #:nodoc: all
+
+  def initialize(mapping, tableinfo, cycle, name, budtime, vizlevel, pics_dir, collapse=false, depanalysis=nil, cardinalities={})
+    #@graph = GraphViz.new(:G, :type => :digraph, :label => "", :ratio => 0.85 )
+    @graph = GraphViz.new(:G, :type => :digraph, :label => "")
+    @graph.node[:fontname] = "Times-Roman"
+    @graph.node[:fontsize] = 18
+    @graph.edge[:fontname] = "Times-Roman"
+    @graph.edge[:fontsize] = 18
+    @tiers = []
+    @cards = cardinalities
+    @name = name
+    @collapse = collapse
+    @depanalysis = depanalysis
+    @budtime = budtime
+    @vizlevel = vizlevel
+    @pics_dir = pics_dir
+    #@internals = {'count' => 1, 'localtick' => 1, 'stdio' => 1, 't_rules' => 1, 't_depends' => 1, 't_depends_tc' => 1, 't_provides' => 1, 't_cycle' => 1}
+    @internals = {'count' => 1, 'localtick' => 1, 'stdio' => 1} #, 't_rules' => 1, 't_depends' => 1, 't_depends_tc' => 1, 't_provides' => 1, 't_cycle' => 1}
+
+    # map: table -> stratum
+    @t2s = {}
+    mapping.each do |m|
+      @t2s[m[0]] = m[1].to_i
+    end
+
+    # map: table -> type
+    @tabinf = {}
+    tableinfo.each do |ti|
+      @tabinf[ti[0].to_s] = ti[1]
+    end
+
+    @redcycle = {}
+    cycle.each do |c|
+      puts "CYCLE: #{c.inspect}"
+      if c[2] and c[3]
+        if !@redcycle[c[0]]
+          @redcycle[c[0]] = []
+        end
+        @redcycle[c[0]] << c[1]
+      end      
+    end
+    
+    @nodes = {}
+    @edges = {}
+    @labels = {}
+  end
+  
+  def safe_t2s(tab)
+    if @t2s[tab]
+      @t2s[tab]
+    else  
+      words = tab.split(",")
+      maxs = 0
+      words.each do |w|
+        if @t2s[w] and @t2s[w] > maxs
+          maxs = @t2s[w]
+        end
+      end
+      return maxs
+    end
+  end
+
+  def name_bag(predicate, bag)
+    if bag[predicate]
+      return bag
+    else
+      bag[predicate] = true
+      res = bag
+      if @redcycle[predicate].nil?
+        return res 
+      end
+      @redcycle[predicate].each do |rp|
+        res = name_bag(rp, res)      
+      end
+    end
+
+    return res
+  end
+
+  def name_of(predicate)
+    # consider doing this in bud
+    # PAA
+    if @redcycle[predicate] and @collapse
+      puts "collapse #{predicate}, redcycle #{@redcycle[predicate].inspect}"
+      via = @redcycle[predicate]
+      bag = name_bag(predicate, {})
+      #str = bag.key_cols.sort.join(", ")
+      str = bag.keys.sort.join(", ")
+      return str
+    else
+      return predicate
+    end 
+  end
+
+  def process(depends)
+
+    # collapsing NEG/+ cycles.
+    # we want to create a function from any predicate to (cycle_name or bottom)
+    # bottom if the predicate is not in a NEG/+ cycle.  otherwise,
+    # its name is "CYC" + concat(sort(predicate names))
+
+    depends.each do |d|
+      #puts "DEP: #{d.inspect}"
+      head = d[1]
+      body = d[3]
+
+      # hack attack
+      if @internals[head] or @internals[body]
+        next
+      end
+
+      head = name_of(head)
+      body = name_of(body)
+      addonce(head, (head != d[1]))
+      addonce(body, (body != d[3]))
+      addedge(body, head, d[2], d[4], (head != d[1]), d[0])
+    end
+  end
+
+  def addonce(node, negcluster)
+    #puts "ADD NODE #{node}"
+    if !@nodes[node]
+      @nodes[node] = @graph.add_node(node)
+      if @cards and @cards[node]
+        @nodes[node].label = node +"\n (#{@cards[node].to_s})"
+        puts "IMAGE IS #{@cards[node]}"
+        #@nodes[node].image = @cards[node]
+      end
+ 
+      if @vizlevel >= 3 
+        @nodes[node].URL = "javascript:openWin(\"#{node}\", #{@budtime})"
+      else
+        @nodes[node].URL = "#{node}.html"
+      end
+    end
+
+    if negcluster
+      # cleaning 
+      res = node
+      node.split(", ").each_with_index do |p, i|
+        if i == 0
+          res = p
+        elsif i % 4 == 0
+          res = res + ",\n" + p
+        else
+          res = res + ", " + p
+        end
+      end
+      #@nodes[node].label = "<b>" + res + "</b>"
+      @nodes[node].label = res
+      @nodes[node].color = "red"
+      @nodes[node].shape = "octagon"
+      @nodes[node].penwidth = 3
+      @nodes[node].URL = "#{File.basename(@name)}_expanded.svg"
+    elsif @tabinf[node] and (@tabinf[node] == "Bud::BudTable")
+      @nodes[node].shape = "rect"
+    end
+  end
+
+  def addedge(body, head, op, nm, negcluster, rule_id=nil)
+    return if body.nil? or head.nil?
+    body = body.to_s
+    head = head.to_s
+    return if negcluster and body == head
+
+    ekey = body + head
+    if !@edges[ekey]
+      @edges[ekey] = @graph.add_edge(@nodes[body], @nodes[head], :penwidth => 5)
+      @edges[ekey].arrowsize = 2
+
+      @edges[ekey].URL = "#{rule_id}.html" unless rule_id.nil?
+      if head =~ /_msg\z/
+        @edges[ekey].minlen = 2
+      else
+        @edges[ekey].minlen = 1.5
+      end
+      @labels[ekey] = {}
+
+    end
+
+    #@edges[ekey].minlen = 5 if negcluster and body == head
+
+    if op == '<+'
+      @labels[ekey][' +/-'] = true
+    elsif op == "<~"
+      @edges[ekey].style = 'dashed'
+    elsif op == "<-"
+      #@labels[ekey] = @labels[ekey] + 'NEG(del)'
+      @labels[ekey][' +/-'] = true
+      @edges[ekey].arrowhead = 'veeodot'
+    end
+    if nm and head != "T"
+      # hm, nonmono
+      @edges[ekey].arrowhead = 'veeodot'
+    end
+  end
+
+  def finish
+    @labels.each_key do |k|
+      #@edges[k].label = @labels[k].key_cols.join(" ")
+      @edges[k].label = @labels[k].keys.join(" ")
+    end
+
+    addonce("S", false)
+    addonce("T", false)
+
+    @nodes["T"].URL = "javascript:advanceTo(#{@budtime+1})"
+    @nodes["S"].URL = "javascript:advanceTo(#{@budtime-1})"
+
+    @nodes["S"].color = "blue"
+    @nodes["T"].color = "blue"
+    @nodes["S"].shape = "diamond"
+    @nodes["T"].shape = "diamond"
+
+    @nodes["S"].penwidth = 3
+    @nodes["T"].penwidth = 3
+
+    @tabinf.each_pair do |k, v|
+
+      unless @nodes[name_of(k.to_s)] or k.to_s =~ /_tbl/ or @internals[k.to_s] or (k.to_s =~ /^t_/ and @budtime != 0)
+        addonce(k.to_s, false)
+      end
+      if v == "Bud::BudPeriodic"
+        puts "adding edge S -> #{@nodes[k.to_s]}"
+        addedge("S", k.to_s, false, false, false)
+      end
+    end
+
+    unless @depanalysis.nil? 
+      @depanalysis.source.each {|s| addedge("S", s.pred, false, false, false) }
+      @depanalysis.sink.each {|s| addedge(s.pred, "T", false, false, false) }
+
+      unless @depanalysis.underspecified.empty?
+        addonce("??", false)
+        @nodes["??"].color = "red"
+        @nodes["??"].shape = "diamond"
+        @nodes["??"].penwidth = 2
+      end
+
+      @depanalysis.underspecified.each do |u|
+        if u.input
+          addedge(u.pred, "??", false, false, false)
+        else
+          addedge("??", u.pred, false, false, false)
+        end
+      end
+    end
+
+    suffix = @collapse ? "collapsed" : "expanded"
+    fn = "#{@name}_#{suffix}.svg"
+    puts "fn is #{fn}"
+    staging = "#{fn}_staging"
+    @graph.output(:svg => staging)
+    @graph.output(:dot => "#{fn}.dot")
+    @graph.output(:png => "#{fn}.png")
+    fin = File.open(staging, "r")
+    fout = File.open(fn, "w")
+    while line = fin.gets
+      fout.puts line.gsub("<title>G</title>", svg_javascript())
+    end
+    fin.close
+    fout.close
+    File.delete(staging) 
+  end
+
+  def output_base
+    if @vizlevel >= 3
+      @pics_dir
+    else
+      "bud_doc"
+    end 
+  end
+
+  def dump(shredded_rules)
+    return if shredded_rules.nil?
+
+    fout = File.new("#{output_base}/style.css", "w")
+    fout.puts css
+    fout.close
+
+    code = {}
+    rules = {}
+    convertor = Syntax::Convertors::HTML.for_syntax "ruby"
+    shredded_rules.each do |s|
+      #fout = File.new("#{output_base}/#{s[0]}.html", "w+")
+      fout = File.new("#{output_base}/#{s[0]}.html", "w+")
+      fout.puts header
+      fout.puts "<h1>Rule #{s[0]}</h1><br>"
+
+      c = convertor.convert(s[3])
+      c.sub!(/^<pre>/, "<pre class=\"code\" style='font-size:20px'>\n")
+      fout.puts c
+      rules[s[0]] = [s[1], s[3]]
+      fout.close
+    end
+
+    rules.each_pair do |k, v|
+      if !code[v[0]]
+        code[v[0]] = ""
+      end
+      #code[v[0]] = "<br># RULE #{k}<br> " + code[v[0]] + "<br>" + v[1]
+      code[v[0]] = "\n# RULE #{k}\n " + code[v[0]] + "\n" + v[1]
+    end
+    @nodes.each_pair do |k, v|
+      fout = File.new("#{output_base}/#{k}.html", "w+")
+      fout.puts header
+      k.split(", ").each do |i|
+        unless code[i].nil?
+          c = convertor.convert(code[i])
+          c.sub!(/^<pre>/, "<pre class=\"code\">\n")
+          fout.puts c
+        end
+      end
+      fout.puts("</body></html>")
+      fout.close
+    end
+  end
+
+
+  def header
+      return "<html><meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>\n<head><link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\" /></head><body>"
+  end
+
+  def css
+    return "pre.code {
+  padding: 1ex 1ex 1ex 1ex;
+  border: 4px groove #CC0000;
+  overflow-x: auto;
+}
+
+pre.code span.attribute { color: #009900; }
+pre.code span.char { color: #F00; }
+pre.code span.class { color: #A020F0; font-weight: bold; }
+pre.code span.comment { color: #0000FF; }
+pre.code span.constant { color: #008B8B; }
+pre.code span.escape { color: #6A5ACD; }
+pre.code span.expr { color: #2222CC; }
+pre.code span.global { color: #11AA44; }
+pre.code span.ident { color: #000000; }
+pre.code span.keyword { color: #A52A2A; font-weight: bold; }
+pre.code span.method { color: #008B8B; }
+pre.code span.module { color: #A020F0; font-weight: bold; }
+pre.code span.number { color: #DD00DD; }
+pre.code span.punct { color: #6A5ACD; }
+pre.code span.regex { color: #DD00DD; }
+pre.code span.string { color: #DD00DD; }
+pre.code span.symbol { color: #008B8B; }
+"
+  end
+
+  
+end
+
+  def svg_javascript
+    return "
+<script type='text/javascript'>
+  <![CDATA[
+
+var windows = new Array()
+var info = new Array()
+
+function openWin(target, time) {
+  win = window.open(target + \"_\" + time + \".html\", target, \"location=no,width=400,height=180,left=0,status=no\");
+  // hm, an associative array, how strange.
+  info[target] = 1
+}
+
+function advanceTo(time) {
+  arr = gup(\"wins\").split(\",\");
+  for (i=0; i < arr.length; i++) {
+    if (arr[i] != \"\") {
+      openWin(arr[i], time);
+    }
+  }
+  str = '';
+  // getting 'key_cols'
+  for (var i in info) {
+    str = str + ',' + i;
+  }
+  self.window.location.href = 'tm_' + time + '_expanded.svg?wins=' + str;
+}
+
+// off the netz
+function gup( name )
+{
+  name = name.replace(/[\[]/,\"\\\[\").replace(/[\]]/,\"\\\]\");
+  var regexS = \"[\\?&]\"+name+\"=([^&#]*)\";
+  var regex = new RegExp( regexS );
+  var results = regex.exec( window.location.href );
+  if( results == null )
+    return \"\";
+  else
+    return results[1];
+}
+
+  ]]>
+</script>
+"
+  end
+

data/lib/bud/joins.rb

@@ -0,0 +1,300 @@
+module Bud
+  class BudJoin < BudCollection
+    attr_accessor :rels, :origrels, :origpreds # :nodoc: all
+    attr_reader :hash_tables # :nodoc: all
+
+    def initialize(rellist, bud_instance, preds=nil) # :nodoc: all
+      @schema = []
+      otherpreds = nil
+      @origpreds = preds
+      @bud_instance = bud_instance
+      @localpreds = nil
+
+      # if any elements on rellist are BudJoins, suck up their contents
+      tmprels = []
+      rellist.each do |r|
+        if r.class <= BudJoin
+          tmprels += r.origrels
+          preds += r.origpreds
+        else
+          tmprels << r
+        end
+      end
+      rellist = tmprels
+      @origrels = rellist
+
+      # recurse to form a tree of binary BudJoins
+      @rels = [rellist[0]]
+      @rels << (rellist.length == 2 ? rellist[1] : BudJoin.new(rellist[1..rellist.length-1], @bud_instance, nil))
+      # derive schema: one column for each table.
+      # duplicated inputs get distinguishing numeral
+      @schema = []
+      index = 0
+      retval = rellist.reduce({}) do |memo, r|
+        index += 1
+        memo[r.tabname.to_s] ||= 0
+        newstr = r.tabname.to_s + ((memo[r.tabname.to_s] > 0) ? ("_" + memo[r.tabname.to_s].to_s) : "")
+        @schema << newstr.to_sym
+        memo[r.tabname.to_s] += 1
+        memo
+      end
+
+      setup_preds(preds) unless preds.nil? or preds.empty?
+      setup_state
+    end
+
+    public
+    def state_id # :nodoc: all
+      Marshal.dump([@rels.map{|r| r.tabname}, @localpreds]).hash
+    end
+
+    # initialize the state for this join to be carried across iterations within a fixpoint
+    private
+    def setup_state
+      sid = state_id
+      @tabname = ("temp_join"+state_id.to_s).to_sym
+      @bud_instance.joinstate[sid] ||= [{:storage => {}, :delta => {}}, {:storage => {}, :delta => {}}]
+      @hash_tables = @bud_instance.joinstate[sid]
+    end
+
+    private_class_method
+    def self.natural_preds(bud_instance, rels)
+      preds = []
+      rels.each do |r|
+        rels.each do |s|
+          matches = r.schema & s.schema
+          matches.each do |c|
+            preds << [bud_instance.send(r.tabname).send(c), bud_instance.send(s.tabname).send(c)] unless r.tabname.to_s >= s.tabname.to_s
+          end
+        end
+      end
+      preds.uniq
+    end
+
+    # flatten joined items into arrays, with attribute accessors inherited
+    # from the input collections, disambiguated via suffix indexes as needed.
+    # similar to <tt>SELECT * FROM ... WHERE...</tt> block in SQL.
+    public
+    def flatten(*preds)
+      setup_preds(preds) unless preds.nil? or preds.size == 0
+      flat_schema = @rels.map{|r| r.schema}.flatten(1)
+      dupfree_schema = []
+      # while loop here (inefficiently) ensures no collisions
+      while dupfree_schema == [] or dupfree_schema.uniq.length < dupfree_schema.length
+        dupfree_schema = []
+        flat_schema.reduce({}) do |memo, r|
+          if r.to_s.include?("_") and ((r.to_s.rpartition("_")[2] =~ /^\d+$/) == 0)
+            r = r.to_s.rpartition("_")[0].to_sym
+          end
+          memo[r] ||= 0
+          if memo[r] == 0
+            dupfree_schema << r.to_s.to_sym
+          else
+            dupfree_schema << (r.to_s + "_" + (memo[r]).to_s).to_sym
+          end
+          memo[r] += 1
+          memo
+        end
+        flat_schema = dupfree_schema
+      end
+      retval = BudScratch.new('temp_flatten', bud_instance, dupfree_schema)
+      retval.uniquify_tabname
+      retval.merge(self.map{|r,s| r + s}, retval.storage)
+    end
+
+    undef do_insert
+
+    public
+    # map each (nested) item in the collection into a string, suitable for placement in stdio
+    def inspected
+      raise BudError, "join left unconverted to binary" if @rels.length > 2
+      self.map{|r1, r2| ["\[ #{r1.inspect} #{r2.inspect} \]"]}
+    end
+
+    public
+    def pro(&blk) # :nodoc: all
+      pairs(&blk)
+    end
+
+    public
+    def each(mode=:both, &block) # :nodoc: all
+      mode = :storage if @bud_instance.stratum_first_iter
+      if mode == :storage
+        methods = [:storage]
+      else
+        methods = [:delta, :storage]
+      end
+
+      methods.each do |left_rel|
+        methods.each do |right_rel|
+          next if (mode == :delta and left_rel == :storage and right_rel == :storage)
+          if @localpreds.nil? or @localpreds.empty?
+            nestloop_join(left_rel, right_rel, &block)
+          else
+            hash_join(left_rel, right_rel, &block)
+          end
+        end
+      end
+      tick_hash_deltas
+    end
+
+    public
+    def each_from_sym(buf_syms, &block) # :nodoc: all
+      buf_syms.each do |s|
+        each(s, &block)
+      end
+    end
+
+    private
+    # r is a tuple
+    # s is an array (combo) of joined tuples
+    def test_locals(r, s, *skips)
+      retval = true
+      if (@localpreds and skips and @localpreds.length > skips.length)
+        # check remainder of the predicates
+        @localpreds.each do |pred|
+          next if skips.include? pred
+          r_offset, s_index, s_offset = join_offsets(pred)
+          if r[r_offset] != s[s_index][s_offset]
+            retval = false
+            break
+          end
+        end
+      end
+      return retval
+    end
+
+    private
+    def nestloop_join(left_rel, right_rel, &block)
+      @rels[0].each_from_sym([left_rel]) do |r|
+        @rels[1].each_from_sym([right_rel]) do |s|
+          s = [s] if origrels.length == 2
+          yield([r] + s) if test_locals(r, s)
+        end
+      end
+    end
+
+    private
+    # calculate the attribute position for the left table in the join ("left_offset")
+    # the right table may itself be a nested tuple from a join, so calculate
+    # the tuple offset ("right_subtuple") and the attribute position within it
+    # ("right_offset")
+    def join_offsets(pred)
+      right_entry = pred[1]
+      right_name, right_offset = right_entry[0], right_entry[1]
+      left_entry = pred[0]
+      left_name, left_offset = left_entry[0], left_entry[1]
+
+      # determine which subtuple of right collection contains the table
+      # referenced in RHS of pred.  note that right collection doesn't contain the
+      # first entry in rels, which is the left collection
+      right_subtuple = 0
+      origrels[1..origrels.length].each_with_index do |t,i|
+        if t.tabname == pred[1][0]
+          right_subtuple = i
+          break
+        end
+      end
+
+      return left_offset, right_subtuple, right_offset
+    end
+
+    def tick_hash_deltas
+      # for hash_join, move old delta hashtables into storage hashtables
+      return if @hash_tables.nil?
+      (0..1).each do |i|
+        @hash_tables[i][:storage].merge!(@hash_tables[i][:delta]) do |k,l,r|
+            l+r
+        end
+        @hash_tables[i][:delta] = {}
+      end
+    end
+
+    # semi-naive symmetric hash join on first predicate
+    private
+    def hash_join(left_sym, right_sym, &block)
+      left_offset, right_subtuple, right_offset = join_offsets(@localpreds.first)
+
+      syms = [left_sym, right_sym]
+
+      syms.each_with_index do |probe_sym, probe_ix|
+        other_ix = 1 - probe_ix # bit-flip
+        other_sym = syms[other_ix]
+        probe_offset = (probe_ix == 0) ? left_offset : right_offset
+
+        # in a delta/storage join we do traditional one-sided hash join
+        # so don't probe from the storage side.
+        # the other side should have been built already!
+        if probe_sym == :storage and probe_sym != other_sym
+          next
+        end
+
+        # ready to do the symmetric hash join
+        rels[probe_ix].each_from_sym([probe_sym]) do |r|
+          r = [r] unless probe_ix == 1 and origrels.length > 2
+          attrval = (probe_ix == 0) ? r[0][left_offset] : r[right_subtuple][right_offset]
+
+          # insert into the prober's hashtable only if symmetric ...
+          if probe_sym == other_sym
+            @hash_tables[probe_ix][probe_sym][attrval] ||= []
+            @hash_tables[probe_ix][probe_sym][attrval] << r
+          end
+
+          # ...and probe the other hashtable
+          next if @hash_tables[other_ix][other_sym][attrval].nil?
+          @hash_tables[other_ix][other_sym][attrval].each do |s_tup|
+            if probe_ix == 0
+              left = r; right = s_tup
+            else
+              left = s_tup; right = r
+            end
+            retval = left + right
+            yield retval if test_locals(left[0], right, @localpreds.first)
+          end
+        end
+      end
+    end
+  end
+
+  class BudLeftJoin < BudJoin # :nodoc: all
+    def initialize(rellist, bud_instance, preds=nil)
+      raise(BudError, "Left Join only defined for two relations") unless rellist.length == 2
+      super(rellist, bud_instance, preds)
+      @origpreds = preds
+      preds.each do |k,v|
+        if k.class <= Array
+          raise Bud::CompileError, "in leftjoin, attribute refs must have style :col1 => :col2"
+        end
+      end
+    end
+
+    public
+    def each(&block) # :nodoc:all
+      super(&block)
+      # previous line finds all the matches.
+      # now its time to ``preserve'' the outer tuples with no matches.
+      # this is totally inefficient: we should fold the identification of non-matches
+      # into the join algorithms.  Another day.
+      # our trick: for each tuple of the outer, generate a singleton relation
+      # and join with inner.  If result is empty, preserve tuple.
+      @rels[0].each do |r|
+        t = @origrels[0].clone_empty
+        # need to uniquify the tablename here to avoid sharing join state with original
+        t.uniquify_tabname
+        t << r
+        j = BudJoin.new([t, @origrels[1]], @bud_instance, @origpreds)
+
+        # the following is "next if j.any?" on storage tuples *only*
+        any = false
+        j.each(:storage) do |j|
+          any = true
+          break
+        end
+        next if any
+
+        nulltup = @origrels[1].null_tuple
+        yield [r, nulltup]
+      end
+    end
+  end
+end