bud 0.0.3 → 0.0.4

data/README

@@ -1,30 +1,47 @@
 # Bud
 
-This is Bud, a.k.a. "Bloom Under Development".  It is an initial cut at a Bloom DSL, using Ruby as a setting.
+This is Bud, a.k.a. "Bloom Under Development".  It is an initial cut at a Bloom
+DSL, using Ruby as a setting.
 
-Please see LICENSE for licensing information.
+See LICENSE for licensing information.
 
-Language cheatsheet in doc/cheat.md
+Language cheatsheet in docs/cheat.md ; see the docs/ directory for other
+documentation.
 
 Main deficiencies at this point are:
-- Inefficient evaluation: Programs are run using semi-naive evaluation strategies, but no further
-  query optimization has been implemented, and little effort has been spent in
-  tuning.
 
-- No Ruby constraints: Within Bloom programs the full power of Ruby is also available, including mutable state.
-  This allows programmers to get outside the Bloom framework and lose cleanliness.
+- Inefficient evaluation: Programs are run using semi-naive evaluation
+  strategies, but no further query optimization has been implemented, and little
+  effort has been spent in tuning.
 
-- Compatibility: Bud only works with Ruby (MRI) 1.8. MRI 1.9, JRuby and other Ruby
-  implementations are currently not supported.
+- No Ruby constraints: Within Bloom programs the full power of Ruby is also
+  available, including mutable state.  This allows programmers to get outside
+  the Bloom framework and lose cleanliness.
 
-To install:
+- Compatibility: Bud only works with Ruby (MRI) 1.8. MRI 1.9, JRuby and other
+  Ruby implementations are currently not supported.
+
+## Installation
+
+To install the latest release:
     % gem install bud
 
-Simple example programs can be found in examples. A much larger set
-of example programs and libraries can be found in the bud-sandbox repo.
+To build and install a new gem from the current development sources:
+    % gem build bud.gemspec ; gem install bud*.gem
+
+Note that GraphViz must be installed.
+
+Simple example programs can be found in examples. A much larger set of example
+programs and libraries can be found in the bud-sandbox repository.
+
+To run the unit tests:
+    % cd test; ruby ts_bud.rb
 
 ## Optional Dependencies
 
-The bud gem has a handful of mandatory dependencies. It also has one optional
-dependency: if you wish to use the Bud collections backed by Zookeeper (the
-"zktable" collection type), the "zookeeper" gem must be installed.
+The bud gem has a handful of mandatory dependencies. It also has two optional
+dependencies: if you wish to use Bud collections backed by Zookeeper or Tokyo
+Cabinet (the "zktable" and "tctable" collection types, respectively), the
+"zookeeper" and/or "tokyocabinet" gems must be installed. Note that before
+installing the "tokyocabinet" gem, the Tokyo Cabinet libraries should be
+installed first.

data/bin/budplot

@@ -2,26 +2,30 @@
 require 'rubygems'
 require 'bud'
 require 'bud/bud_meta'
-require 'bud/graphs'
 require 'bud/depanalysis'
+require 'bud/graphs'
+require 'bud/viz_util'
 
-# args: N, files_n, classes_m
-
-def process(mods)
-  classdef = "class FooBar\ninclude Bud\n" + mods.map{|m| "include #{m}"}.join("\n")  +  "\nend\n FooBar.new"
-  puts "classdef #{classdef}"
-  d = eval(classdef)
-
-  d.t_rules.each {|s| puts "RULE: #{s.inspect}" }
-  #d.t_depends.each {|s| puts "DEP: #{s.inspect}" }
+include VizUtil
 
-  da = d.meta_parser.depanalysis
+def is_module?(m)
+  begin
+    return (eval("defined?(#{m})") == "constant")
+  rescue SyntaxError
+    return false
+  end
+end
 
-  #puts "MP info src=#{da.source.length}, snk=#{da.sink.length}, under=#{da.underspecified.length}"
+def process(mods)
+  mods.each do |m|
+    unless is_module? m
+      puts "Error: unable to find definition for module \"#{m}\""
+      exit
+    end
+  end
 
-  #d.meta_parser.depanalysis.source.each do |s|
-  #  puts "SRC: #{s}"
-  #end
+  classdef = "class FooBar\ninclude Bud\n" + mods.map{|m| "include #{m}"}.join("\n") + "\nend\n FooBar.new"
+  d = eval(classdef)
 
   interfaces = {}
   d.t_provides.each do |name, is_input|
@@ -48,87 +52,60 @@ def process(mods)
     end
   end
 
-  svg = "bud_doc/" + mods.join("_") + "_viz" 
-  write_index(inp, outp, priv, svg)
-  gv = GraphGen.new(d.t_stratum, tabinf, d.t_cycle, svg, -1, 1, ".", true, d.meta_parser.depanalysis)
-  gv.process(d.t_depends)
-  gv.dump(d.t_rules)
-  gv.finish
-
-  gv2 = GraphGen.new(d.t_stratum, tabinf, d.t_cycle, svg, -1, 1, ".", false, d.meta_parser.depanalysis)
-  gv2.process(d.t_depends)
-  gv2.dump(d.t_rules)
-  gv2.finish
+  viz_name = "bud_doc/" + mods.join("_") + "_viz"
+  write_index(inp, outp, priv, viz_name)
+  graph_from_instance(d, "#{viz_name}_collapsed", "bud_doc", true)
+  graph_from_instance(d, "#{viz_name}_expanded", "bud_doc", false)
 end
 
-def write_index(inp, outp, priv, svg)
+def write_index(inp, outp, priv, viz_name)
   f = File.open("bud_doc/index.html", "w")
   f.puts "<html>"
-  f.puts "<embed src=\"#{ENV['PWD']}/#{svg}_collapsed.svg\" width=\"100%\" height=\"60%\" type=\"image/svg+xml\" pluginspage=\"http://www.adobe.com/svg/viewer/install/\" />"
+  f.puts "<embed src=\"#{ENV['PWD']}/#{viz_name}_collapsed.svg\" width=\"100%\" height=\"60%\" type=\"image/svg+xml\" pluginspage=\"http://www.adobe.com/svg/viewer/install/\" />"
 
   f.puts "<table border='1' valign='top' width = '100%'><tr valign='top'>"
   f.puts "<td valign='top'>"
-  f.puts "<h2> Input Interfaces </h2>"
+  f.puts "<h2>Input Interfaces</h2>"
   do_table(f, inp)
   f.puts "</td><td>"
-  f.puts "<h2> Output Interfaces </h2>"
+  f.puts "<h2>Output Interfaces</h2>"
   do_table(f, outp)
   f.puts "</td><td>"
-  f.puts "<h2> Private State </h2>"
-  do_table(f, priv, true)
+  f.puts "<h2>Private State</h2>"
+  do_table(f, priv)
   f.puts "</td>"
   f.puts "</tr></table>"
   f.puts "</html>"
   f.close
 end
 
-def do_table(f, info, type=false)
-  next if info.nil?
+def do_table(f, info)
   f.puts "<table border='1'>"
-  info.sort{|a, b| a[0].to_s <=> b[0].to_s}.each do |inf|
-    #f.puts "<h3>#{inf[0]}</h3>"
-    #f.puts  "&nbsp; (#{inf[1].class.to_s.gsub('Bud::Bud', '')})<br>" if type
-    unless inf[1].schema.nil?
-      f.puts "<tr><td><b>#{inf[0]}</b></td>" + inf[1].schema.map{|i| "<td>#{i}</td>"}.join(" ") + "</tr>"
-    end
+  info.sort{|a, b| a[0].to_s <=> b[0].to_s}.each do |tbl_name, tbl_impl|
+    next if tbl_impl.schema.nil?
+    key_s = tbl_impl.key_cols.join(", ")
+    key_s = "[]" if key_s == ""
+    val_s = tbl_impl.val_cols.join(", ")
+    f.puts "<tr><td><b>#{tbl_name}</b></td>"
+    f.puts "<td>#{key_s}</td><td>#{val_s}</td></tr>"
   end
   f.puts "</table>"
 end
 
-@shreddies = []
-@provides = []
-@demands = []
-@tabinf = {}
-@port = 12345
-files = []
-classes = []
-
 if ARGV.length < 2
-  puts "len #{ARGV.length} USAGE:\nruby plotter.rb LIST_OF_FILES LIST_OF_MODULES"
+  puts "Usage: budplot LIST_OF_FILES LIST_OF_MODULES"
   exit
 end
 
 `mkdir bud_doc`
 
 modules = []
-(0..ARGV.length-1).each do |i|
-  begin
-    f = File.open(ARGV[i])
-    mods = false
-  rescue
-    mods = true
-  end
-
-  unless mods
-    puts "DO #{ARGV[i]}"
-    eval ( "require \"#{ARGV[i]}\"")
+ARGV.each do |arg|
+  if File.exists? arg
+    eval "require '#{arg}'"
   else
-    puts "Work on #{ARGV[i]}"
-    #instant(ARGV[i])
-    modules << ARGV[i]
-    puts "OK"
+    modules << arg
   end
 end
 
 process(modules)
-

data/bin/budtimelines

@@ -0,0 +1,235 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'dbm'
+require 'bud'
+require 'bud/graphs'
+require 'bud/viz_util'
+require 'getopt/std'
+
+include VizUtil
+
+module Depends
+  state do
+    table :depends, [:rid, :lhs, :op, :rhs, :nm]
+  end
+end
+
+class GlobalDepAnalyzer
+  include Bud
+  include Depends
+  # this module's purpose and mechanism are very similar
+  # to those of basic stratification.  here, we are interested
+  # in NM paths that DO cross temporal edges.
+
+  state do
+    table :depends_tc, [:lhs, :rhs, :via, :nm]
+  end
+
+  bloom do
+    depends_tc <= depends{|d| [d.lhs, d.rhs, d.rhs, d.nm]}
+    depends_tc <= (depends * depends_tc).pairs(:rhs => :lhs) do |d, tc|
+      [d.lhs, tc.rhs, d.rhs, (d.nm or tc.nm)]
+    end
+  end
+end
+
+module TPSchema
+  state do
+    table :deltas, [:bud_time, :tab, :nm]
+    table :zerod_cards, [:bud_time, :table, :cnt]
+    table :nm_tab, [:table]
+    table :collapsible_base, [:start, :fin]
+    table :collapsible, [:start, :fin]
+    scratch :collapsible_tmp, [:start, :fin]
+    scratch :lcl_best_interval, [:start, :fin]
+    table :best_interval, [:start, :fin]
+  end
+end
+
+module DeltaLogic
+  include TPSchema
+  bloom do
+    zerod_cards <= cardinalities
+    zerod_cards <= (times * depends).pairs do |t, d|
+      unless cardinalities{|c| c[1] if c[0] == t.bud_time}.include? d[1]
+        [t.bud_time, d[1], 0]
+      end
+    end
+
+
+    nm_tab <= depends do |d|
+      [d[1]] if d[4]
+    end
+
+    deltas <= (zerod_cards * zerod_cards).pairs(:table => :table) do |c1, c2|
+      if c1.bud_time == c2.bud_time - 1 and c1.table == c2.table and c1.cnt != c2.cnt
+        if nm_tab.include? [c1.table]
+          [c2.bud_time, c1.table, true]
+        else 
+          [c2.bud_time, c1.table, false]
+        end
+      end
+    end
+  end
+end
+
+module VanillaTraceProcessing
+  include TPSchema
+  include DeltaLogic
+
+  bloom do
+    collapsible_base <= times do |t|
+      unless deltas{|d| d.bud_time if d.nm}.include? t.bud_time
+        [t.bud_time-1, t.bud_time]
+      end
+    end
+
+    collapsible <= collapsible_base
+
+    collapsible <= (collapsible_base * collapsible).pairs(:fin => :start) do |b, c|
+      puts "another collapsible row; now #{b.inspect} - #{c.inspect}"
+      [b.start, c.fin]
+    end
+
+    best_interval <= collapsible do |c|
+      unless collapsible{|c1| c1.start == c.start and c1.fin > c.fin}.any? \
+       or collapsible{|c2| c2.fin == c.fin and c2.start < c.start}.any?
+        c
+      end
+    end
+  end
+end
+
+class SimpleTraceProcessor
+  include Bud
+  include TraceCardinality
+  include Depends
+  include VanillaTraceProcessing
+    
+end
+
+def collapse(intervals, host, time)
+  return time unless @opts["C"]
+  # worth rethinking when the # of intervals/instance gets high
+  intervals[host].each do |i|
+    if time > i[0] and time < i[1]
+      return i[1]
+    end
+  end
+  return time
+end
+
+def usage
+  puts "USAGE:"
+  exit
+end
+
+usage unless ARGV[0]
+usage if ARGV[0] == '--help'
+
+@opts = Getopt::Std.getopts("CLo:")
+
+snd_info = {}
+rcv_info = {}
+clean_arg = []
+intervals = {}
+
+
+da = GlobalDepAnalyzer.new
+
+ARGV.each do |arg_raw|
+  elems = arg_raw.split("_")
+  arg = elems[1..3].join("_")
+  clean_arg << arg
+  snd_info[arg] = []
+  rcv_info[arg] = []
+
+  meta, data = get_meta2("#{arg_raw}/bud_")
+  tp = SimpleTraceProcessor.new
+
+  meta[:depends].each do |m|
+    tp.depends << m  
+    da.depends << m
+  end
+
+  data.each do |d|
+    tp.full_info << d
+    if meta[:tabinf].map{|m| m[0] if m[1] == "Bud::BudChannel"}.include? d[1]
+      if d[1] =~ /_snd\z/
+        snd_info[arg] << d 
+      else
+        rcv_info[arg] << d
+      end
+    elsif meta[:tabinf].map{|m| m[0] if m[1] == "Bud::BudPeriodic"}.include? d[1]
+    end
+  end
+
+  tp.tick
+
+
+  puts "entries in collapsible: #{tp.collapsible.length}"
+  puts "entries in base: #{tp.collapsible_base.length}"
+  puts "entries in deltas: #{tp.deltas.length}"
+
+  intervals[arg] = []
+  tp.best_interval.each do |n| 
+    puts "BEST INTERVAL[#{arg}]: #{n.inspect}"
+    intervals[arg] << n
+  end
+end
+
+da.tick
+nmreach = {}
+da.depends_tc.each do |d|
+  nmreach[d[0]] = {} unless nmreach[d[0]]
+  nmreach[d[0]][d[1]] = d[3]
+end
+
+# our local intervals relations are too optimistic.  to say that intervals[foo] = [2, 5]
+# is merely to say that nothing NM happened locally btw 2 and 5.  it is only safe to collapse
+# 2 and 5 if during this interval, we could not have BOTH caused and perceived the results of 
+# a NM deduction.  we can (again, conservatively) ensure that this is not the case by showing
+# that from 2-5, there exist no messages A and B s.t. we sent A and received B in the interval 2..5
+# and B <n A (where <n means is globally NM-derivable from).
+
+# we can of course do better than this even with the data we have, but it is complicated and we'll
+# get to it later.
+
+plot_data = []
+snd_info.each_pair do |k1, v1|
+  rcv_info.each_pair do |k2, v2|
+    v1.each do |lval|
+      v2.each do |rval|
+        unless k1 == k2
+          # erm, tuple equivalence?
+          l = lval[2].clone
+          r = rval[2].clone
+          l.shift
+          r.shift
+          if l.inspect == r.inspect
+            plot_data << [l.inspect, k1, k2, lval[0], rval[0], rval[1]]
+          end
+        else
+          if collapse(intervals, k1, lval[0]) == collapse(intervals, k2, rval[0]) and lval[0] < rval[0]
+            unless nmreach[rval[1]].nil?
+              if nmreach[rval[1]][lval[1].gsub(/_snd\z/, "")]
+                match = intervals[k1].find_all{|i| i[1] == collapse(intervals, k1, lval[0])}.first
+                intervals[k1].delete(match)
+                intervals[k1] << [match[0], lval[0]]
+                intervals[k1] << [lval[0]+1, match[1]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+rpd = plot_data.map do |p|
+  [p[0], p[1], p[2], collapse(intervals, p[1], p[3]), collapse(intervals, p[2], p[4]), p[5]]
+end
+
+st = SpaceTime.new(rpd, @opts["L"])
+st.process
+st.finish("spacetime_trace_#{clean_arg.join('-')}", @opts["o"])

data/bin/budvis

@@ -1,49 +1,33 @@
 #!/usr/bin/env ruby
 require 'rubygems'
-require 'tokyocabinet'
+require 'dbm'
 require 'bud'
 require 'bud/graphs'
-include TokyoCabinet
+require 'bud/viz_util'
 
-# prototype offline viz
-
-BUD_TC_DIR = "#{ARGV[0]}/bud_"
+include VizUtil
 
+BUD_DBM_DIR = "#{ARGV[0]}/bud_"
 
 class VizHelper
   include Bud
+  include TraceCardinality
 
-  state do
-    table :full_info, [:bud_time, :table, :row]
-    scratch :cardinalities, [:bud_time, :table] => [:cnt]
-    scratch :times, [:bud_time]
-  end
-
-  def initialize(strata, tabinf, cycle, depends, rules, dir)
-    @t_strata = strata
+  def initialize(tabinf, cycle, depends, rules, dir)
     @t_tabinf = tabinf
     @t_cycle = cycle
     @t_depends = depends
-    @dir = dir
     @t_rules = rules
+    @dir = dir
     super()
   end
 
-  bloom :counting do
-    cardinalities <= full_info.group([full_info.bud_time, full_info.table], count)
-    times <= full_info.map{|f| [f.bud_time]}
-  end
-
   def summarize(dir, schema)
     table_io = {}
-    timeseries = {}
     cardinalities.sort{|a, b| a[0] <=> b[0]}.each do |card|
       table_io["#{card.table}_#{card.bud_time}"] = start_table(dir, card.table, card.bud_time, schema[card.table])
-      unless timeseries[card.table]
-        timeseries[card.table] = []
-      end
-      timeseries[card.table] << card.cnt
     end
+
     full_info.each do |info|
       write_table_content(table_io["#{info.table}_#{info.bud_time}"], info.row)
     end
@@ -52,19 +36,6 @@ class VizHelper
       end_table(tab)
     end
 
-    ts2 = {}
-    timeseries.each_pair do |k, v|
-      fn = v
-      puts "GOT #{fn}"
-      #ts2[k] = "#{ENV['PWD']}/#{fn}"
-      ts2[k] = v
-    end
-
-    sum = GraphGen.new(@t_strata, @t_tabinf, @t_cycle, "#{@dir}/summary", -1, 3, @dir, false, nil, ts2)
-    sum.process(@t_depends)
-    sum.dump(@t_rules)
-    sum.finish
-
     # fix: nested loops
     times.sort.each do |time|
       card_info = {}
@@ -74,24 +45,18 @@ class VizHelper
         end
       end
 
-      gv = GraphGen.new(@t_strata, @t_tabinf, @t_cycle, "#{@dir}/tm_#{time.bud_time}", time.bud_time, 3, @dir, false, nil, card_info)
-      gv.process(@t_depends)
-      gv.dump(@t_rules)
-      gv.finish
-
+      d = "#{@dir}/tm_#{time.bud_time}"
+      write_graphs(@t_tabinf, @t_cycle, @t_depends, @t_rules, d, @dir, nil, false, nil, time.bud_time, card_info)
     end
-
   end
 
   def start_table(dir, tab, time, schema)
     str = "#{dir}/#{tab}_#{time}.html"
-    #fout = File.new("#{dir}/#{tab}_#{time}.html", "w")
     fout = File.new(str, "w")
 
-    #fout.puts "<h1>#{tab} #{time_node_header()}</h1>"
     fout.puts "<html><title>#{tab} @ #{time}</title>"
     fout.puts "<table border=1>"
-    fout.puts "<tr>" + schema.map{|s| "<th> #{s} </th>"}.join(" ") + "<tr>"
+    fout.puts "<tr>" + schema.map{|s| "<th> #{s} </th>"}.join(" ") + "<tr>" unless schema.nil?
     fout.close
     return str
   end
@@ -111,91 +76,28 @@ class VizHelper
   end
 end
 
-def deserialize_table(tab, strict)
-  # oy.  meta only
-  ret = []
-  tab.each_pair do |k, v|
-    #key = Marshal.load(k)
-    key = MessagePack.unpack(k)
-    time = key.shift
-    raise "non-zero budtime.  sure this is metadata?" if time != 0 and strict
-    tup = key
-    #Marshal.load(v).each{|v| tup << v }
-    MessagePack.unpack(v).each{|val| tup << val }
-    ret << tup
-  end
-  tab.close
-  return ret
-end
-
 def usage
   puts "Usage:"
-  puts "Running a Bud program with option :visualize => 3 will cause a TC directory TC_dir to be created (Class_ObjectId_Port)"
-  puts "> ruby visualize.rb TC_dir"
-  puts "This will create a series of svg files in TC_dir, the root of which will be named tm_0_expanded.svg.  Open in a browser.\n"
+  puts "Running a Bud program with option :trace => true will cause a DBM directory DBM_dir to be created (Class_ObjectId_Port)"
+  puts "> budvis DBM_dir"
+  puts "This will create a series of svg files in DBM_dir, the root of which will be named tm_0.svg.  Open in a browser.\n"
   puts "e.g."
   puts "> ruby test/tc_carts.rb"
-  puts "> ruby visualize.rb BCS_2159661360_"
-  puts "> open -a /Applications/Google\ Chrome.app/ BCS_2159661360_/tm_0_expanded.svg"
+  puts "> budvis DBM_BCS_2159661360_"
+  puts "> open DBM_BCS_2159661360_/tm_0.svg"
+  puts "\nWith the SVG file open in a browser, you may navigate forward and backward in time"
+  puts "by clicking the T and S nodes, respectively."
   exit
 end
 
-# begin "main"
-
 usage unless ARGV[0]
+usage if ARGV[0] == '--help'
 
-@tables = {}
-
-Dir.new(BUD_TC_DIR).entries.each do |file|
-  next if file =~ /^\./
-  puts "FILE is #{file}"
-  hdb = TokyoCabinet::HDB.new
-  ret = hdb.open("#{BUD_TC_DIR}/#{file}", HDB::OREADER)
-  raise "db not found" unless ret
-  @tables[file] = hdb
-end
-
-
-# let's try to do a visualization
-strata = deserialize_table(@tables['t_stratum_vizlog.tch'], true)
-tabinf = deserialize_table(@tables['t_table_info_vizlog.tch'], true)
-tabscm = deserialize_table(@tables['t_table_schema_vizlog.tch'], true)
-cycle = deserialize_table(@tables['t_cycle_vizlog.tch'], true)
-depends = deserialize_table(@tables['t_depends_vizlog.tch'], true)
-rules = deserialize_table(@tables['t_rules_vizlog.tch'], true)
-
-
-schminf = {}
-tabscm.each do |ts|
-  puts "TABSCM: #{ts.inspect}"
-  tab = ts[0].to_s
-  unless schminf[tab]
-    schminf[tab] = []
-  end
-  schminf[tab][ts[2]] = ts[1]
-end
-
-gv = GraphGen.new(strata, tabinf, cycle, "OUTPUT", -1, 1, "plotter_out")
-gv.process(depends)
-gv.finish
-
-
-vh = VizHelper.new(strata, tabinf, cycle, depends, rules, ARGV[0])
-
-@tables.each_pair do |name, contents|
-  name = name.gsub("_vizlog.tch", "")
-  contents.each_pair do |k, v|
-    key = MessagePack.unpack(k)
-    time = key[0]
-    row = key
-    MessagePack.unpack(v).each{ |val| row << val }
-    if name == "t_table_info.tch" or name == "t_table_schema.tch"
-      #vh.full_info << [0, name, row]
-    else
-      vh.full_info << [time, name, row]
-    end
-  end
+meta, data = get_meta2(BUD_DBM_DIR)
+vh = VizHelper.new(meta[:tabinf], meta[:cycle], meta[:depends], meta[:rules], ARGV[0])
+data.each do |d|
+  vh.full_info << d
 end
 
 vh.tick
-vh.summarize(ARGV[0], schminf)
+vh.summarize(ARGV[0], meta[:schminf])

data/bin/rebl

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+require 'rubygems'
 require 'bud/rebl'
 
 ReblShell::run