bud 0.0.8 → 0.1.0.pre1

data/README

@@ -10,15 +10,11 @@ documentation.
 
 Main deficiencies at this point are:
 
-- Inefficient evaluation: Programs are run using semi-naive evaluation
-  strategies, but no further query optimization has been implemented, and little
-  effort has been spent in tuning.
-
 - No Ruby constraints: Within Bloom programs the full power of Ruby is also
   available, including mutable state.  This allows programmers to get outside
   the Bloom framework and lose cleanliness.
 
-- Compatibility: Bud only works with Ruby (MRI) 1.8. MRI 1.9, JRuby and other
+- Compatibility: Bud only works with Ruby (MRI) 1.8 and 1.9. JRuby and other
   Ruby implementations are currently not supported.
 
 ## Installation
@@ -39,8 +35,6 @@ To run the unit tests:
 
 ## Optional Dependencies
 
-The bud gem has a handful of mandatory dependencies. It also has two optional
-dependencies: if you wish to use Bud collections backed by Zookeeper or Tokyo
-Cabinet, the "zookeeper" and/or "tokyocabinet" gems must be installed. Note that
-before installing the "tokyocabinet" gem, the Tokyo Cabinet libraries should be
-installed first.
+The bud gem has a handful of mandatory dependencies. It also has one optional
+dependency: if you wish to use Bud collections backed by Zookeeper, the
+"zookeeper" gem must be installed.

data/bin/budplot

@@ -2,7 +2,6 @@
 require 'rubygems'
 require 'bud'
 require 'bud/bud_meta'
-require 'bud/depanalysis'
 require 'bud/graphs'
 require 'bud/meta_algebra'
 require 'bud/viz_util'
@@ -33,7 +32,7 @@ def make_instance(mods)
       if mods.length == 1
         return mod_klass.new
       else
-        puts "Error: cannot intermix classes and modules"
+        puts "Error: cannot intermix class \"#{mod_klass}\" with modules"
         exit
       end
     elsif mod_klass.class != Module

data/docs/cheat.md

@@ -114,13 +114,12 @@ Built-in scratch collection to be used on the lhs of a rule; permanently halts t
 If the item `[:kill]` is inserted, the Bud OS process (including all Bud instances) is also halted.
 
 ### sync ###
-Persistent collection mapped to an external storage engine, with synchronous write-flushing each timestep.  Supported storage engines: `:dbm` and `:tokyo`.<br>
+Persistent collection mapped to an external storage engine, with synchronous write-flushing each timestep.  
 Default attributes: `[:key] => [:val]`.
 
     sync :s1, :dbm
-    sync :s2, :tokyo, [:k1, :k2] => [:v1, :v2]
 
-Further info: [DBM](http://en.wikipedia.org/wiki/Dbm), [Tokyo Cabinet](http://fallabs.com/tokyocabinet/).
+Currently only [dbm](http://en.wikipedia.org/wiki/Dbm) is supported. Support for tokyo cabinet present in an earlier release has been removed.<br>
 
 ### store ###
 Persistent collection mapped to an external storage engine, with asynchronous write-flushing.  Supported storage engines: `:zookeeper`.<br>
@@ -173,8 +172,6 @@ implicit map:
     
 `flat_map`:
 
-    require 'backports' # flat_map not included in Ruby 1.8 by default
-
     t3 <= bc.flat_map do |t| # unnest a collection-valued attribute
       bc.col4.map { |sub| [t.col1, t.col2, t.col3, sub] }
     end
@@ -323,16 +320,6 @@ The schema of a temp collection in inherited from the rhs; if the rhs has no
 schema, a simple one is manufactured to suit the data found in the rhs at
 runtime: `[c0, c1, ...]`.
 
-`with`<br>
-With statements define a temp collection that can be referenced only within the scope of the associated block.  They are useful when you "fork" in a dataflow into two lhs destinations:
-
-    with :biggies <= request {|r| r if r.quantity > 100}, begin
-      to_process <= (biggies * known_good).lefts(:key=>:key)
-      denied <= (biggies * known_good).nopairs(:key=>key)
-    end
-
-The advantage of using `with` over `temp` is modularity: all the rules referencing `biggies` have to be bundled together, making it easier to see that the contents of `request` with quantity > 100 are handled properly.  
-
 ## Bud Modules ##
 A Bud module combines state (collections) and logic (Bloom rules). Using modules allows your program to be decomposed into a collection of smaller units.
 

data/examples/basics/paths.rb

@@ -1,5 +1,5 @@
 # simple shortest paths
-# note use of program.tick at bottom to run a single timestemp 
+# note use of program.tick at bottom to run a single timestep
 # and inspect relations
 require 'rubygems'
 require 'bud'
@@ -9,19 +9,19 @@ class ShortestPaths
 
   state do
     table :link, [:from, :to, :cost]
-    table :path, [:from, :to, :next, :cost]
-    table :shortest, [:from, :to] => [:next, :cost]
+    table :path, [:from, :to, :nxt, :cost]
+    table :shortest, [:from, :to] => [:nxt, :cost]
   end
 
   # recursive rules to define all paths from links
   bloom :make_paths do
     # base case: every link is a path
-    path <= link {|e| [e.from, e.to, e.to, e.cost]}
+    path <= link {|l| [l.from, l.to, l.to, l.cost]}
     
     # inductive case: make path of length n+1 by connecting a link to a path of
     # length n
     path <= (link*path).pairs(:to => :from) do |l,p|
-      [l.from, p.to, p.from, l.cost+p.cost]
+      [l.from, p.to, l.to, l.cost+p.cost]
     end
   end
 
@@ -43,11 +43,11 @@ program.link <= [['a', 'b', 1],
                  ['d', 'e', 1]]
 
 program.tick # one timestamp is enough for this simple program
-program.shortest.sort.each {|t| puts t.inspect}
+program.shortest.to_a.sort.each {|t| puts t.inspect}
 
 puts "----"
 
 # now lets add an extra link and recompute
 program.link << ['e', 'f', 1]
 program.tick
-program.shortest.sort.each {|t| puts t.inspect}
+program.shortest.to_a.sort.each {|t| puts t.inspect}

data/lib/bud/aggs.rb

@@ -11,7 +11,7 @@ module Bud
     #     a. :ignore tells the caller to ignore this input
     #     b. :keep tells the caller to save this input
     #     c. :replace tells the caller to keep this input alone
-    #     d. [:delete, t1, t2, ...] tells the caller to delete the remaining tuples
+    #     d. :delete, [t1, t2, ...] tells the caller to delete the remaining tuples
     #  For things that do not descend from ArgExemplary, the 2nd part can simply be nil.
     def trans(the_state, val)
       return the_state, :ignore
@@ -42,6 +42,8 @@ module Bud
     def trans(the_state, val)
       if the_state < val 
         return the_state, :ignore
+      elsif the_state == val
+        return the_state, :keep
       else 
         return val, :replace
       end
@@ -86,41 +88,35 @@ module Bud
   def choose(x)
     [Choose.new, x]
   end
-  
-  class ChooseRand < ArgExemplary #:nodoc: all
-    @@reservoir_size = 1 # Vitter's reservoir sampling, k = 1
-    def init(x=nil)
-      the_state = {:cnt => 1, :vals => [x]}
+
+  class ChooseOneRand < ArgExemplary #:nodoc: all
+    def init(x=nil) # Vitter's reservoir sampling, sample size = 1
+      the_state = {:cnt => 1, :val => x}
     end
     
     def trans(the_state, val)
       the_state[:cnt] += 1
-      if the_state[:cnt] < @@reservoir_size
-        the_state[:vals] << val
-        return the_state, :keep
+      j = rand(the_state[:cnt])
+      if j < 1 # replace
+        old_val = the_state[:val]
+        the_state[:val] = val
+        return the_state, :replace
       else
-        j = rand(the_state[:cnt])
-        if j < @@reservoir_size
-          old_tup = the_state[:vals][j]
-          the_state[:vals][j] = val 
-          return the_state, [:delete, old_tup]
-        else
-          return the_state, :keep
-        end
+        return the_state, :ignore
       end
     end
     def tie(the_state, val)
       true
     end
     def final(the_state)
-      the_state[:vals][rand(the_state[@@reservoir_size])]
+      the_state[:val] # XXX rand(@@reservoir_size will always be 0, since @@reservoir_size is 1
     end
   end
 
   # exemplary aggregate method to be used in Bud::BudCollection.group.  
   # randomly chooses among x entries being aggregated.
   def choose_rand(x=nil)
-    [ChooseRand.new, x]
+    [ChooseOneRand.new, x]
   end
 
   class Sum < Agg #:nodoc: all

data/lib/bud/bud_meta.rb

@@ -1,66 +1,45 @@
 require 'bud/rewrite'
-require 'bud/state'
-require 'parse_tree'
 require 'pp'
 
-class BudMeta #:nodoc: all
-  attr_reader :depanalysis
 
+class BudMeta #:nodoc: all
   def initialize(bud_instance, declarations)
     @bud_instance = bud_instance
     @declarations = declarations
+    @dependency_analysis = nil # the results of bud_meta are analyzed further using a helper bloom instance. See depanalysis())
   end
 
   def meta_rewrite
-    shred_rules
-    top_stratum = stratify
-    stratum_map = binaryrel2map(@bud_instance.t_stratum)
-
-    rewritten_strata = Array.new(top_stratum + 2) { [] }
-    no_attr_rewrite_strata = Array.new(top_stratum + 2) { [] }
-    @bud_instance.t_rules.each do |d|
-      if d.op.to_s == '<='
-        # Deductive rules are assigned to strata based on the basic Datalog
-        # stratification algorithm
-        belongs_in = stratum_map[d.lhs]
-        belongs_in ||= 0
-        rewritten_strata[belongs_in] << d.src
-        no_attr_rewrite_strata[belongs_in] << d.orig_src
-      else
-        # All temporal rules are placed in the last stratum
-        rewritten_strata[top_stratum + 1] << d.src
-        no_attr_rewrite_strata[top_stratum + 1] << d.orig_src
-      end
-    end
+    shred_rules # capture dependencies, rewrite rules
 
-    @depanalysis = DepAnalysis.new
-    @bud_instance.t_depends_tc.each {|d| @depanalysis.depends_tc << d}
-    @bud_instance.t_provides.each {|p| @depanalysis.providing << p}
-    3.times { @depanalysis.tick_internal }
+    stratified_rules = []
+    if @bud_instance.toplevel == @bud_instance
+      nodes, stratum_map, top_stratum = stratify_preds
 
-    @depanalysis.underspecified.each do |u|
-      puts "Warning: underspecified dataflow: #{u.inspect}"
-      @bud_instance.t_underspecified << u
-    end
-    @depanalysis.source.each do |s|
-      @bud_instance.sources[s.first] = true
-    end
-    @depanalysis.sink.each do |s|
-      @bud_instance.sinks[s.first] = true
-    end
-      
-    dump_rewrite(no_attr_rewrite_strata) if @bud_instance.options[:dump_rewrite]
+      # stratum_map = {fully qualified pred  => stratum}
 
-    return rewritten_strata, no_attr_rewrite_strata
-  end
+      #slot each rule into the stratum corresponding to its lhs pred (from stratum_map)
+      stratified_rules = Array.new(top_stratum + 2) { [] }  # stratum -> [ rules ]
+      @bud_instance.t_rules.each do |rule|
+        if rule.op.to_s == '<='
+          # Deductive rules are assigned to strata based on the basic Datalog
+          # stratification algorithm
+          belongs_in = stratum_map[rule.lhs]
+          belongs_in ||= 0
+          stratified_rules[belongs_in] << rule
+        else
+          # All temporal rules are placed in the last stratum
+          stratified_rules[top_stratum + 1] << rule
+        end
+      end
+      # stratified_rules[0] may be empty if none of the nodes at stratum 0 are on the lhs
+      # stratified_rules[top_stratum+1] will be empty if there are no temporal rules.
+      # Cleanup
+      stratified_rules = stratified_rules.reject{|r| r.empty?}
+      dump_rewrite(stratified_rules) if @bud_instance.options[:dump_rewrite]
 
-  def binaryrel2map(rel)
-    map = {}
-    rel.each do |s|
-      raise Bud::Error unless s.length == 2
-      map[s[0]] = s[1]
     end
-    return map
+    return stratified_rules
   end
 
   def shred_rules
@@ -87,11 +66,18 @@ class BudMeta #:nodoc: all
   end
 
   def rewrite_rule_block(klass, block_name, seed)
-    parse_tree = ParseTree.translate(klass, block_name)
-    return unless parse_tree.first
+    return unless klass.respond_to? :__bloom_asts__
 
-    pt = Unifier.new.process(parse_tree)
+    pt = klass.__bloom_asts__[block_name]
+    return if pt.nil?
+
+    pt = Marshal.load(Marshal.dump(pt)) #deep clone because RuleRewriter mucks up pt.
     pp pt if @bud_instance.options[:dump_ast]
+    tmp_expander = TempExpander.new
+    pt = tmp_expander.process(pt)
+    tmp_expander.tmp_tables.each do |t|
+      @bud_instance.temp(t.to_sym)
+    end
 
     rv = check_rule_ast(pt)
     unless rv.nil?
@@ -112,12 +98,30 @@ class BudMeta #:nodoc: all
       end
       raise Bud::CompileError, "#{error_msg} in rule block \"#{block_name}\"#{src_msg}"
     end
-
     rewriter = RuleRewriter.new(seed, @bud_instance)
     rewriter.process(pt)
     return rewriter
   end
 
+  def get_qual_name(pt)
+    # expect to see a parse tree corresponding to a dotted name
+    #    a.b.c == s(:call, s1, :c, (:args))
+    # where s1 == s(:call, s2, :b, (:args))
+    # where s2 == s(:call, nil,:a, (:args))
+
+    tag, recv, name, args = pt
+    return nil unless tag == :call or args.length == 1
+
+    if recv
+      qn = get_qual_name(recv)
+      return nil if qn.nil? or qn.size == 0
+      qn = qn + "." + name.to_s
+    else
+      qn = name.to_s
+    end
+    qn
+  end
+
   # Perform some basic sanity checks on the AST of a rule block. We expect a
   # rule block to consist of a :defn, a nested :scope, and then a sequence of
   # statements. Each statement is a :call node. Returns nil (no error found), a
@@ -147,10 +151,9 @@ class BudMeta #:nodoc: all
       tag, lhs, op, rhs = n
 
       # Check that LHS references a named collection
-      return n if lhs.nil? or lhs.sexp_type != :call
-      lhs_name = lhs[2].to_sym
-      unless @bud_instance.tables.has_key? lhs_name
-        return [n, "collection does not exist: '#{lhs_name}'"]
+      lhs_name = get_qual_name(lhs)
+      unless lhs_name and @bud_instance.tables.has_key? lhs_name.to_sym
+        return [n, "Collection does not exist: '#{lhs_name}'"]
       end
 
       return [n, "illegal operator: '#{op}'"] unless [:<, :<=].include? op
@@ -176,36 +179,121 @@ class BudMeta #:nodoc: all
     return nil # No errors found
   end
 
-  def stratify
-    strat = Stratification.new
-    @bud_instance.t_depends.each {|d| strat.depends << d}
-    strat.tick_internal
-
-    # Copy computed data back into Bud runtime
-    @bud_instance.stratum_collection_map = []
-    strat.stratum.each do |s|
-      @bud_instance.t_stratum << s
-      @bud_instance.stratum_collection_map[s[1]] ||= []
-      @bud_instance.stratum_collection_map[s[1]] << s[0].to_sym
+
+  Node = Struct.new :name, :status, :stratum, :edges, :in_lhs, :in_body, :in_cycle, :is_neg_head
+  # Node.status is one of :init, :in_progress, :done
+  Edge = Struct.new :to, :op, :neg, :temporal
+
+  def stratify_preds
+    bud = @bud_instance.toplevel
+    nodes = {}
+    bud.t_depends.each do |d|
+      #t_depends [:bud_instance, :rule_id, :lhs, :op, :body] => [:nm]
+      lhs = (nodes[d.lhs.to_s] ||= Node.new(d.lhs.to_s, :init, 0, [], true, false, false, false))
+      lhs.in_lhs = true
+      body = (nodes[d.body.to_s] ||= Node.new(d.body.to_s, :init, 0, [], false, true, false, false))
+      temporal = d.op != "<=" 
+      lhs.edges << Edge.new(body, d.op, d.nm, temporal)
+      body.in_body = true
     end
-    strat.depends_tc.each {|d| @bud_instance.t_depends_tc << d}
-    strat.cycle.each {|c| @bud_instance.t_cycle << c}
-    if strat.top_strat.length > 0
-      top = strat.top_strat.first.stratum
-    else
-      top = 1
+
+    nodes.values.each {|n| calc_stratum(n, false, false, [n.name])}
+    # Normalize stratum numbers because they may not be 0-based or consecutive
+    remap = {}
+    # if the nodes stratum numbers are [2, 3, 2, 4], remap = {2 => 0, 3 => 1, 4 => 2} 
+    nodes.values.map {|n| n.stratum}.uniq.sort.each_with_index{|num, i|
+      remap[num] = i
+    }
+    stratum_map = {}
+    top_stratum = -1
+    nodes.each_pair do |name, n|
+      n.stratum = remap[n.stratum]
+      stratum_map[n.name] = n.stratum
+      top_stratum = max(top_stratum, n.stratum)
     end
+    analyze_dependencies(nodes)
+    return nodes, stratum_map, top_stratum
+  end
 
-    return top
+  def max(a, b) ; a > b ? a : b ; end
+
+  def calc_stratum(node, neg, temporal, path)
+    if node.status == :in_process
+      node.in_cycle = true
+      if neg and !temporal and node.is_neg_head
+        raise Bud::CompileError, "unstratifiable program: #{path.uniq.join(',')}"
+      end
+    elsif node.status == :init
+      node.status = :in_process
+      node.edges.each do |edge|
+        node.is_neg_head = edge.neg
+        next if edge.op != "<="
+        body_stratum = calc_stratum(edge.to, (neg or edge.neg), (edge.temporal or temporal), path + [edge.to.name])
+        node.is_neg_head = false #reset for next edge
+        node.stratum = max(node.stratum, body_stratum + (edge.neg ? 1 : 0))
+      end
+      node.status = :done
+    end
+    node.stratum
+  end
+
+
+  def analyze_dependencies(nodes)  # nodes = {node name => node}
+    bud = @bud_instance
+                                                      
+    preds_in_lhs = nodes.inject(Set.new) {|preds, name_n| preds.add(name_n[0]) if name_n[1].in_lhs; preds}
+    preds_in_body = nodes.inject(Set.new) {|preds, name_n| preds.add(name_n[0]) if name_n[1].in_body; preds}
+
+    bud.t_provides.each do |p|
+      pred, input = p.interface, p.input
+      if input
+        # an interface pred is a source if it is an input and it is not in any rule's lhs
+        #bud.sources << [pred] unless (preds_in_lhs.include? pred)
+        unless preds_in_body.include? pred.to_s
+          # input interface is underspecified if not used in any rule body
+          bud.t_underspecified << [pred, true] # true indicates input mode
+          puts "Warning: input interface #{pred} not used"
+        end
+      else
+        # an interface pred is a sink if it is not an input and it is not in any rule's body
+        #(if it is in the body, then it is an intermediate node feeding some lhs)
+        #bud.sinks << [pred] unless (preds_in_body.include? pred)
+        unless preds_in_lhs.include? pred.to_s
+          # output interface underspecified if not in any rule's lhs
+          bud.t_underspecified << [pred, false]  #false indicates output mode.
+          puts "Warning: output interface #{pred} not used"
+        end
+      end
+    end
+  end
+
+  def depanalysis
+    if @dependency_analysis.nil?
+      require 'bud/depanalysis'
+      da = ::DepAnalysis.new
+      da.providing <+ @bud_instance.tables[:t_provides].to_a
+      da.depends <+ @bud_instance.t_depends.map{|d| [d.lhs, d.op, d.body, d.nm]}
+
+      #@bud_instance.tables[:t_provides].each {|t| da.providing <+ t}
+      #@bud_instance.tables[:t_depends].each {|t| da.depends_tc <+ t}
+      da.tick_internal
+      @dependency_analysis = da
+    end
+    @dependency_analysis
   end
 
   def dump_rewrite(strata)
     fout = File.new("#{@bud_instance.class}_rewritten.txt", "w")
     fout.puts "Declarations:"
 
-    strata.each_with_index do |src_ary, i|
-      text = src_ary.join("\n")
-      fout.puts "R[#{i}]:\n#{text}"
+    strata.each_with_index do |rules, i|
+      fout.print "=================================\n"
+      fout.print "Stratum #{i}\n"
+      rules.each do |r|
+        fout.puts "#{r.bud_obj.class}##{r.bud_obj.object_id} #{r.rule_id}"
+        fout.puts "\tsrc:      #{r.src}"
+        fout.puts "\torig src: #{r.orig_src}"
+      end
     end
     fout.close
   end

data/lib/bud/bust/bust.rb

@@ -68,23 +68,30 @@ module Bust
           if @request =~ /GET .* HTTP*/
             puts "GET shouldn't have body" if @body
             # select the appropriate elements from the table
-            desired_elements = (eval "@bud." + table_name).find_all do |t|
-              uri_params.all? {|k, v| (eval "t." + k.to_s) == v[0]}
+            desired_elements = @bud.send(table_name.to_sym).find_all do |t|
+              uri_params.all? {|k, v|
+                (eval "t." + k.to_s) == v[0]
+              }
             end
             @session.print success
+            desired_elements = desired_elements .map{|elem|
+              elem.class <= Struct ? elem.to_a : elem
+            }
             @session.print desired_elements.to_json
           elsif @request =~ /POST .* HTTP*/
             # instantiate a new tuple
             tuple_to_insert = []
             @body.each do |k, v|
-              index = (eval "@bud." + table_name).cols.find_index(k.to_sym)
+              index = @bud.send(table_name.to_sym).cols.find_index(k.to_sym)
               for i in (tuple_to_insert.size..index)
                 tuple_to_insert << nil
               end
               tuple_to_insert[index] = v[0]
             end
             # actually insert the puppy
-            @bud.async_do { (eval "@bud." + table_name) <+ [tuple_to_insert] }
+            @bud.async_do {
+              @bud.send(table_name.to_sym) <+ [tuple_to_insert]
+            }
             @session.print success
           end
         rescue Exception