bud 0.0.8 → 0.1.0.pre1

data/lib/bud/source.rb

@@ -0,0 +1,109 @@
+require 'rubygems'
+require 'ruby_parser'
+require 'bud/errors'
+
+module Source
+  $cached_file_info = Struct.new(:curr_file, :lines, :last_state_bloom_line).new
+
+  #Reads the block corresponding to the location (string of the form "file:line_num").
+  #Returns an ast for the block
+  def Source.read_block(location)
+    raise Bud::CompileError, "Source must be present in a file; cannot read interactive shell or eval block" if location.start_with? '('
+    location =~ /^(.*):(\d+)/
+    filename, num = $1, $2.to_i
+    raise Bud::Error, "Couldn't determine filename from backtrace" if filename.nil?
+    lines = cache(filename, num)
+    # Note: num is 1-based.
+
+    src_asts = [] # array of SrcAsts to be returned
+    ruby_parser = RubyParser.new
+
+    stmt = ""   # collection of lines that form one complete ruby statement
+    endok = true #
+    ast = nil
+    lines[num .. -1].each do |l|
+      next if l =~ /^\s*#/
+      break if endok and l =~ /^\s*([}]|end)/
+      stmt += l + "\n"
+      begin
+        ast = ruby_parser.parse stmt
+        endok = true
+      rescue => ex
+        #        puts "Syntax Error on #{l}: #{ex}"
+        endok = false
+        ast = nil
+      end
+    end
+    ast
+  end
+
+  def Source.cache(filename, num)  # returns array of lines
+    if $cached_file_info.curr_file == filename
+      retval = $cached_file_info.lines
+      if $cached_file_info.last_state_bloom_line == num
+        # have no use for the cached info any more. reset it.
+        $cached_file_info.lines = []
+        $cached_file_info.curr_file = ""
+        $cached_file_info.last_state_bloom_line = -1
+      end
+    else
+      $cached_file_info.last_state_bloom_line = -1
+      $cached_file_info.curr_file = filename
+      $cached_file_info.lines = []
+      retval = []
+      File.open(filename, "r").each_with_index {|line, i|
+        retval << line
+        if line =~ /^ *(bloom|state)/
+          $cached_file_info.last_state_bloom_line = i
+        end
+      }
+      $cached_file_info.lines = retval
+    end
+    retval # array of lines.
+  end
+
+  # Tok is string tokenizer that extracts a substring matching the
+  # supplied regex, and internally advances past the matched substring.
+  # Leading white space is ignored.
+  # tok = Tok.new("foo 123")
+  # x = tok =~ /\w+/  # => x == 'foo'
+  # y = tok =~ /\d+/  # => y = '123'
+  class Tok
+    attr_accessor :str, :group
+    def initialize(str)
+      @str = str
+      @group = nil
+    end
+
+    # match regex at beginning of string, and advance. Return matched token
+    def =~(regex)
+      s = @str
+      skiplen = 0
+      if s =~ /^\s*/
+        skiplen = $&.length
+        s = s[skiplen .. -1]
+      end
+      if (s =~ regex) == 0
+        # Regexp.last_match is local to this thread and method; squirrel
+        # it away for use in tok.[]
+        @group = Regexp.last_match
+        skiplen += $&.length
+        @str = @str[skiplen .. -1]
+        return $&
+      else
+        nil
+      end
+    end
+
+    # get the nth subgroup match
+    # t = Tok.new("a1122b"); t =~ /a(1+)(2+)b/ ; #=> t[0] =  a1122b; t[1] = 11; t[2] = 22
+    def [](n)
+      @group ? @group[n] : nil
+    end
+    def pushBack(str)
+      @str = str + @str
+    end
+
+    def to_s; @str; end
+  end
+end

data/lib/bud/state.rb

@@ -1,6 +1,5 @@
 module Bud
   ######## methods for registering collection types
-  private
   def define_collection(name)
     if @tables.has_key? name
       raise Bud::CompileError, "collection already exists: #{name}"
@@ -21,8 +20,6 @@ module Bud
     end
   end
 
-  public
-
   def input # :nodoc: all
     true
   end
@@ -33,8 +30,9 @@ module Bud
 
   # declare a transient collection to be an input or output interface
   def interface(mode, name, schema=nil)
+    define_collection(name)
     t_provides << [name.to_s, mode]
-    scratch(name, schema)
+    @tables[name] = (mode ? Bud::BudInputInterface : BudOutputInterface).new(name, self, schema)
   end
 
   # declare an in-memory, non-transient collection.  default schema <tt>[:key] => [:val]</tt>.
@@ -43,16 +41,19 @@ module Bud
     @tables[name] = Bud::BudTable.new(name, self, schema)
   end
 
-  # declare a syncronously-flushed persistent collection.  default schema <tt>[:key] => [:val]</tt>.
+  # declare a collection-generating expression.  default schema <tt>[:key] => [:val]</tt>. 
+  def coll_expr(name, expr, schema=nil)
+    define_collection(name)
+    @tables[name] = Bud::BudCollExpr.new(name, self, expr, schema)
+  end
+  
+  # declare a syncronously-flushed persistent collection.  default schema <tt>[:key] => [:val]</tt>. 
   def sync(name, storage, schema=nil)
     define_collection(name)
     case storage
     when :dbm
       @tables[name] = Bud::BudDbmTable.new(name, self, schema)
       @dbm_tables[name] = @tables[name]
-    when :tokyo
-      @tables[name] = Bud::BudTcTable.new(name, self, schema)
-      @tc_tables[name] = @tables[name]
     else
       raise Bud::Error, "unknown synchronous storage engine #{storage.to_s}"
     end
@@ -84,6 +85,12 @@ module Bud
     @tables[name] = Bud::BudReadOnly.new(name, self, schema)
   end
 
+  def signal(name, schema=nil)
+    define_collection(name)
+    @tables[name] = Bud::BudSignal.new(name, self, schema)
+  end
+
+
   # declare a scratch in a bloom statement lhs.  schema inferred from rhs.
   def temp(name)
     define_collection(name)
@@ -97,7 +104,7 @@ module Bud
     @tables[name] = Bud::BudChannel.new(name, self, schema, loopback)
     @channels[name] = @tables[name]
   end
-
+  
   # declare a transient network collection that delivers facts back to the
   # current Bud instance. This is syntax sugar for a channel that always
   # delivers to the IP/port of the current Bud instance. Default schema

data/lib/bud/storage/dbm.rb

@@ -2,8 +2,9 @@ require 'dbm'
 
 module Bud
   # Persistent table implementation based on ndbm.
-  class BudDbmTable < BudCollection # :nodoc: all
+  class BudDbmTable < BudPersistentCollection # :nodoc: all
     def initialize(name, bud_instance, given_schema)
+      @invalidated = true
       dbm_dir = bud_instance.options[:dbm_dir]
       raise Bud::Error, "dbm support must be enabled via 'dbm_dir'" unless dbm_dir
       if bud_instance.port.nil?
@@ -51,6 +52,10 @@ module Bud
       end
     end
 
+    def length
+      @dbm.length + @delta.length
+    end
+
     def has_key?(k)
       check_enumerable(k)
       key_s = MessagePack.pack(k)
@@ -65,20 +70,25 @@ module Bud
     end
 
     def make_tuple(k_ary, v_ary)
-      t = Array.new(k_ary.length + v_ary.length)
+      #t = Array.new(k_ary.length + v_ary.length)
+      t = @struct.new
       @key_colnums.each_with_index do |k,i|
         t[k] = k_ary[i]
       end
       val_cols.each_with_index do |c,i|
         t[cols.index(c)] = v_ary[i]
       end
-      tuple_accessors(t)
+      t
     end
 
     def each(&block)
       each_from([@delta], &block)
       each_storage(&block)
     end
+    
+    def each_raw(&block)
+      each_storage(&block)
+    end
 
     def each_from(bufs, &block)
       bufs.each do |b|
@@ -130,17 +140,45 @@ module Bud
 
     # move deltas to on-disk storage, and new_deltas to deltas
     def tick_deltas
-      merge_to_db(@delta)
-      @delta = @new_delta
+      unless @delta.empty?
+        merge_to_db(@delta)
+        @tick_delta += @delta.values
+        @delta.clear
+      end
+      unless @new_delta.empty?
+        @delta = @new_delta
+        @new_delta = {}
+      end
+      return !(@delta.empty?)
+    end
+    
+    public
+    def flush_deltas
+      unless @delta.empty?
+        merge_to_db(@delta)
+        @tick_delta += @delta.values
+        @delta.clear
+      end
+      merge_to_db(@new_delta)
       @new_delta = {}
     end
 
+    # This is verbatim from BudTable.  Need to DRY up.  Should we be a subclass of BudTable?
+    public 
+    def pending_delete(o)
+      if o.class <= Bud::PushElement
+         o.wire_to_delete self
+       elsif o.class <= Bud::BudCollection
+         o.pro.wire_to_delete self
+       else
+         @to_delete = @to_delete + o.map{|t| prep_tuple(t) unless t.nil?}
+       end
+    end
     superator "<-" do |o|
-      o.each do |tuple|
-        @to_delete << tuple unless tuple.nil?
-      end
+      pending_delete(o)
     end
-
+    
+    
     def insert(tuple)
       key = get_key_vals(tuple)
       merge_tuple_to_db(key, tuple)
@@ -150,9 +188,7 @@ module Bud
 
     # Remove to_delete and then add pending to db
     def tick
-      raise Bud::Error, "orphaned tuples in @delta for #{@tabname}" unless @delta.empty?
-      raise Bud::Error, "orphaned tuples in @new_delta for #{@tabname}" unless @new_delta.empty?
-
+      deleted = nil
       @to_delete.each do |tuple|
         k = get_key_vals(tuple)
         k_str = MessagePack.pack(k)
@@ -161,18 +197,28 @@ module Bud
           db_cols = MessagePack.unpack(cols_str)
           delete_cols = val_cols.map{|c| tuple[cols.index(c)]}
           if db_cols == delete_cols
-            @dbm.delete k_str
+            deleted ||= @dbm.delete k_str
           end
         end
       end
       @to_delete = []
 
-      merge_to_db(@pending)
-      @pending = {}
-
+      @invalidated = !deleted.nil?
+      unless @pending.empty?
+        @delta = @pending
+#        merge_to_db(@pending)
+        @pending = {}
+      end
       flush
     end
 
+    def invalidate_cache
+    end
+
+    def method_missing(sym, *args, &block)
+      @dbm.send sym, *args, &block
+    end
+
     public
     def length
       @dbm.length

data/lib/bud/storage/zookeeper.rb

@@ -6,7 +6,7 @@ end
 
 module Bud
   # Persistent table implementation based on Zookeeper.
-  class BudZkTable < BudCollection # :nodoc: all
+  class BudZkTable < BudPersistentCollection # :nodoc: all
     def initialize(name, zk_path, zk_addr, bud_instance)
       unless defined? HAVE_ZOOKEEPER
         raise Bud::Error, "zookeeper gem is not installed: zookeeper-backed stores cannot be used"
@@ -94,7 +94,7 @@ module Bud
         data = get_r[:data]
         # XXX: For now, conflate empty string values with nil values
         data ||= ""
-        new_children[c] = tuple_accessors([c, data])
+        new_children[c] = [c, data]
       end
 
       # We successfully fetched all the children of @zk_path; arrange to install

data/lib/bud/viz.rb

@@ -6,7 +6,6 @@ require 'bud/state'
 class VizOnline #:nodoc: all
   def initialize(bud_instance)
     @bud_instance = bud_instance
-    return if bud_instance.class == Stratification or @bud_instance.class == DepAnalysis
     @meta_tables = {'t_rules' => 1, 't_depends' => 1, 't_table_info' => 1, 't_cycle' => 1, 't_stratum' => 1, 't_depends_tc' => 1, 't_table_schema' => 1, 't_provides' => 1}
     @bud_instance.options[:dbm_dir] = "DBM_#{@bud_instance.class}_#{bud_instance.options[:tag]}_#{bud_instance.object_id}_#{bud_instance.port}"
     @table_info = bud_instance.tables[:t_table_info]
@@ -55,10 +54,16 @@ class VizOnline #:nodoc: all
     collection.each do |row|
       if collection.class == Hash
         row = row[1]
-      end
-      if collection.class == Bud::BudPeriodic
+      elsif collection.class == Bud::BudPeriodic
         row = row[0]
       end
+
+      # bud.t_depends and t_rules have bud object in field[0]. Remove them since
+      # bud instances cannot/must not be serialized.
+      if row[0].class <= Bud
+        row = row.to_a if row.class != Array
+        row = row[1..-1] if row[0].class <= Bud
+      end
       newrow = [tab, @bud_instance.budtime, row]
       begin
         @logtab << newrow
@@ -69,7 +74,6 @@ class VizOnline #:nodoc: all
   end
 
   def do_cards
-    return if @bud_instance.class == Stratification or @bud_instance.class == DepAnalysis
     @bud_instance.tables.each do |t|
       tab = t[0]
       next if tab == "the_big_log"

data/lib/bud/viz_util.rb

@@ -80,9 +80,10 @@ module VizUtil #:nodoc: all
     bit = bud_instance.builtin_tables 
     VizUtil.ma_tables.each_pair{|k, v| bit[k] = v}
 
-    write_graphs(tabinf, bit, bud_instance.t_cycle,
+    depanalysis =  bud_instance.meta_parser.depanalysis
+    write_graphs(tabinf, bit, depanalysis.cycle,
                  bud_instance.t_depends, bud_instance.t_rules, viz_name,
-                 output_base, fmt, collapse, bud_instance.meta_parser.depanalysis, -1, nil, 
+                 output_base, fmt, collapse, depanalysis, -1, nil,
                  get_labels(bud_instance), begins)
     begins
   end
@@ -103,7 +104,7 @@ module VizUtil #:nodoc: all
     return {} unless bud_instance.respond_to? :lps
     # sort the paths.  sort the paths to the same destination by length.
     aps = {}
-    ap_interm = bud_instance.lps.sort do |a, b|
+    ap_interm = bud_instance.lps.to_a.sort do |a, b|
       if a.to == b.to then
         a.path.length <=> b.path.length
       else
@@ -175,14 +176,14 @@ module VizUtil #:nodoc: all
     rules = {}
     convertor = Syntax::Convertors::HTML.for_syntax "ruby"
     shredded_rules.each do |s|
-      fout = File.new("#{output_base}/#{s[0]}.html", "w+")
+      fout = File.new("#{output_base}/#{s.rule_id}.html", "w+")
       fout.puts header
-      fout.puts "<h1>Rule #{s[0]}</h1><br>"
+      fout.puts "<h1>Rule #{s.rule_id}</h1><br>"
 
-      c = convertor.convert(s[4])
+      c = convertor.convert(s.orig_src)
       c.sub!(/^<pre>/, "<pre class=\"code\" style='font-size:20px'>\n")
       fout.puts c
-      rules[s[0]] = [s[1], s[4]]
+      rules[s.rule_id] = [s.lhs, s.orig_src]
       fout.close
     end
 
@@ -308,9 +309,9 @@ END_JS
         data << [time, tab, tup]
       end
     end
-    
+
     meta_tabs.each_value do |tab|
-      meta[tab] ||= [] 
+      meta[tab] ||= []
     end
 
     meta[:schminf] = {}