bud 0.0.4 → 0.0.5

data/lib/bud/rewrite.rb

@@ -8,11 +8,11 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
     @bud_instance = bud_instance
     @ops = {:<< => 1, :< => 1, :<= => 1}
     @monotonic_whitelist = {
-          :== => 1, :+ => 1, :<= => 1, :- => 1, :< => 1, :> => 1,
-          :* => 1, :pairs => 1, :matches => 1, :combos => 1, :flatten => 1,
-          :lefts => 1, :rights => 1, :map => 1, :flat_map => 1, :pro => 1,
-          :schema => 1, :keys => 1, :values => 1, :payloads => 1, :~ => 1
-      }
+      :== => 1, :+ => 1, :<= => 1, :- => 1, :< => 1, :> => 1,
+      :* => 1, :pairs => 1, :matches => 1, :combos => 1, :flatten => 1,
+      :lefts => 1, :rights => 1, :map => 1, :flat_map => 1, :pro => 1,
+      :schema => 1, :keys => 1, :values => 1, :payloads => 1, :~ => 1
+    }
     @temp_ops = {:-@ => 1, :~ => 1, :+@ => 1}
     @tables = {}
     @nm = false
@@ -115,7 +115,9 @@ class RuleRewriter < Ruby2Ruby # :nodoc: all
     drain(exp)
   end
 
-  # Look for top-level map on a base-table on rhs, and rewrite to pro
+  # We want to rewrite "map" calls on BudCollections to "pro" calls. It is hard
+  # to do this accurately (issue #225), so we just replace map calls liberally
+  # and define Enumerable#pro as an alias for "map".
   def map2pro(exp)
     if exp[1] and exp[1][0] and exp[1][0] == :iter \
       and exp[1][1] and exp[1][1][1] and exp[1][1][1][0] == :call
@@ -152,10 +154,11 @@ class AttrNameRewriter < SexpProcessor # :nodoc: all
       if exp[2] and exp[2][0] == :lasgn and @collnames.size == 1 #single-table iter
         raise Bud::CompileError, "nested redefinition of block variable \"#{exp[2][1]}\" not allowed" if @iterhash[exp[2][1]]
         @iterhash[exp[2][1]] = @collnames[0]
-      elsif exp[2] and exp[2][0] == :lasgn and @collnames.size > 1 # join iter with lefts/rights
-        if exp[1] and exp[1][2] == :lefts
+      elsif exp[2] and exp[2][0] == :lasgn and @collnames.size > 1 and exp[1] # join iter with lefts/rights
+        case exp[1][2]
+        when :lefts
           @iterhash[exp[2][1]] = @collnames[0] 
-        elsif exp[1] and exp[1][2] == :rights
+        when :rights
           @iterhash[exp[2][1]] = @collnames[1]
         else
           raise Bud::CompileError, "nested redefinition of block variable \"#{exp[2][1]}\" not allowed" if @iterhash[exp[2][1]]
@@ -315,6 +318,7 @@ class TempExpander < SexpProcessor # :nodoc: all
     super()
     self.require_empty = false
     self.expected = Sexp
+    @keyword = :temp
 
     @tmp_tables = []
     @did_work = false
@@ -322,7 +326,6 @@ class TempExpander < SexpProcessor # :nodoc: all
 
   def process_defn(exp)
     tag, name, args, scope = exp
-
     if name.to_s =~ /^__bloom__.+/
       block = scope[1]
 
@@ -337,35 +340,41 @@ class TempExpander < SexpProcessor # :nodoc: all
         # correct the misparsing.
         if n.sexp_type == :iter
           iter_body = n.sexp_body
-
-          if iter_body.first.sexp_type == :call
-            call_node = iter_body.first
-
-            _, recv, meth, meth_args = call_node
-            if meth == :temp and recv.nil?
-              _, lhs, op, rhs = meth_args.sexp_body.first
-
-              old_rhs_body = rhs.sexp_body
-              rhs[1] = s(:iter)
-              rhs[1] += old_rhs_body
-              rhs[1] += iter_body[1..-1]
-              block[i] = n = call_node
-              @did_work = true
-            end
+          new_n = fix_temp_decl(iter_body)
+          unless new_n.nil?
+            block[i] = n = new_n
+            @did_work = true
           end
         end
 
         _, recv, meth, meth_args = n
-        if meth == :temp and recv.nil?
-          block[i] = rewrite_temp(n)
+        if meth == @keyword and recv.nil?         
+          block[i] = rewrite_me(n)
           @did_work = true
         end
       end
     end
-
     s(tag, name, args, scope)
   end
 
+  def fix_temp_decl(iter_body)
+    if iter_body.first.sexp_type == :call
+      call_node = iter_body.first
+
+      _, recv, meth, meth_args = call_node
+      if meth == @keyword and recv.nil?
+        _, lhs, op, rhs = meth_args.sexp_body.first
+
+        old_rhs_body = rhs.sexp_body
+        rhs[1] = s(:iter)
+        rhs[1] += old_rhs_body
+        rhs[1] += iter_body[1..-1]
+        return call_node
+      end
+    end
+    return nil
+  end
+
   def get_state_meth(klass)
     return if @tmp_tables.empty?
     block = s(:block)
@@ -375,12 +384,12 @@ class TempExpander < SexpProcessor # :nodoc: all
       block << s(:call, nil, :temp, args)
     end
 
-    meth_name = Module.make_state_meth_name(klass).to_s + "__tmp"
+    meth_name = Module.make_state_meth_name(klass).to_s + "__" + @keyword.to_s
     return s(:defn, meth_name.to_sym, s(:args), s(:scope, block))
   end
 
   private
-  def rewrite_temp(exp)
+  def rewrite_me(exp)
     _, recv, meth, args = exp
 
     raise Bud::CompileError unless recv == nil
@@ -397,6 +406,107 @@ class TempExpander < SexpProcessor # :nodoc: all
   end
 end
 
+# We do four things here for each "with" block
+# 1) Remove it from the AST
+# 2) Use rewrite_me in the parent class to get the collection name pushed onto @tmp_tables.
+# 3) Extract the definition of the "with" collection and push it onto @with_defns
+# 4) Extract the rules in the body of the "with" block and push it onto @with_rules
+
+class WithExpander < TempExpander
+  attr_reader :with_rules, :with_defns
+  def initialize
+    super()
+    @keyword = :with
+    @with_rules = []
+    @with_defns = []
+  end
+
+  def process_defn(exp)
+    tag, name, args, scope = exp
+    if name.to_s =~ /^__bloom__.+/
+      block = scope[1]
+
+      block.each_with_index do |n,i|
+        if i == 0
+          raise Bud::CompileError if n != :block
+          next
+        end
+
+        # temp declarations are misparsed if the RHS contains certain constructs
+        # (e.g., group, "do |f| ... end" rather than "{|f| ... }").  Rewrite to
+        # correct the misparsing.
+        if n.sexp_type == :iter
+          block[i] = nil
+          iter_body = n.sexp_body
+          n = fix_temp_decl(iter_body)
+          @with_defns.push n
+          @did_work = true unless n.nil?
+        end
+
+        _, recv, meth, meth_args = n
+        if meth == @keyword and recv.nil?
+          block[i] = nil
+          n = rewrite_me(n)
+          @with_defns.push n
+          @did_work = true unless n.nil?
+        end
+      end
+    end
+    block.compact! unless block.nil? # remove the nils that got pulled out
+
+    return s(tag, name, args, scope)
+  end
+
+  def get_state_meth(klass)
+    return if @tmp_tables.empty?
+    block = s(:block)
+
+    t = @tmp_tables.pop
+    args = s(:arglist, s(:lit, t.to_sym))
+    block << s(:call, nil, :temp, args)
+
+    meth_name = Module.make_state_meth_name(klass).to_s + "__" + @keyword.to_s
+    return s(:defn, meth_name.to_sym, s(:args), s(:scope, block))
+  end
+  
+  private
+  def rewrite_me(exp)
+    _, recv, meth, args = exp
+
+    raise Bud::CompileError unless recv == nil
+    nest_call = args.sexp_body.first
+    raise Bud::CompileError unless nest_call.sexp_type == :call
+
+    nest_recv, nest_op, nest_args = nest_call.sexp_body
+    raise Bud::CompileError unless nest_recv.sexp_type == :lit
+
+    tmp_name = nest_recv.sexp_body.first
+    @tmp_tables.push tmp_name
+    nest_block = args.sexp_body[1]
+    if nest_block.first == :call
+      # a one-rule block doesn't get wrapped in a block.  wrap it ourselves.
+      nest_block = s(:block, nest_block)
+    end
+    @with_rules.push nest_block
+    new_recv = s(:call, nil, tmp_name, s(:arglist))
+    return s(:call, new_recv, nest_op, nest_args)
+  end  
+  
+  undef get_state_meth
+  
+  public
+  def get_state_meth(klass)
+    return if @tmp_tables.empty?
+    block = s(:block)
+
+    args = s(:arglist, s(:lit, @tmp_tables.pop.to_sym))
+    block << s(:call, nil, :temp, args)
+
+    meth_name = Module.make_state_meth_name(klass).to_s + "__" + @keyword.to_s
+    return s(:defn, meth_name.to_sym, s(:args), s(:scope, block))
+  end
+end
+
 class DefnRenamer < SexpProcessor # :nodoc: all
   def initialize(local_name, rename_tbl)
     super()
@@ -448,10 +558,24 @@ module ModuleRewriter # :nodoc: all
   # the import site. Note that additional rewrites are needed to ensure that
   # code in the import site that accesses module contents does the right thing;
   # see Bud#rewrite_local_methods.
+  
+  @@with_id = 0 # upon initialize
+  def self.with_id
+    @@with_id
+  end
+  
+  def self.incr_with_id
+    @@with_id += 1
+  end
+  
   def self.do_import(import_site, mod, local_name)
-    ast = get_module_ast(mod)
+    # ast_process_withs modifies its argument as a side-effect 
+    # and returns a matching ast.
+    # hence we run it before the other rewrites.
+    ast = ast_process_withs(mod)
     ast = ast_flatten_nested_refs(ast, mod.bud_import_table)
     ast = ast_process_temps(ast, mod)
+    
     ast, new_mod_name = ast_rename_module(ast, import_site, mod, local_name)
     rename_tbl = {}
     ast = ast_rename_methods(ast, local_name, rename_tbl)
@@ -461,7 +585,6 @@ module ModuleRewriter # :nodoc: all
     str = Ruby2Ruby.new.process(ast)
     rv = import_site.module_eval str
     raise Bud::BudError unless rv.nil?
-
     return new_mod_name
   end
 
@@ -486,15 +609,16 @@ module ModuleRewriter # :nodoc: all
   def self.get_raw_parse_tree(klass)
     pt = RawParseTree.new(false)
     klassname = klass.name
+    klassname = klass.to_s if klassname.empty? #("anon_" + Process.pid.to_s + "_" + klass.object_id.to_s) if klassname.empty
     klassname = klassname.to_sym
 
     code = if Class === klass then
-             sc = klass.superclass
-             sc_name = ((sc.nil? or sc.name.empty?) ? "nil" : sc.name).intern
-             [:class, klassname, [:const, sc_name]]
-           else
-             [:module, klassname]
-           end
+      sc = klass.superclass
+      sc_name = ((sc.nil? or sc.name.empty?) ? "nil" : sc.name).intern
+      [:class, klassname, [:const, sc_name]]
+    else
+      [:module, klassname]
+    end
 
     method_names = klass.private_instance_methods false
     # protected methods are included in instance_methods, go figure!
@@ -542,9 +666,63 @@ module ModuleRewriter # :nodoc: all
       # Insert the new extra state method into the module's AST
       ast << new_meth
     end
-
     return ast
   end
+  
+  def self.ast_mangle_with(w,klass)
+    r2r = Ruby2Ruby.new
+    
+    while st = w.get_state_meth(klass)
+      # generate the module
+      tmpmod = Module.new
+
+      # add a state block to define a temp for the collection name
+      state_src = r2r.process(st)        
+      tmpmod.module_eval(state_src)
+
+      # add a bloom block
+      bloom_blk = s(:defn, :__bloom__rules, s(:args), s(:scope, s(:block)))
+      inblk = bloom_blk[3][1]
+
+      # add in the rule that was in the "with" definition
+      newdefn = w.with_defns.pop
+      inblk << newdefn unless newdefn.nil?
+
+      # add in all the rules from the body of the "with" block
+      newrules = w.with_rules.pop
+      newrules.each_with_index do |ast, i|
+        inblk << ast unless i == 0
+      end
+      bloom_src = r2r.process(bloom_blk)
+
+      # eval all that Ruby we generated and import new Module into our code
+      tmpmod.module_eval(bloom_src)
+      modname = "with__"+ModuleRewriter.with_id.to_s
+      klass.import tmpmod => modname.to_sym
+
+      ModuleRewriter.incr_with_id
+    end
+  end
+    
+  def self.ast_process_withs(mod)
+      # strategy to handle withs:
+      # 1) run WithExpander#process to delete the "with" blocks and extract their contents
+      # 2) get the state and rules mangled appropriately into modules
+      # 3) run mod.import on each
+      # 4) call self.get_raw_parse_tree on the result to generate an AST
+
+      ast = get_module_ast(mod)
+      w = WithExpander.new
+      ast = w.process(ast)
+      mod_s, name_s, blocks = ast[0], ast[1], ast[2..-1]
+      tag, name, args, scope = blocks[0]
+      
+      self.ast_mangle_with(w,mod) 
+      
+      retval = Unifier.new.process(self.get_raw_parse_tree(mod))
+      return retval
+      # return s(mod_s, name_s, *blocks)
+  end
 
   # Rename the given module's name to be a mangle of import site, imported
   # module, and local bind name. We also rename all the instance methods defined

data/lib/bud/server.rb

@@ -17,28 +17,31 @@ module Bud
         message_received(obj)
       end
 
+      begin
+        @bud.tick_internal if @bud.running_async
+      rescue Exception
+        # If we raise an exception here, EM dies, which causes problems (e.g.,
+        # other Bud instances in the same process will crash). Ignoring the
+        # error isn't best though -- we should do better (#74).
+        puts "Exception handling network messages: #{$!}"
+        puts "Inbound messages:"
+        @bud.inbound.each do |m|
+          puts "    #{m[1].inspect} (channel: #{m[0]})"
+        end
+        @bud.inbound.clear
+      end
+
       @bud.rtracer.sleep if @bud.options[:rtrace]
     end
 
     def message_received(obj)
-      # puts "#{@bud.ip_port} <= #{obj.inspect}"
-      unless (obj.class <= Array and obj.length == 2 and not
-              @bud.tables[obj[0].to_sym].nil? and obj[1].class <= Array)
+      unless (obj.class <= Array and obj.length == 2 and
+              @bud.tables.include?(obj[0].to_sym) and obj[1].class <= Array)
         raise BudError, "Bad inbound message of class #{obj.class}: #{obj.inspect}"
       end
 
       @bud.rtracer.recv(obj) if @bud.options[:rtrace]
-
       @bud.inbound << obj
-      begin
-        @bud.tick unless @bud.lazy
-      rescue Exception
-        # If we raise an exception here, EM dies, which causes problems (e.g.,
-        # other Bud instances in the same process will crash). Ignoring the
-        # error isn't best though -- we should do better (#74).
-        puts "Exception handling network message (channel '#{obj[0]}'): #{$!}"
-        puts caller.join("\n")
-      end
     end
   end
 end

data/lib/bud/state.rb

@@ -21,7 +21,7 @@ module Bud
       end
     end
   end
-
+  
   public
 	
   def input # :nodoc: all
@@ -38,17 +38,52 @@ module Bud
     scratch(name, schema)
   end
 
-  # declare a persistent collection.  default schema <tt>[:key] => [:val]</tt>
+  # declare an in-memory, non-transient collection.  default schema <tt>[:key] => [:val]</tt>. 
   def table(name, schema=nil)
     define_collection(name)
     @tables[name] = Bud::BudTable.new(name, self, schema)
   end
+  
+  # declare a syncronously-flushed persistent collection.  default schema <tt>[:key] => [:val]</tt>. 
+  def sync(name, storage, schema=nil)
+    define_collection(name)
+    case storage
+    when :dbm
+      @tables[name] = Bud::BudDbmTable.new(name, self, schema)
+      @dbm_tables[name] = @tables[name]
+    when :tokyo
+      @tables[name] = Bud::BudTcTable.new(name, self, schema)
+      @tc_tables[name] = @tables[name]
+    else
+      raise BudError, "Unknown synchronous storage engine #{storage.to_s}"
+    end
+  end
+  
+  def store(name, storage, schema=nil)
+    define_collection(name)
+    case storage
+    when :zookeeper
+      # treat "schema" as a hash of options
+      options = schema
+      raise BudError, "Zookeeper tables require a :path option" if options[:path].nil?
+      options[:addr] ||= "localhost:2181"
+      @tables[name] = Bud::BudZkTable.new(name, options[:path], options[:addr], self)
+      @zk_tables[name] = @tables[name]
+    else
+      raise BudError, "Unknown async storage engine #{storage.to_s}"
+    end
+  end
 
   # declare a transient collection.  default schema <tt>[:key] => [:val]</tt>
   def scratch(name, schema=nil)
     define_collection(name)
     @tables[name] = Bud::BudScratch.new(name, self, schema)
   end
+  
+  def readonly(name, schema=nil)
+    define_collection(name)
+    @tables[name] = Bud::BudReadOnly.new(name, self, schema)
+  end
 
   # declare a scratch in a bloom statement lhs.  schema inferred from rhs.
   def temp(name)
@@ -56,7 +91,7 @@ module Bud
     # defer schema definition until merge
     @tables[name] = Bud::BudTemp.new(name, self, nil, true)
   end
-
+  
   # declare a transient network collection.  default schema <tt>[:address, :val] => []</tt>
   def channel(name, schema=nil, loopback=false)
     define_collection(name)
@@ -84,7 +119,7 @@ module Bud
   def periodic(name, period=1)
     define_collection(name)
     raise BudError if @periodics.has_key? [name]
-    @periodics << [name, gen_id, period]
+    @periodics << [name, period]
     @tables[name] = Bud::BudPeriodic.new(name, self)
   end
 
@@ -98,25 +133,4 @@ module Bud
     @tables[name] = Bud::BudTerminal.new(name, [:line], self)
     @channels[name] = @tables[name]
   end
-
-  # declare a TokyoCabinet table
-  def tctable(name, schema=nil)
-    define_collection(name)
-    @tables[name] = Bud::BudTcTable.new(name, self, schema)
-    @tc_tables[name] = @tables[name]
-  end
-  
-  # declare a dbm table
-  def dbm_table(name, schema=nil)
-    define_collection(name)
-    @tables[name] = Bud::BudDbmTable.new(name, self, schema)
-    @dbm_tables[name] = @tables[name]
-  end
-
-  # declare an Apache ZooKeeper table
-  def zktable(name, path, addr="localhost:2181")
-    define_collection(name)
-    @tables[name] = Bud::BudZkTable.new(name, path, addr, self)
-    @zk_tables[name] = @tables[name]
-  end
 end

data/lib/bud/storage/dbm.rb

@@ -6,11 +6,14 @@ module Bud
     def initialize(name, bud_instance, given_schema)
       dbm_dir = bud_instance.options[:dbm_dir]
       raise BudError, "dbm support must be enabled via 'dbm_dir'" unless dbm_dir
+      if bud_instance.port.nil?
+        raise BudError, "use of dbm storage requires an explicit port to be specified in Bud initialization options"
+      end
+
       unless File.exists?(dbm_dir)
         Dir.mkdir(dbm_dir)
         puts "Created directory: #{dbm_dir}" unless bud_instance.options[:quiet]
       end
-
       dirname = "#{dbm_dir}/bud_#{bud_instance.port}"
       unless File.exists?(dirname)
         Dir.mkdir(dirname)
@@ -81,6 +84,7 @@ module Bud
           each_storage(&block)
         else
           b.each_value do |v|
+            tick_metrics if bud_instance.options[:metrics]
             yield v
           end
         end
@@ -91,6 +95,7 @@ module Bud
       @dbm.each do |k,v|
         k_ary = MessagePack.unpack(k)
         v_ary = MessagePack.unpack(v)
+        tick_metrics if bud_instance.options[:metrics]
         yield make_tuple(k_ary, v_ary)
       end
     end

data/lib/bud/storage/tokyocabinet.rb

@@ -10,6 +10,10 @@ module Bud
     def initialize(name, bud_instance, given_schema)
       tc_dir = bud_instance.options[:tc_dir]
       raise BudError, "TC support must be enabled via 'tc_dir'" unless tc_dir
+      if bud_instance.port.nil?
+        raise BudError, "use of dbm storage requires an explicit port to be specified in Bud initialization options"
+      end
+
       unless File.exists?(tc_dir)
         Dir.mkdir(tc_dir)
         puts "Created directory: #{tc_dir}" unless bud_instance.options[:quiet]
@@ -86,6 +90,7 @@ module Bud
           each_storage(&block)
         else
           b.each_value do |v|
+            tick_metrics if bud_instance.options[:metrics]
             yield v
           end
         end
@@ -96,6 +101,7 @@ module Bud
       @hdb.each do |k,v|
         k_ary = MessagePack.unpack(k)
         v_ary = MessagePack.unpack(v)
+        tick_metrics if bud_instance.options[:metrics]
         yield make_tuple(k_ary, v_ary)
       end
     end

data/lib/bud/storage/zookeeper.rb

@@ -9,7 +9,7 @@ module Bud
   class BudZkTable < BudCollection # :nodoc: all
     def initialize(name, zk_path, zk_addr, bud_instance)
       unless defined? HAVE_ZOOKEEPER
-        raise BudError, "zookeeper gem is not installed: zktables cannot be used"
+        raise BudError, "zookeeper gem is not installed: zookeeper-backed stores cannot be used"
       end
 
       # schema = {[:key] => [:val]}
@@ -109,9 +109,9 @@ module Bud
       }
 
       # If we have new data, force a new Bud tick in the near future
-      if need_tick and not @bud_instance.lazy
+      if need_tick and @bud_instance.running_async
         EventMachine::schedule {
-          @bud_instance.tick
+          @bud_instance.tick_internal
         }
       end
     end
@@ -164,15 +164,15 @@ module Bud
     end
 
     superator "<+" do |o|
-      raise BudError, "Illegal use of <+ with zktable '#{@tabname}' on left"
+      raise BudError, "Illegal use of <+ with zookeeper store '#{@tabname}' on left"
     end
 
     def <=(o)
-      raise BudError, "Illegal use of <= with zktable '#{@tabname}' on left"
+      raise BudError, "Illegal use of <= with zookeeper store '#{@tabname}' on left"
     end
 
     def <<(o)
-      raise BudError, "Illegal use of << with zktable '#{@tabname}' on left"
+      raise BudError, "Illegal use of << with zookeeper store '#{@tabname}' on left"
     end
   end
 end

data/lib/bud/viz.rb

@@ -58,7 +58,11 @@ class VizOnline #:nodoc: all
         row = row[1]
       end
       newrow = [tab, @bud_instance.budtime, row]
-      @logtab << newrow
+      begin
+        @logtab << newrow
+      rescue
+        raise "ERROR!  #{@logtab} << #{newrow}"
+      end
 
     end
   end