bud 0.9.4 → 0.9.5

data/lib/bud/server.rb

@@ -55,11 +55,22 @@ class Bud::BudServer < EM::Connection #:nodoc: all
   end
 
   def message_received(obj)
-    unless (obj.class <= Array and obj.length == 2 and
-            @bud.tables.include?(obj[0].to_sym) and obj[1].class <= Array)
+    unless (obj.class <= Array and obj.length == 3 and
+            @bud.tables.include?(obj[0].to_sym) and
+            obj[1].class <= Array and obj[2].class <= Array)
       raise Bud::Error, "bad inbound message of class #{obj.class}: #{obj.inspect}"
     end
 
+    # Deserialize any nested marshalled values
+    tbl_name, tuple, marshall_indexes = obj
+    marshall_indexes.each do |i|
+      if i < 0 || i >= tuple.length
+        raise Bud::Error, "bad inbound message: marshalled value at index #{i}, #{obj.inspect}"
+      end
+      tuple[i] = Marshal.load(tuple[i])
+    end
+
+    obj = [tbl_name, tuple]
     @bud.rtracer.recv(obj) if @bud.options[:rtrace]
     @filter_buf[obj[0].to_sym] ||= []
     @filter_buf[obj[0].to_sym] << obj[1]

data/lib/bud/source.rb

@@ -1,6 +1,4 @@
 require 'rubygems'
-require 'ruby_parser'
-require 'bud/errors'
 
 module Source
   $cached_file_info = Struct.new(:curr_file, :lines, :last_state_bloom_line).new
@@ -8,35 +6,53 @@ module Source
   # Reads the block corresponding to the location (string of the form
   # "file:line_num").  Returns an ast for the block.
   def Source.read_block(location)
-    raise Bud::IllegalSourceError, "source must be present in a file; cannot read interactive shell or eval block" if location.start_with? '('
+    if location.start_with? '('
+      raise Bud::IllegalSourceError, "source must be present in a file; cannot read interactive shell or eval block"
+    end
     location =~ /^(.*):(\d+)/
     filename, num = $1, $2.to_i
-    raise Bud::IllegalSourceError, "couldn't determine filename from backtrace" if filename.nil?
+    if filename.nil?
+      raise Bud::IllegalSourceError, "couldn't determine filename from backtrace"
+    end
     lines = cache(filename, num)
     # Note: num is 1-based.
 
-    src_asts = [] # array of SrcAsts to be returned
-    ruby_parser = RubyParser.new
-
-    stmt = ""   # collection of lines that form one complete ruby statement
-    endok = true #
+    parser = make_parser
+    stmt = ""       # collection of lines that form one complete Ruby statement
     ast = nil
     lines[num .. -1].each do |l|
       next if l =~ /^\s*#/
-      break if endok and l =~ /^\s*([}]|end)/
-      stmt += l + "\n"
-      begin
-        ast = ruby_parser.parse stmt
-        endok = true
-      rescue => ex
-        #        puts "Syntax Error on #{l}: #{ex}"
-        endok = false
-        ast = nil
+      if l =~ /^\s*([}]|end)/
+        # We found some syntax that looks like it might terminate the Ruby
+        # statement. Hence, try to parse it; if we don't find a syntax error,
+        # we're done.
+        begin
+          ast = parser.parse stmt
+          break
+        rescue
+          ast = nil
+        end
       end
+      stmt += l + "\n"
     end
     ast
   end
 
+  # ruby_parser 3.x can produce either 1.8- or 1.9-compatible ASTs. Since we
+  # want to eventually turn the ASTs back into code we can eval() using the
+  # current Ruby runtime, we want to generate an appropriately compatible AST.
+  def Source.make_parser
+    maj, min, patch = RUBY_VERSION.split(".")
+    case [maj.to_i, min.to_i]
+    when [1, 9] then
+      Ruby19Parser.new
+    when [1, 8] then
+      Ruby18Parser.new
+    else
+      raise Bud::Error, "unrecognized RUBY_VERSION: #{RUBY_VERSION}"
+    end
+  end
+
   def Source.cache(filename, num)  # returns array of lines
     if $cached_file_info.curr_file == filename
       retval = $cached_file_info.lines

data/lib/bud/state.rb

@@ -2,7 +2,7 @@ module Bud
   ######## methods for registering collection types
   private
   def check_collection_name(name)
-    if @tables.has_key? name
+    if @tables.has_key? name or @lattices.has_key? name
       raise Bud::CompileError, "collection already exists: #{name}"
     end
 
@@ -26,6 +26,18 @@ module Bud
     end
   end
 
+  def define_lattice(name)
+    check_collection_name(name)
+
+    self.singleton_class.send(:define_method, name) do |*args, &blk|
+      if blk.nil?
+        return @lattices[name]
+      else
+        return @lattices[name].pro(&blk)
+      end
+    end
+  end
+
   public
   def input # :nodoc: all
     true
@@ -140,4 +152,81 @@ module Bud
     @tables[name] = Bud::BudTerminal.new(name, [:line], self)
     @channels[name] = @tables[name]
   end
+
+  # an alternative approach to declaring interfaces
+  def interfaces(direction, collections)
+    mode = case direction
+      when :input then true
+      when :output then false
+    else
+      raise Bud::CompileError, "unrecognized interface type #{direction}"
+    end
+    collections.each do |tab|
+      t_provides << [tab.to_s, mode]
+    end 
+  end
+
+  # Define methods to implement the state declarations for every registered kind
+  # of lattice.
+  def load_lattice_defs
+    Bud::Lattice.global_mfuncs.each do |m|
+      next if RuleRewriter::MONOTONE_WHITELIST.include? m
+      if Bud::BudCollection.instance_methods.include? m.to_s
+        puts "monotone method #{m} conflicts with non-monotonic method in BudCollection"
+      end
+    end
+
+    Bud::Lattice.global_morphs.each do |m|
+      next if RuleRewriter::MONOTONE_WHITELIST.include? m
+      if Bud::BudCollection.instance_methods.include? m.to_s
+        puts "morphism #{m} conflicts with non-monotonic method in BudCollection"
+      end
+    end
+
+    # Sanity-check lattice definitions
+    # XXX: We should do this only once per lattice
+    Bud::Lattice.lattice_kinds.each do |wrap_name, klass|
+      unless klass.method_defined? :merge
+        raise Bud::CompileError, "lattice #{wrap_name} does not define a merge function"
+      end
+
+      # If a method is marked as monotone in any lattice, every lattice that
+      # declares a method of that name must also mark it as monotone.
+      meth_list = klass.instance_methods(false).to_set
+      Bud::Lattice.global_mfuncs.each do |m|
+        next unless meth_list.include? m.to_s
+        unless klass.mfuncs.include? m
+          raise Bud::CompileError, "method #{m} in #{wrap_name} must be monotone"
+        end
+      end
+
+      # Apply a similar check for morphs
+      Bud::Lattice.global_morphs.each do |m|
+        next unless meth_list.include? m.to_s
+        unless klass.morphs.include? m
+          raise Bud::CompileError, "method #{m} in #{wrap_name} must be a morph"
+        end
+      end
+
+      # Similarly, check for non-monotone lattice methods that are found in the
+      # builtin list of monotone operators. The "merge" method is implicitly
+      # monotone (XXX: should it be declared as a morph or monotone function?)
+      meth_list.each do |m_str|
+        m = m_str.to_sym
+        next unless RuleRewriter::MONOTONE_WHITELIST.include? m
+        # XXX: ugly hack. We want to allow lattice class implementations to
+        # define their own equality semantics.
+        next if m == :==
+        unless klass.mfuncs.include?(m) || klass.morphs.include?(m) || m == :merge
+          raise Bud::CompileError, "method #{m} in #{wrap_name} must be monotone"
+        end
+      end
+
+      # XXX: replace "self" with toplevel?
+      self.singleton_class.send(:define_method, wrap_name) do |lat_name|
+        define_lattice(lat_name)
+        @lattices[lat_name] = Bud::LatticeWrapper.new(lat_name, klass, self)
+      end
+    end
+  end
 end

data/lib/bud/storage/zookeeper.rb

@@ -8,40 +8,63 @@ module Bud
   # Persistent table implementation based on Zookeeper.
   class BudZkTable < BudPersistentCollection # :nodoc: all
     def initialize(name, zk_path, zk_addr, bud_instance)
-      unless defined? HAVE_ZOOKEEPER
+      unless defined? Bud::HAVE_ZOOKEEPER
         raise Bud::Error, "zookeeper gem is not installed: zookeeper-backed stores cannot be used"
       end
 
-      # schema = {[:key] => [:val]}
-      super(name, bud_instance, nil)
+      super(name, bud_instance, [:key] => [:val, :opts])
 
-      zk_path = zk_path.chomp("/") unless zk_path == "/"
       @zk = Zookeeper.new(zk_addr)
+      zk_path = zk_path.chomp("/") unless zk_path == "/"
       @zk_path = zk_path
       @base_path = @zk_path
       @base_path += "/" unless @zk_path.end_with? "/"
       @store_mutex = Mutex.new
+      @zk_mutex = Mutex.new
       @next_storage = {}
       @saw_delta = false
       @child_watch_id = nil
-      @stat_watch_id = nil
     end
 
     # Since the watcher callbacks might invoke EventMachine, we wait until after
     # EM startup to start watching for Zk events.
     def start_watchers
-      # NB: Watcher callbacks are invoked in a separate Ruby thread.
-      @child_watcher = Zookeeper::WatcherCallback.new { get_and_watch }
-      @stat_watcher = Zookeeper::WatcherCallback.new { stat_and_watch }
+      # Watcher callbacks are invoked in a separate Ruby thread. Note that there
+      # is a possible deadlock between invoking watcher callbacks and calling
+      # close(): if we get a watcher event and a close at around the same time,
+      # the close might fire first. Closing the Zk handle will block on
+      # dispatching outstanding watchers, but it does so holding the @zk_mutex,
+      # causing a deadlock. Hence, we just have the watcher callback spin on the
+      # @zk_mutex, aborting if the handle is ever closed.
+      @child_watcher = Zookeeper::Callbacks::WatcherCallback.new do
+        while true
+          break if @zk.closed?
+          if @zk_mutex.try_lock
+            get_and_watch unless @zk.closed?
+            @zk_mutex.unlock
+            break
+          end
+        end
+      end
+
+      @stat_watcher = Zookeeper::Callbacks::WatcherCallback.new do
+        while true
+          break if @zk.closed?
+          if @zk_mutex.try_lock
+            stat_and_watch unless @zk.closed?
+            @zk_mutex.unlock
+            break
+          end
+        end
+      end
+
       stat_and_watch
     end
 
     def stat_and_watch
       r = @zk.stat(:path => @zk_path, :watcher => @stat_watcher)
-      @stat_watch_id = r[:req_id]
 
       unless r[:stat].exists
-        cancel_child_watch
         # The given @zk_path doesn't exist, so try to create it. Unclear
         # whether this is always the best behavior.
         r = @zk.create(:path => @zk_path)
@@ -54,27 +77,10 @@ module Bud
       get_and_watch unless @child_watch_id
     end
 
-    def cancel_child_watch
-      if @child_watch_id
-        @zk.unregister_watcher(@child_watch_id)
-        @child_watch_id = nil
-      end
-    end
-
-    def cancel_stat_watch
-      if @stat_watch_id
-        @zk.unregister_watcher(@stat_watch_id)
-        @stat_with_id = nil
-      end
-    end
-
     def get_and_watch
       r = @zk.get_children(:path => @zk_path, :watcher => @child_watcher)
+      return unless r[:stat].exists
       @child_watch_id = r[:req_id]
-      unless r[:stat].exists
-        cancel_child_watch
-        return
-      end
 
       # XXX: can we easily get snapshot isolation?
       new_children = {}
@@ -128,8 +134,8 @@ module Bud
         ephemeral = false
         sequence = false
 
-        if t.length > 2
-          opts = t.last.first
+        opts = t.opts
+        unless opts.nil?
           if opts[:ephemeral] == true
             ephemeral = true
           end
@@ -150,9 +156,8 @@ module Bud
     end
 
     def close
-      cancel_child_watch
-      cancel_stat_watch
-      @zk.close
+      # See notes in start_watchers.
+      @zk_mutex.synchronize { @zk.close }
     end
 
     superator "<~" do |o|

data/lib/bud/viz.rb

@@ -1,5 +1,4 @@
 require 'bud/state'
-require 'set'
 
 class VizOnline #:nodoc: all
   attr_reader :logtab

data/lib/bud.rb

@@ -1,16 +1,27 @@
 require 'rubygems'
+gem 'ruby2ruby', '>= 2.0.1'
+gem 'ruby_parser', '>= 3.0.2'
+
 require 'eventmachine'
 require 'msgpack'
+require 'ruby2ruby'
+require 'ruby_parser'
+require 'set'
 require 'socket'
 require 'superators19'
 require 'thread'
-require 'bud/errors'
 
+require 'bud/errors'
 require 'bud/monkeypatch'
 
 require 'bud/aggs'
 require 'bud/bud_meta'
 require 'bud/collections'
+require 'bud/executor/elements.rb'
+require 'bud/executor/group.rb'
+require 'bud/executor/join.rb'
+require 'bud/lattice-core'
+require 'bud/lattice-lib'
 require 'bud/metrics'
 require 'bud/rtrace'
 require 'bud/server'
@@ -19,10 +30,6 @@ require 'bud/storage/dbm'
 require 'bud/storage/zookeeper'
 require 'bud/viz'
 
-require 'bud/executor/elements.rb'
-require 'bud/executor/group.rb'
-require 'bud/executor/join.rb'
-
 ILLEGAL_INSTANCE_ID = -1
 SIGNAL_CHECK_PERIOD = 0.2
 
@@ -61,7 +68,7 @@ $bud_instances = {}        # Map from instance id => Bud instance
 # :main: Bud
 module Bud
   attr_reader :budtime, :inbound, :options, :meta_parser, :viz, :rtracer, :dsock
-  attr_reader :tables, :builtin_tables, :channels, :zk_tables, :dbm_tables, :app_tables
+  attr_reader :tables, :builtin_tables, :channels, :zk_tables, :dbm_tables, :app_tables, :lattices
   attr_reader :push_sources, :push_elems, :push_joins, :scanners, :merge_targets
   attr_reader :this_stratum, :this_rule, :rule_orig_src, :done_bootstrap
   attr_accessor :stratified_rules
@@ -109,6 +116,7 @@ module Bud
     options[:print_wiring] ||= ENV["BUD_PRINT_WIRING"].to_i > 0
     @qualified_name = ""
     @tables = {}
+    @lattices = {}
     @channels = {}
     @dbm_tables = {}
     @zk_tables = {}
@@ -144,6 +152,7 @@ module Bud
     # NB: If using an ephemeral port (specified by port = 0), the actual port
     # number won't be known until we start EM
 
+    load_lattice_defs
     builtin_state
     resolve_imports
     call_state_methods
@@ -248,6 +257,11 @@ module Bud
           tables[qname.to_sym] = t
         end
       end
+      mod_inst.lattices.each_pair do |name, t|
+        qname = "#{local_name}.#{name}".to_sym
+        raise Bud::Error if lattices.has_key? qname
+        lattices[qname] = t
+      end
       mod_inst.t_rules.each do |imp_rule|
         qname = "#{local_name}.#{imp_rule.lhs}"
         self.t_rules << [imp_rule.bud_obj, imp_rule.rule_id, qname, imp_rule.op,
@@ -257,7 +271,7 @@ module Bud
         qlname = "#{local_name}.#{imp_dep.lhs}"
         qrname = "#{local_name}.#{imp_dep.body}"
         self.t_depends << [imp_dep.bud_obj, imp_dep.rule_id, qlname,
-                           imp_dep.op, qrname, imp_dep.nm]
+                           imp_dep.op, qrname, imp_dep.nm, imp_dep.in_body]
       end
       mod_inst.t_provides.each do |imp_pro|
         qintname = "#{local_name}.#{imp_pro.interface}"
@@ -309,9 +323,10 @@ module Bud
     @stratified_rules.each_with_index { |rules, stratum| eval_rules(rules, stratum) }
 
     # Prepare list of tables that will be actively used at run time. First, all
-    # the user-defined ones.  We start @app_tables off as a set, then convert to
-    # an array later.
+    # the user-defined tables and lattices.  We start @app_tables off as a set,
+    # then convert to an array later.
     @app_tables = (@tables.keys - @builtin_tables.keys).map {|t| @tables[t]}.to_set
+    @app_tables += @lattices.values
 
     # Check scan and merge_targets to see if any builtin_tables need to be added as well.
     @scanners.each do |scs|
@@ -335,7 +350,7 @@ module Bud
         wired_to = []
         working.each do |e|
           e.wirings.each do |out|
-            if (out.class <= PushElement and not seen.member?(out))
+            if ((out.class <= PushElement || out.class <= LatticePushElement) and not seen.member?(out))
               seen << out
               wired_to << out
             end
@@ -419,7 +434,7 @@ module Bud
   def prepare_invalidation_scheme
     num_strata = @push_sorted_elems.size
     if $BUD_SAFE
-      @app_tables = @tables.values # No tables excluded
+      @app_tables = @tables.values + @lattices.values # No collections excluded
 
       rescan = Set.new
       invalidate = @app_tables.select {|t| t.class <= BudScratch}.to_set
@@ -445,7 +460,10 @@ module Bud
     nm_targets = Set.new
     t_rules.each do |rule|
       lhs = rule.lhs.to_sym
-      @tables[lhs].is_source = false if rule.op == "<="
+      if rule.op == "<="
+        # Note that lattices cannot be sources
+        @tables[lhs].is_source = false if @tables.has_key? lhs
+      end
       nm_targets << lhs if rule.nm_funcs_called
     end
 
@@ -459,7 +477,7 @@ module Bud
       @push_sorted_elems[stratum].each do |elem|
         rescan << elem if elem.rescan_at_tick
 
-        if elem.outputs.any?{|tab| not(tab.class <= PushElement) and nm_targets.member? tab.qualified_tabname.to_sym }
+        if elem.outputs.any?{|tab| not(tab.class <= PushElement) and not(tab.class <= LatticePushElement) and nm_targets.member? tab.qualified_tabname.to_sym }
           rescan.merge(elem.wired_by)
         end
       end
@@ -499,6 +517,25 @@ module Bud
       end
     end
     @reset_list = to_reset.to_a
+
+    # For each lattice, find the set of tables that should be rescanned when
+    # there is a new delta for the lattice. That is, if we have a rule like:
+    # "t2 <= t1 {|t| [t.key, lat_foo]}", whenever there is a delta on lat_foo we
+    # should rescan t1 (to produce tuples with the updated lat_foo value).
+    # TODO:
+    # (1) support non-join ops to be rescanned (+ tests) + lambdas
+    # (2) if t1 is fed by rules r1 and r2 but only r1 references lattice x,
+    #     don't trigger rescan of r2 on deltas for x (hard)
+    t_depends.each do |dep|
+      src, dst = dep.body.to_sym, dep.lhs.to_sym
+      if @lattices.has_key? src and @tables.has_key? dst and dep.in_body
+        src_lat = @lattices[src]
+        dst_tbl = @tables[dst]
+        dst_tbl.non_temporal_predecessors.each do |e|
+          src_lat.rescan_on_merge << e
+        end
+      end
+    end
   end
 
   # given rescan, invalidate sets, compute transitive closure
@@ -962,7 +999,10 @@ module Bud
     end
     bootstrap
 
-    @tables.each_value {|t| t.bootstrap} if toplevel == self
+    if toplevel == self
+      @tables.each_value {|t| t.bootstrap}
+      @lattices.each_value {|l| l.bootstrap}
+    end
     @done_bootstrap = true
   end
 
@@ -1100,7 +1140,7 @@ module Bud
 
     # for BUD reflection
     table :t_rules, [:bud_obj, :rule_id] => [:lhs, :op, :src, :orig_src, :nm_funcs_called]
-    table :t_depends, [:bud_obj, :rule_id, :lhs, :op, :body] => [:nm]
+    table :t_depends, [:bud_obj, :rule_id, :lhs, :op, :body] => [:nm, :in_body]
     table :t_provides, [:interface] => [:input]
     table :t_underspecified, t_provides.schema
     table :t_stratum, [:predicate] => [:stratum]