bud 0.0.2

data/lib/bud/storage/tokyocabinet.rb

@@ -0,0 +1,170 @@
+require 'tokyocabinet'
+
+module Bud
+  # Persistent table implementation based on TokyoCabinet.
+  class BudTcTable < BudCollection # :nodoc: all
+    def initialize(name, bud_instance, given_schema)
+      tc_dir = bud_instance.options[:tc_dir]
+      raise BudError, "TC support must be enabled via 'tc_dir'" unless tc_dir
+      unless File.exists?(tc_dir)
+        Dir.mkdir(tc_dir)
+        puts "Created directory: #{tc_dir}" unless bud_instance.options[:quiet]
+      end
+
+      dirname = "#{tc_dir}/bud_#{bud_instance.port}"
+      unless File.exists?(dirname)
+        Dir.mkdir(dirname)
+        puts "Created directory: #{dirname}" unless bud_instance.options[:quiet]
+      end
+
+      super(name, bud_instance, given_schema)
+      @to_delete = []
+
+      @hdb = TokyoCabinet::HDB.new
+      db_fname = "#{dirname}/#{name}.tch"
+      flags = TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT
+      if bud_instance.options[:tc_truncate] == true
+        flags |= TokyoCabinet::HDB::OTRUNC
+      end
+      if !@hdb.open(db_fname, flags)
+        raise BudError, "Failed to open TokyoCabinet DB '#{db_fname}': #{@hdb.errmsg}"
+      end
+      @hdb.tranbegin
+    end
+
+    def init_storage
+      # XXX: we can't easily use the @storage infrastructure provided by
+      # BudCollection; issue #33
+      @storage = nil
+    end
+
+    def [](key)
+      key_s = MessagePack.pack(key)
+      val_s = @hdb[key_s]
+      if val_s
+        return make_tuple(key, MessagePack.unpack(val_s))
+      else
+        return @delta[key]
+      end
+    end
+
+    def has_key?(k)
+      key_s = MessagePack.pack(k)
+      return true if @hdb.has_key? key_s
+      return @delta.has_key? k
+    end
+
+    def include?(tuple)
+      key = @key_colnums.map{|k| tuple[k]}
+      value = self[key]
+      return (value == tuple)
+    end
+
+    def make_tuple(k_ary, v_ary)
+      t = Array.new(k_ary.length + v_ary.length)
+      @key_colnums.each_with_index do |k,i|
+        t[k] = k_ary[i]
+      end
+      val_cols.each_with_index do |c,i|
+        t[schema.index(c)] = v_ary[i]
+      end
+      tuple_accessors(t)
+    end
+
+    def each(&block)
+      each_from([@delta], &block)
+      each_storage(&block)
+    end
+
+    def each_from(bufs, &block)
+      bufs.each do |b|
+        if b == @storage then
+          each_storage(&block)
+        else
+          b.each_value do |v|
+            yield v
+          end
+        end
+      end
+    end
+
+    def each_storage(&block)
+      @hdb.each do |k,v|
+        k_ary = MessagePack.unpack(k)
+        v_ary = MessagePack.unpack(v)
+        yield make_tuple(k_ary, v_ary)
+      end
+    end
+
+    def flush
+      @hdb.trancommit
+    end
+
+    def close
+      @hdb.close
+    end
+
+    def merge_to_hdb(buf)
+      buf.each do |key,tuple|
+        merge_tuple(key, tuple)
+      end
+    end
+
+    def merge_tuple(key, tuple)
+      val = val_cols.map{|c| tuple[schema.index(c)]}
+      key_s = MessagePack.pack(key)
+      val_s = MessagePack.pack(val)
+      if @hdb.putkeep(key_s, val_s) == false
+        old_tuple = self[key]
+        raise_pk_error(tuple, old_tuple) if tuple != old_tuple
+      end
+    end
+
+    # move deltas to TC, and new_deltas to deltas
+    def tick_deltas
+      merge_to_hdb(@delta)
+      @delta = @new_delta
+      @new_delta = {}
+    end
+
+    superator "<-" do |o|
+      o.each do |tuple|
+        @to_delete << tuple unless tuple.nil?
+      end
+    end
+
+    def insert(tuple)
+      key = @key_colnums.map{|k| tuple[k]}
+      merge_tuple(key, tuple)
+    end
+
+    alias << insert
+
+    # Remove to_delete and then add pending to HDB
+    def tick
+      @to_delete.each do |tuple|
+        k = @key_colnums.map{|c| tuple[c]}
+        k_str = MessagePack.pack(k)
+        cols_str = @hdb[k_str]
+        unless cols_str.nil?
+          hdb_cols = MessagePack.unpack(cols_str)
+          delete_cols = val_cols.map{|c| tuple[schema.index(c)]}
+          if hdb_cols == delete_cols
+            @hdb.delete k_str
+          end
+        end
+      end
+      @to_delete = []
+
+      merge_to_hdb(@pending)
+      @pending = {}
+
+      @hdb.trancommit
+      @hdb.tranbegin
+    end
+
+    def method_missing(sym, *args, &block)
+      @hdb.send sym, *args, &block
+    end
+  end
+end

data/lib/bud/storage/zookeeper.rb

@@ -0,0 +1,178 @@
+begin
+  require 'zookeeper'
+  Bud::HAVE_ZOOKEEPER = true
+rescue LoadError
+end
+
+module Bud
+  # Persistent table implementation based on Zookeeper.
+  class BudZkTable < BudCollection # :nodoc: all
+    def initialize(name, zk_path, zk_addr, bud_instance)
+      unless defined? HAVE_ZOOKEEPER
+        raise BudError, "zookeeper gem is not installed: zktables cannot be used"
+      end
+
+      # schema = {[:key] => [:val]}
+      super(name, bud_instance, nil)
+
+      zk_path = zk_path.chomp("/") unless zk_path == "/"
+      @zk = Zookeeper.new(zk_addr)
+      @zk_path = zk_path
+      @base_path = @zk_path
+      @base_path += "/" unless @zk_path.end_with? "/"
+      @store_mutex = Mutex.new
+      @next_storage = {}
+      @saw_delta = false
+      @child_watch_id = nil
+      @stat_watch_id = nil
+    end
+
+    # Since the watcher callbacks might invoke EventMachine, we wait until after
+    # EM startup to start watching for Zk events.
+    def start_watchers
+      # NB: Watcher callbacks are invoked in a separate Ruby thread.
+      @child_watcher = Zookeeper::WatcherCallback.new { get_and_watch }
+      @stat_watcher = Zookeeper::WatcherCallback.new { stat_and_watch }
+      stat_and_watch
+    end
+
+    def clone_empty
+      raise BudError
+    end
+
+    def stat_and_watch
+      r = @zk.stat(:path => @zk_path, :watcher => @stat_watcher)
+      @stat_watch_id = r[:req_id]
+
+      unless r[:stat].exists
+        cancel_child_watch
+        # The given @zk_path doesn't exist, so try to create it. Unclear
+        # whether this is always the best behavior.
+        r = @zk.create(:path => @zk_path)
+        if r[:rc] != Zookeeper::ZOK and r[:rc] != Zookeeper::ZNODEEXISTS
+          raise
+        end
+      end
+
+      # Make sure we're watching for children
+      get_and_watch unless @child_watch_id
+    end
+
+    def cancel_child_watch
+      if @child_watch_id
+        @zk.unregister_watcher(@child_watch_id)
+        @child_watch_id = nil
+      end
+    end
+
+    def cancel_stat_watch
+      if @stat_watch_id
+        @zk.unregister_watcher(@stat_watch_id)
+        @stat_with_id = nil
+      end
+    end
+
+    def get_and_watch
+      r = @zk.get_children(:path => @zk_path, :watcher => @child_watcher)
+      @child_watch_id = r[:req_id]
+      unless r[:stat].exists
+        cancel_child_watch
+        return
+      end
+
+      # XXX: can we easily get snapshot isolation?
+      new_children = {}
+      r[:children].each do |c|
+        child_path = @base_path + c
+
+        get_r = @zk.get(:path => child_path)
+        unless get_r[:stat].exists
+          puts "ZK: failed to fetch child: #{child_path}"
+          return
+        end
+
+        data = get_r[:data]
+        # XXX: For now, conflate empty string values with nil values
+        data ||= ""
+        new_children[c] = tuple_accessors([c, data])
+      end
+
+      # We successfully fetched all the children of @zk_path; arrange to install
+      # the new data into @storage at the next Bud tick
+      need_tick = false
+      @store_mutex.synchronize {
+        @next_storage = new_children
+        if @storage != @next_storage
+          need_tick = true
+          @saw_delta = true
+        end
+      }
+
+      # If we have new data, force a new Bud tick in the near future
+      if need_tick and not @bud_instance.lazy
+        EventMachine::schedule {
+          @bud_instance.tick
+        }
+      end
+    end
+
+    def tick
+      @store_mutex.synchronize {
+        return unless @saw_delta
+        @storage = @next_storage
+        @next_storage = {}
+        @saw_delta = false
+      }
+    end
+
+    def flush
+      each_from([@pending]) do |t|
+        path = @base_path + t.key
+        data = t.val
+        ephemeral = false
+        sequence = false
+
+        if t.length > 2
+          opts = t.last.first
+          if opts[:ephemeral] == true
+            ephemeral = true
+          end
+          if opts[:sequence] == true
+            sequence = true
+          end
+        end
+
+        r = @zk.create(:path => path, :data => data,
+                       :ephemeral => ephemeral, :sequence => sequence)
+        if r[:rc] == Zookeeper::ZNODEEXISTS
+          puts "Ignoring duplicate insert: #{t.inspect}"
+        elsif r[:rc] != Zookeeper::ZOK
+          puts "Failed create of #{path}: #{r.inspect}"
+        end
+      end
+      @pending.clear
+    end
+
+    def close
+      cancel_child_watch
+      cancel_stat_watch
+      @zk.close
+    end
+
+    superator "<~" do |o|
+      pending_merge(o)
+    end
+
+    superator "<+" do |o|
+      raise BudError, "Illegal use of <+ with zktable '#{@tabname}' on left"
+    end
+
+    def <=(o)
+      raise BudError, "Illegal use of <= with zktable '#{@tabname}' on left"
+    end
+
+    def <<(o)
+      raise BudError, "Illegal use of << with zktable '#{@tabname}' on left"
+    end
+  end
+end

data/lib/bud/stratify.rb

@@ -0,0 +1,83 @@
+require 'rubygems'
+require 'bud'
+
+class Stratification # :nodoc: all
+  include Bud
+
+  state do
+    # Data inserted by client (Bud rewrite code)
+    table :depends, [:rule, :head, :op, :body, :neg]
+
+    scratch :depends_clean, [:head, :body, :neg, :temporal]
+    scratch :depends_tc, [:head, :body, :via, :neg, :temporal]
+    scratch :cycle, [:predicate, :via, :neg, :temporal]
+    table :stratum_base, [:predicate, :stratum]
+    table :stratum, [:predicate, :stratum]
+    table :top_strat, [:stratum]
+  end
+
+  def declaration
+    strata[0] = lambda {
+      depends_clean <= depends do |d|
+        is_temporal = (d.op.to_s =~ /<[\+\-\~]/)
+        [d.head, d.body, d.neg, is_temporal]
+      end
+
+      # Compute the transitive closure of "depends_clean" to detect cycles in
+      # the deductive fragment of the program.
+      depends_tc <= depends_clean do |d|
+        [d.head, d.body, d.body, d.neg, d.temporal]
+      end
+      depends_tc <= (depends_clean * depends_tc).pairs(:body => :head) do |b, r|
+        [b.head, r.body, b.body, (b.neg or r.neg), (b.temporal or r.temporal)]
+      end
+
+      cycle <= depends_tc do |d|
+        if d.head == d.body
+          if d.neg and !d.temporal
+            raise Bud::CompileError, "unstratifiable program: #{d.inspect}"
+          else
+            [d.head, d.via, d.neg, d.temporal]
+          end
+        end
+      end
+    
+      # we initially assign all predicates to stratum 0
+      stratum_base <= depends {|d| [d.body, 0]}
+    }
+
+    strata[1] = lambda {
+      # classic stratification:
+      # if A depends on B, A is >= B.
+      # if A depends nonmonotonically on B, A > B.
+      # if A are B are co-dependent, give up. 
+      #  (don't need to do this, b/c we've ruled out deductive cycles)
+      # stratum choice will represent local evaluation order,
+      # so we need only consider 'synchronous' dependencies (<=)
+
+      # pass 1: assume no deductive cycles
+      # do "vanilla stratification" on deductive rules
+      stratum_base <= (depends * stratum_base).pairs(:body => :predicate) do |d, s|
+        if d.op.to_s == '<='
+          if d.neg
+            # BUMP
+            [d.head, s.stratum + 1]
+          else
+            # HOIST
+            [d.head, s.stratum]
+          end
+        end
+      end
+    }
+
+    strata[2] = lambda {
+      stratum <= stratum_base.group([stratum_base.predicate], max(stratum_base.stratum))
+    }
+
+    strata[3] = lambda {
+      # there is no good reason that top_strat can't be computed in strata[3] over stratum_base.
+      # however, when it is deduced that way, it is empty after a tick
+      top_strat <= stratum.group([], max(stratum.stratum))
+    }
+  end
+end

data/lib/bud/viz.rb

@@ -0,0 +1,65 @@
+require 'rubygems'
+require 'syntax/convertors/html'
+require 'gchart'
+require 'digest/md5'
+require 'bud/state'
+
+class VizOnline #:nodoc: all
+  def initialize(bud_instance)
+    @bud_instance = bud_instance
+    return if bud_instance.class == Stratification or @bud_instance.class == DepAnalysis
+    @meta_tables = {'t_rules' => 1, 't_depends' => 1, 't_table_info' => 1, 't_cycle' => 1, 't_stratum' => 1, 't_depends_tc' => 1, 't_table_schema' => 1}
+    @bud_instance.options[:tc_dir] = "TC_#{@bud_instance.class}_#{bud_instance.options[:tag]}_#{bud_instance.object_id}_#{bud_instance.port}"
+    @table_info = new_tab(:t_table_info, [:tab_name, :tab_type], @bud_instance)
+    @table_schema = new_tab(:t_table_schema, [:tab_name, :col_name, :ord], @bud_instance)
+
+    @logtab = {}
+    tmp_set = []
+    @bud_instance.tables.each do |name, tbl|
+      next if name.to_s =~ /_vizlog\z/
+
+      # Temp collections don't have a schema until a fact has been inserted into
+      # them; for now, we just include an empty schema for them in the viz
+      if tbl.schema.nil?
+        schema = []
+      else
+        schema = tbl.schema.clone
+      end
+      tmp_set << [name, schema, tbl.class.to_s]
+    end
+
+    tmp_set.each do |t|
+      news = [:c_bud_time]
+      @table_schema << [t[0], :c_bud_time, 0]
+      t[1].each_with_index do |s, i|
+        news << s
+        @table_schema << [t[0], s, i+1]
+      end
+      lt = "#{t[0]}_vizlog".to_sym
+      @logtab[t[0]] = new_tab(lt, news, @bud_instance)
+      @table_info << [t[0], t[2]]
+    end
+  end
+
+  def new_tab(name, schema, instance)
+    ret = Bud::BudTcTable.new(name, instance, schema)
+    instance.tables[name] = ret
+    return ret
+  end
+
+  def do_cards
+    return if @bud_instance.class == Stratification or @bud_instance.class == DepAnalysis
+    @bud_instance.tables.each do |t|
+      tab = t[0]
+      next if tab.to_s =~ /_vizlog\z/
+      next if @meta_tables[tab.to_s] and @bud_instance.budtime > 0
+      next unless @logtab[tab]
+      t[1].each do |row|
+        newrow = [@bud_instance.budtime]
+        row.each{ |r| newrow << r }
+        @logtab[tab] << newrow
+      end
+      @logtab[tab].tick
+    end
+  end
+end