bud 0.9.5 → 0.9.6

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTY4MjlkMTY1NjZlYTZiOTVjNmYyMmIzYTliNDVlZDIwZjA0YjY4Yw==
+  data.tar.gz: !binary |-
+    MTE0ZGY3MDg5NzBmMTJkYTExNDNjYTcyNTFhOThmMzk0ZTM4YzgxOA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    OGJmYmMwZjU4Y2RlZWUyNDVkMGFkYTczZDhlNjk5MGQ5NmVlZDIyYzQyMjAz
+    NzEyZWI0YTI1MjVjM2VhZGRjMWE4ZDM4MGVlYzA4YzliYmJjNGUwZjEyY2I2
+    YjZhOTVjNzk5MGVjNWJkMTA0YWRmODk3MDAyYmVlOTM4MmVmYzA=
+  data.tar.gz: !binary |-
+    YjZkOTlkYzQwODJkZTI4YjU0YmFhMmY4MDY5Mjg3Y2ZjMzUyN2ZmZTAyOGI5
+    M2M3NWQ5OTgyM2U2ZjA3NGIzMjkwZjhjMTBhNWFmYzc3OTNiYmZjOWMzNmU3
+    MTkzZmI1MzJlODNkMmE2Y2E1NzU5ODYzNDlhM2Q3N2Q3MDk2Yzg=

data/History.txt

@@ -1,8 +1,32 @@
+== 0.9.6 / 2013-02-25
+
+* Support syntax sugar for initializing lattices (#294). For example, rather
+  than writing "foo <= Bud::MaxLattice.new(2)", you can now just write "foo <=
+  2". Note that, due to a bug in the superator gem, you cannot use this syntax
+  with the <+ operator.
+* Allow nested lattice values to be sent over channels (#295, patch from Josh
+  Rosen). Lattice values could previously be sent as a field value in a tuple
+  delivered over a channel, but they could not be embedded arbitrarily deeply
+  within tuples (e.g., lattices nested within an array could not previously be
+  sent as a field value).
+* Support "Comparable" for Bud::Lattice values
+* Add support for lattices to rebl
+* Reject attempts to insert into stdio via <+ (#288)
+* Restore functionality of reading from stdio (i.e., stdin) with MRI 1.9
+* Improve MRI 1.8 compatibility
+* Require ruby_parser >= 3.1.0
+* Fix bug in bootstrap blocks for lattice values
+* Fix bug in rescan/invalidation for rules that reference lattice values and use
+  the <+ or <~ operators (#290)
+* Fix bug in rescan logic for notin operator (#291)
+* Fix bug in notin operators involving self joins (#298)
+* Fix error when output from a notin operator was fed into a lattice
+
 == 0.9.5 / 2012-11-24
 
 * Lattice branch (Bloom^L) merged
 * Compatibility with recent versions of ruby_parser (3.0.2+) and ruby2ruby
-  (2.0.1+) -- #. Older versions of these two gems are no longer supported
+  (2.0.1+). Older versions of these two gems are no longer supported
 * Add support for aggregate functions that take multiple input columns
 * Add built-in aggregate function accum_pair(x, y), which produces a Set of
   pairs (two-element arrays [x,y])

data/bin/budplot

@@ -77,8 +77,8 @@ def process(mods)
   d = make_instance(mods)
 
   interfaces = {}
-  d.t_provides.each do |name, is_input|
-    interfaces[name] = is_input
+  d.t_provides.to_a.each do |prov|
+    interfaces[prov.interface] = prov.input
   end
 
   tabinf = {}
@@ -201,13 +201,18 @@ def get_trace_data
 end
 
 if ARGV.length < 2
-  puts "Usage: budplot LIST_OF_FILES LIST_OF_MODULES_OR_CLASSES"
+  puts "Usage: budplot [-I PATH_TO_RUBY_INCLUDES] LIST_OF_FILES LIST_OF_MODULES_OR_CLASSES"
   exit
 end
 
-@opts = Getopt::Std.getopts("t:")
-
-
+@opts = Getopt::Std.getopts("I:t:")
+unless @opts["I"].nil?
+  if @opts["I"].class == Array
+    @opts["I"].each{|i| $:.unshift i} 
+  else  
+    $:.unshift @opts["I"]
+  end
+end
 
 @data = get_trace_data
 `mkdir bud_doc`

data/bin/budtimelines

@@ -10,7 +10,7 @@ include VizUtil
 
 module Depends
   state do
-    table :depends, [:rid, :lhs, :op, :rhs, :nm, :in_body]
+    table :depends, [:bud_obj, :rid, :lhs, :op, :rhs, :nm, :in_body]
   end
 end
 
@@ -49,7 +49,7 @@ end
 module DeltaLogic
   include TPSchema
   bloom do
-    zerod_cards <= cardinalities{|c| c + [c.bud_time-1]}
+    zerod_cards <= cardinalities{|c| c.to_a + [c.bud_time-1]}
     zerod_cards <= (times * depends).pairs do |t, d|
       unless cardinalities{|c| c[1] if c[0] == t.bud_time}.include? d[1]
         [t.bud_time, d[1], 0, t.bud_time - 1]

data/bin/budvis

@@ -28,7 +28,13 @@ usage unless ARGV[0]
 usage if ARGV[0] == '--help'
 
 meta, data = get_meta2(BUD_DBM_DIR)
-vh = VizHelper.new(meta[:tabinf], meta[:cycle], meta[:depends], meta[:rules], ARGV[0], meta[:provides])
+
+# prune outbufs from tabinf
+tabinf = meta[:tabinf].find_all do |k|
+  !(k[1] == "Bud::BudChannel" and k[0] =~ /_snd\z/)
+end
+
+vh = VizHelper.new(tabinf, meta[:cycle], meta[:depends], meta[:rules], ARGV[0], meta[:provides])
 data.each do |d|
   vh.full_info << d
 end

data/docs/README.md

@@ -20,14 +20,14 @@ try the following:
   `budplot` tools for visualizing Bloom program analyses.
 * [bfs.md][bfs]: A walkthrough of the Bloom distributed filesystem.
 
-[intro]:          /bloom-lang/bud/blob/master/docs/intro.md
-[getstarted]:     /bloom-lang/bud/blob/master/docs/getstarted.md
-[operational]:    /bloom-lang/bud/blob/master/docs/operational.md
-[cheat]:          /bloom-lang/bud/blob/master/docs/cheat.md
-[modules]:        /bloom-lang/bud/blob/master/docs/modules.md
-[ruby_hooks]:     /bloom-lang/bud/blob/master/docs/ruby_hooks.md
-[visualizations]: /bloom-lang/bud/blob/master/docs/visualizations.md
-[bfs]:            /bloom-lang/bud/blob/master/docs/bfs.md
+[intro]:          /docs/intro.md
+[getstarted]:     /docs/getstarted.md
+[operational]:    /docs/operational.md
+[cheat]:          /docs/cheat.md
+[modules]:        /docs/modules.md
+[ruby_hooks]:     /docs/ruby_hooks.md
+[visualizations]: /docs/visualizations.md
+[bfs]:            /docs/bfs.md
 
 In addition, the [bud-sandbox](http://github.com/bloom-lang/bud-sandbox) GitHub
 repository contains lots of useful libraries and example programs built using

data/docs/cheat.md

@@ -224,14 +224,14 @@ Output the facts in `bc` that do not appear in `bc2`, as follows. First, we form
      :attr2`).
 
   2. If a code block is specified, invoke the block on every pair of matching
-     tuples in the join result. Any matches for which the block returns `nil`
+     tuples in the join result. Any matches for which the block returns `false`
      are removed from `t`.
 
 Finally, we output every tuple of `bc` that does *not* appear in `t`.
 
     # output items from foo if (a) there is no matching key in bar, or
     # (b) all matching keys in bar have a smaller value
-    stdio <~ foo.notin(bar, :key=>:key) {|f, b| true if f.val <= b.val}
+    stdio <~ foo.notin(bar, :key=>:key) {|f, b| f.val <= b.val}
 
     
 ## SQL-style grouping/aggregation (and then some) ##

data/lib/bud.rb

@@ -265,7 +265,7 @@ module Bud
       mod_inst.t_rules.each do |imp_rule|
         qname = "#{local_name}.#{imp_rule.lhs}"
         self.t_rules << [imp_rule.bud_obj, imp_rule.rule_id, qname, imp_rule.op,
-                         imp_rule.src, imp_rule.orig_src, imp_rule.nm_funcs_called]
+                         imp_rule.src, imp_rule.orig_src, imp_rule.unsafe_funcs_called]
       end
       mod_inst.t_depends.each do |imp_dep|
         qlname = "#{local_name}.#{imp_dep.lhs}"
@@ -326,7 +326,7 @@ module Bud
     # the user-defined tables and lattices.  We start @app_tables off as a set,
     # then convert to an array later.
     @app_tables = (@tables.keys - @builtin_tables.keys).map {|t| @tables[t]}.to_set
-    @app_tables += @lattices.values
+    @app_tables.merge(@lattices.values)
 
     # Check scan and merge_targets to see if any builtin_tables need to be added as well.
     @scanners.each do |scs|
@@ -346,7 +346,7 @@ module Bud
       seen = Set.new(working)
       sorted_elems = [] # sorted elements in this stratum
       while not working.empty?
-        sorted_elems += working
+        sorted_elems.concat(working)
         wired_to = []
         working.each do |e|
           e.wirings.each do |out|
@@ -453,18 +453,19 @@ module Bud
 
     # By default, all tables are considered sources unless they appear on the
     # lhs.  We only consider non-temporal rules because invalidation is only
-    # about this tick.  Also, we track (in nm_targets) those tables that are the
-    # targets of user-defined code blocks that call non-monotonic functions
-    # (such as budtime). Elements that feed these tables are forced to rescan
-    # their contents, and thus forced to re-execute these code blocks.
-    nm_targets = Set.new
+    # about this tick.  Also, we track (in unsafe_targets) those tables that are
+    # the targets of user-defined code blocks that call "unsafe" functions that
+    # produce a different value in every tick (e.g., budtime). Elements that
+    # feed these tables are forced to rescan their contents, and thus forced to
+    # re-execute these code blocks.
+    unsafe_targets = Set.new
     t_rules.each do |rule|
       lhs = rule.lhs.to_sym
       if rule.op == "<="
         # Note that lattices cannot be sources
         @tables[lhs].is_source = false if @tables.has_key? lhs
       end
-      nm_targets << lhs if rule.nm_funcs_called
+      unsafe_targets << lhs if rule.unsafe_funcs_called
     end
 
     # Compute a set of tables and elements that should be explicitly told to
@@ -477,7 +478,7 @@ module Bud
       @push_sorted_elems[stratum].each do |elem|
         rescan << elem if elem.rescan_at_tick
 
-        if elem.outputs.any?{|tab| not(tab.class <= PushElement) and not(tab.class <= LatticePushElement) and nm_targets.member? tab.qualified_tabname.to_sym }
+        if elem.outputs.any?{|tab| not(tab.class <= PushElement) and not(tab.class <= LatticePushElement) and unsafe_targets.member? tab.qualified_tabname.to_sym }
           rescan.merge(elem.wired_by)
         end
       end
@@ -489,8 +490,11 @@ module Bud
     @default_rescan = rescan.to_a
     @default_invalidate = invalidate.to_a
 
-    puts "Default rescan: #{rescan.inspect}" if $BUD_DEBUG
-    puts "Default inval: #{invalidate.inspect}" if $BUD_DEBUG
+    if $BUD_DEBUG
+      puts "Default rescan: #{rescan.inspect}"
+      puts "Default inval: #{invalidate.inspect}"
+      puts "Unsafe targets: #{unsafe_targets.inspect}"
+    end
 
     # Now compute for each table that is to be scanned, the set of dependent
     # tables and elements that will be invalidated if that table were to be
@@ -518,27 +522,43 @@ module Bud
     end
     @reset_list = to_reset.to_a
 
-    # For each lattice, find the set of tables that should be rescanned when
-    # there is a new delta for the lattice. That is, if we have a rule like:
+    # For each lattice, find the collections that should be rescanned when there
+    # is a new delta for the lattice. That is, if we have a rule like:
     # "t2 <= t1 {|t| [t.key, lat_foo]}", whenever there is a delta on lat_foo we
     # should rescan t1 (to produce tuples with the updated lat_foo value).
+    #
     # TODO:
-    # (1) support non-join ops to be rescanned (+ tests) + lambdas
-    # (2) if t1 is fed by rules r1 and r2 but only r1 references lattice x,
+    # (1) if t1 is fed by rules r1 and r2 but only r1 references lattice x,
     #     don't trigger rescan of r2 on deltas for x (hard)
     t_depends.each do |dep|
-      src, dst = dep.body.to_sym, dep.lhs.to_sym
-      if @lattices.has_key? src and @tables.has_key? dst and dep.in_body
+      src, target_name = dep.body.to_sym, dep.lhs.to_sym
+      if @lattices.has_key? src and dep.in_body
         src_lat = @lattices[src]
-        dst_tbl = @tables[dst]
-        dst_tbl.non_temporal_predecessors.each do |e|
-          src_lat.rescan_on_merge << e
+        if @tables.has_key? target_name
+          target = @tables[target_name]
+        else
+          target = @lattices[target_name]
+        end
+
+        # Conservatively, we rescan all the elements that feed the lhs (target)
+        # collection via positive (non-deletion) rules; we then also need to
+        # potentially rescan ancestors of those elements as well (e.g., setting
+        # a stateless PushElement to rescan does nothing; we want to tell its
+        # ancestor ScannerElement to rescan).
+        #
+        # XXX: do we need to consider all transitively reachable nodes for
+        # rescan?
+        lat_rescan = target.positive_predecessors.to_set
+        lat_inval = Set.new
+        target.positive_predecessors.each do |e|
+          e.add_rescan_invalidate(lat_rescan, lat_inval)
         end
+        src_lat.rescan_on_delta.merge(lat_rescan)
       end
     end
   end
 
-  # given rescan, invalidate sets, compute transitive closure
+  # Given rescan, invalidate sets, compute transitive closure
   def rescan_invalidate_tc(stratum, rescan, invalidate)
     rescan_len = rescan.size
     invalidate_len = invalidate.size
@@ -1139,14 +1159,14 @@ module Bud
     @periodics = table :periodics_tbl, [:pername] => [:period]
 
     # for BUD reflection
-    table :t_rules, [:bud_obj, :rule_id] => [:lhs, :op, :src, :orig_src, :nm_funcs_called]
+    table :t_cycle, [:predicate, :via, :neg, :temporal]
     table :t_depends, [:bud_obj, :rule_id, :lhs, :op, :body] => [:nm, :in_body]
     table :t_provides, [:interface] => [:input]
-    table :t_underspecified, t_provides.schema
+    table :t_rules, [:bud_obj, :rule_id] => [:lhs, :op, :src, :orig_src, :unsafe_funcs_called]
     table :t_stratum, [:predicate] => [:stratum]
-    table :t_cycle, [:predicate, :via, :neg, :temporal]
     table :t_table_info, [:tab_name, :tab_type]
     table :t_table_schema, [:tab_name, :col_name, :ord, :loc]
+    table :t_underspecified, t_provides.schema
 
     # Identify builtin tables as such
     @builtin_tables = @tables.clone if toplevel
@@ -1186,7 +1206,7 @@ module Bud
       begin
         eval_rule(rule.bud_obj, rule.src)
       rescue Exception => e
-        err_msg = "** Exception while wiring rule: #{rule.src}\n ****** #{e}"
+        err_msg = "** Exception while wiring rule: #{rule.orig_src}\n ****** #{e}"
         # Create a new exception for accomodating err_msg, but reuse original backtrace
         new_e = (e.class <= Bud::Error) ? e.class.new(err_msg) : Bud::Error.new(err_msg)
         new_e.set_backtrace(e.backtrace)

data/lib/bud/collections.rb

@@ -52,7 +52,7 @@ module Bud
       # user-specified schema.
       @cols.each do |s|
         if s.to_s.start_with? "@"
-          raise Bud::Error, "illegal use of location specifier (@) in column #{s} of non-channel collection #{tabname}"
+          raise Bud::CompileError, "illegal use of location specifier (@) in column #{s} of non-channel collection #{tabname}"
         end
       end
 
@@ -284,7 +284,12 @@ module Bud
 
     public
     def non_temporal_predecessors
-      @wired_by.select {|elem| elem.outputs.include? self}
+      @wired_by.select {|e| e.outputs.include? self}
+    end
+
+    public
+    def positive_predecessors
+      @wired_by.select {|e| e.outputs.include?(self) || e.pendings.include?(self)}
     end
 
     public
@@ -869,6 +874,8 @@ module Bud
       given_schema ||= [:@address, :val]
       @is_loopback = loopback
       @locspec_idx = nil
+      @wire_buf = StringIO.new
+      @packer = MessagePack::Packer.new(@wire_buf)
 
       # We're going to mutate the caller's given_schema (to remove the location
       # specifier), so make a deep copy first. We also save a ref to the
@@ -959,19 +966,40 @@ module Bud
         # particular, lattice values and Class instances), we first encode such
         # values using Marshal, and then encode the entire tuple with
         # MsgPack. Obviously, this is gross. The wire format also includes an
-        # array of indices, indicating which fields hold Marshall'd objects.
+        # array of indices, indicating which fields hold Marshall'd values.
+        @packer.write_array_header(3)
+        @packer.write(qualified_tabname.to_s)
+        # The second element, wire_tuple, is an array.  We will write it one
+        # element at a time:
+        @packer.write_array_header(t.length)
+        @packer.flush
         marshall_indexes = []
-        wire_tuple = Array.new(t.length)
-        t.each_with_index do |f, i|
+        t.each_with_index do |f,i|
+          # Performance optimization for cases where we know that we can't
+          # marshal the field using MsgPack:
           if [Bud::Lattice, Class].any?{|t| f.class <= t}
             marshall_indexes << i
-            wire_tuple[i] = Marshal.dump(f)
+            @wire_buf << Marshal.dump(f).to_msgpack
           else
-            wire_tuple[i] = f
+            begin
+              @wire_buf << f.to_msgpack
+            rescue NoMethodError
+              # If MsgPack can't marshal the field, fall back to Marshal.
+              # This handles fields that contain nested non-MsgPack-able
+              # objects (in these cases, the entire field is Marshal'd.)
+              marshall_indexes << i
+              @wire_buf << Marshal.dump(f).to_msgpack
+            end
           end
         end
-        wire_str = [qualified_tabname.to_s, wire_tuple, marshall_indexes].to_msgpack
-        toplevel.dsock.send_datagram(wire_str, the_locspec[0], the_locspec[1])
+        @packer.write(marshall_indexes)
+        @packer.flush
+        toplevel.dsock.send_datagram(@wire_buf.string,
+                                     the_locspec[0], the_locspec[1])
+
+        # Reset output buffer
+        @wire_buf.rewind
+        @wire_buf.truncate(0)
       end
       @pending.clear
     end
@@ -1015,45 +1043,28 @@ module Bud
     end
 
     superator "<+" do |o|
-      raise Bud::Error, "illegal use of <+ with channel '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <+ with channel '#{@tabname}' on left"
     end
 
     undef merge
 
     def <=(o)
-      raise Bud::Error, "illegal use of <= with channel '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with channel '#{@tabname}' on left"
     end
   end
 
   class BudTerminal < BudScratch # :nodoc: all
-    def initialize(name, given_schema, bud_instance, prompt=false) # :nodoc: all
-      super(name, bud_instance, given_schema)
+    def initialize(name, bud_instance, prompt=false) # :nodoc: all
+      super(name, bud_instance, [:line])
       @prompt = prompt
     end
 
     public
     def start_stdin_reader # :nodoc: all
-      # XXX: Ugly hack. Rather than sending terminal data to EM via UDP,
-      # we should add the terminal file descriptor to the EM event loop.
-      @reader = Thread.new do
+      Thread.new do
         begin
-          toplevel = @bud_instance.toplevel
           while true
-            out_io = get_out_io
-            out_io.print("#{tabname} > ") if @prompt
-
-            in_io = toplevel.options[:stdin]
-            s = in_io.gets
-            break if s.nil? # Hit EOF
-            s = s.chomp if s
-            tup = [s]
-
-            ip = toplevel.ip
-            port = toplevel.port
-            EventMachine::schedule do
-              socket = EventMachine::open_datagram_socket("127.0.0.1", 0)
-              socket.send_datagram([tabname, tup, []].to_msgpack, ip, port)
-            end
+            break unless read_line
           end
         rescue Exception
           puts "terminal reader thread failed: #{$!}"
@@ -1063,13 +1074,34 @@ module Bud
       end
     end
 
+    # XXX: Ugly hack. Rather than sending terminal data to EM via UDP, we should
+    # add the terminal file descriptor to the EM event loop.
+    private
+    def read_line
+      toplevel = @bud_instance.toplevel
+      if @prompt
+        get_out_io.print("#{tabname} > ")
+      end
+
+      in_io = toplevel.options[:stdin]
+      input_str = in_io.gets
+      return false if input_str.nil? # Hit EOF
+      input_str.chomp!
+
+      EventMachine::schedule do
+        socket = EventMachine::open_datagram_socket("127.0.0.1", 0)
+        socket.send_datagram([tabname, [input_str], []].to_msgpack,
+                             toplevel.ip, toplevel.port)
+      end
+
+      return true
+    end
+
     public
     def flush #:nodoc: all
       out_io = get_out_io
-      @pending.each_value do |p|
-        out_io.puts p[0]
-        out_io.flush
-      end
+      @pending.each_value {|p| out_io.puts p[0]}
+      out_io.flush
       @pending.clear
     end
 
@@ -1090,7 +1122,6 @@ module Bud
         @tick_delta.clear
       end
       @invalidated = true # channels and terminals are always invalidated.
-      raise Bud::Error, "orphaned pending tuples in terminal" unless @pending.empty?
     end
 
     public
@@ -1101,7 +1132,11 @@ module Bud
 
     public
     def <=(o) #:nodoc: all
-      raise Bud::Error, "illegal use of <= with terminal '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with terminal '#{@tabname}' on left"
+    end
+
+    superator "<+" do |o|
+      raise Bud::CompileError, "illegal use of <+ with terminal '#{@tabname}' on left"
     end
 
     superator "<~" do |o|
@@ -1121,26 +1156,28 @@ module Bud
     def get_out_io
       rv = @bud_instance.toplevel.options[:stdout]
       rv ||= $stdout
-      raise Bud::Error, "attempting to write to terminal #{tabname} that was already closed" if rv.closed?
+      if rv.closed?
+        raise Bud::Error, "attempt to write to closed terminal '#{tabname}'"
+      end
       rv
     end
   end
 
   class BudPeriodic < BudScratch # :nodoc: all
     def <=(o)
-      raise Bud::Error, "illegal use of <= with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with periodic '#{tabname}' on left"
     end
 
     superator "<~" do |o|
-      raise Bud::Error, "illegal use of <~ with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <~ with periodic '#{tabname}' on left"
     end
 
     superator "<-" do |o|
-      raise Bud::Error, "illegal use of <- with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <- with periodic '#{tabname}' on left"
     end
 
     superator "<+" do |o|
-      raise Bud::Error, "illegal use of <+ with periodic '#{tabname}' on left"
+      raise Bud::CompileError, "illegal use of <+ with periodic '#{tabname}' on left"
     end
 
     def tick
@@ -1272,11 +1309,11 @@ module Bud
 
   class BudReadOnly < BudCollection # :nodoc: all
     superator "<+" do |o|
-      raise CompileError, "illegal use of <+ with read-only collection '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <+ with read-only collection '#{@tabname}' on left"
     end
     public
     def merge(o)  #:nodoc: all
-      raise CompileError, "illegal use of <= with read-only collection '#{@tabname}' on left"
+      raise Bud::CompileError, "illegal use of <= with read-only collection '#{@tabname}' on left"
     end
     public
     def invalidate_cache