tupelo 0.9 → 0.10

data/example/pregel/distributed.rb

@@ -0,0 +1,129 @@
+# Distributed version of pagerank.rb.
+
+# TODO
+#
+# Improvements noted in the article.
+# Scale better with subspaces and sqlite or other data structure.
+# And the subspaces could be defined by consistent hashing and smarter
+# partitioning.
+# Also, need to handle crashed process and lost tuple (as lease.rb maybe).
+# Would be nice to have remote option as well.
+# Abstract out domain-specific code from generic framework code.
+# Option to compare result with that of pagerank.rb using same seed.
+
+require 'tupelo/app'
+
+NUM_WORKERS = 4
+NUM_VERTICES = 10
+PRNG_SEED = 1234
+
+def update vertex, incoming_messages, vs_dst
+  vertex = vertex.dup
+  incoming_messages ||= []
+  outgoing_messages = []
+  v_me = vertex["id"]
+  rank = vertex["rank"]
+  step = vertex["step"]
+  active = true
+
+  if step < 50
+    rank = 0.15 / NUM_VERTICES + 0.85 * incoming_messages.inject(0.0) {|sum, m|
+      sum + m["rank"]}
+    outgoing_rank = rank / vs_dst.size
+    outgoing_messages = vs_dst.map {|v_dst|
+      {src: v_me, dst: v_dst, step: step + 1, rank: outgoing_rank}}
+  else
+    active = false
+  end
+
+  vertex["rank"] = rank
+  vertex["active"] = active
+  vertex["step"] += 1
+
+  [vertex, outgoing_messages]
+end
+
+Tupelo.application do
+  
+  NUM_WORKERS.times do |i|
+    child passive: true do
+      log.progname = "worker #{i}"
+
+      loop do
+        step = vertex = nil
+        transaction do
+          step = read(step: Numeric)["step"]
+          vertex = take id: nil, step: step, rank: nil, active: true
+          # wait for a vertex to be available on current step, but retry if
+          # step changes
+        end
+
+        v_me = vertex["id"]
+        vs_dst = read_all(src: v_me, dst: nil).map {|h| h["dst"]}
+        vs_src = read_all(src: nil, dst: v_me).map {|h| h["src"]}
+        
+        incoming_messages = transaction do
+          vs_src.inject([]) do |ms, v_src|
+            while m=take_nowait(src: v_src, dst: v_me, step: step, rank: nil)
+              ms << m
+            end
+            ms
+          end
+        end
+
+        vertex, outgoing_messages = update(vertex, incoming_messages, vs_dst)
+        write vertex, *outgoing_messages
+        
+        transaction do
+          n = take(count: Integer, active: vertex["active"])["count"]
+          write count: n + 1, active: vertex["active"]
+        end
+      end
+    end
+  end
+  
+  child do
+    log.progname = "coordinator"
+
+    step = 0
+    v_ids = (0...NUM_VERTICES).to_a
+
+    v_ids.each do |v_id|
+      write id: v_id, step: step, rank: 1.0/NUM_VERTICES, active: true
+    end
+
+    srand PRNG_SEED
+    v_ids.each do |v_src|
+      v_ids.sample(4).each do |v_dst|
+        write src: v_src, dst: v_dst
+      end
+    end
+
+    write count: 0, active: true
+    write count: 0, active: false
+    loop do
+      log "step: #{step}"
+      transaction do
+        write step: step
+        take count: nil, active: true
+        write count: 0, active: true
+      end
+      
+      # wait for all vertices to finish step and maybe become inactive
+      done = transaction do
+        n_active = read(count: nil, active: true)["count"]
+        read(count: NUM_VERTICES - n_active, active: false)
+        n_active == 0
+      end
+      
+      if done
+        vs = read_all(id: nil, step: nil, rank: nil, active: nil)
+        log vs.sort_by {|v| v["id"]}.map {|v| v["rank"]}
+        exit
+      end
+      
+      take step: step
+      step += 1
+    end
+  end
+end

data/example/pregel/pagerank.rb

@@ -0,0 +1,72 @@
+# port of pagerank.py (not distributed)
+
+require './pregel'
+require 'narray'
+
+NUM_WORKERS = 4
+NUM_VERTICES = 10
+
+def main
+  vertices = NUM_VERTICES.times.map {|j|
+    PageRankVertex.new(j, 1.0/NUM_VERTICES, [])}
+  create_edges(vertices)
+  
+  pr_test = pagerank_test(vertices)
+  puts "Test computation of pagerank:\n%p" % pr_test
+  
+  pr_pregel = pagerank_pregel(vertices)
+  puts "Pregel computation of pagerank:\n%p" % pr_pregel
+  
+  diff = pr_pregel - pr_test
+  puts "Difference between the two pagerank vectors:\n%p" % diff
+  
+  norm = Math.sqrt(diff * diff)
+  puts "The norm of the difference is: %p" % norm
+end
+
+def create_edges vertices
+  vertices.each do |vertex|
+    vertex.out_vertices = vertices.sample(4)
+  end
+end
+
+def pagerank_test vertices
+  ident = NMatrix.float(NUM_VERTICES, NUM_VERTICES).unit
+  g = NMatrix.float(NUM_VERTICES, NUM_VERTICES)
+
+  vertices.each do |vertex|
+    num_out_vertices = vertex.out_vertices.size
+    vertex.out_vertices.each do |out_vertex|
+      g[vertex.id, out_vertex.id] = 1.0/num_out_vertices
+        # node reversed dimensions, a funny feature of NArray!
+    end
+  end
+
+  mp = (1.0/NUM_VERTICES)*NVector.float(NUM_VERTICES).fill!(1)
+  return 0.15 * ((ident - 0.85 * g).inverse) * mp
+end
+
+def pagerank_pregel vertices
+  pregel = Pregel.new(vertices, NUM_WORKERS)
+  pregel.run()
+  return NVector.to_na(pregel.vertices.map {|vertex| vertex.value})
+end
+
+class PageRankVertex < Vertex
+  def update
+    if superstep < 50
+      @value = 0.15 / NUM_VERTICES +
+        0.85 * incoming_messages.inject(0.0) {|sum, (vertex, pagerank)|
+          sum + pagerank}
+      outgoing_pagerank = value / out_vertices.size
+      @outgoing_messages = out_vertices.map {|vertex|
+        [vertex, outgoing_pagerank]}
+    else
+      @active = false
+    end
+  end
+end
+
+if __FILE__ ==  $0
+  main
+end

data/example/pregel/pregel.rb

@@ -0,0 +1,102 @@
+# port of pregel.py (not distributed)
+
+class Vertex
+  attr_reader :id, :value, :active, :superstep
+  attr_accessor :out_vertices
+  attr_reader :incoming_messages, :outgoing_messages
+  
+  def initialize id, value, out_vertices
+    @id = id
+    @value = value
+    @out_vertices = out_vertices
+    @incoming_messages = []
+    @outgoing_messages = []
+    @active = true
+    @superstep = 0
+  end
+  
+  def next_superstep
+    @superstep += 1
+    @incoming_messages = []
+  end
+end
+
+class Pregel
+  attr_reader :vertices, :num_workers, :partition
+
+  def initialize vertices, num_workers
+    @vertices = vertices
+    @num_workers = num_workers
+    @partition = nil
+  end
+  
+  def run
+    @partition = partition_vertices
+    while check_active
+      superstep
+      redistribute_messages
+    end
+  end
+  
+  # returns {0 => [vertex,...],.... (num_workers-1) => [...]}
+  def partition_vertices
+    vertices.group_by {|vertex| worker(vertex)}
+  end
+  
+  def worker vertex
+    vertex.hash % num_workers
+  end
+  
+  def superstep
+    workers = []
+    partition.each_value do |vertex_list|
+      workers << Worker.new(vertex_list)
+    end
+
+    workers.each do |worker|
+      worker.join
+    end
+  end
+
+  def redistribute_messages
+    vertices.each do |vertex|
+      vertex.next_superstep
+    end
+    
+    vertices.each do |vertex|
+      vertex.outgoing_messages.each do |receiving_vertex, message|
+        receiving_vertex.incoming_messages << [vertex, message]
+      end
+    end
+  end
+
+  def check_active
+    vertices.any? {|vertex| vertex.active}
+  end
+end
+
+class Worker
+  attr_reader :vertices
+
+  def initialize vertices
+    @vertices = vertices
+    @thread = Thread.new {run}
+    @thread.abort_on_exception = true
+  end
+  
+  def join
+    @thread.join
+  end
+
+  def run
+    superstep
+  end
+
+  def superstep
+    vertices.each do |vertex|
+      if vertex.active
+        vertex.update
+      end
+    end
+  end
+end

data/example/pregel/remote.rb

@@ -0,0 +1,165 @@
+# Distributed version of pagerank.rb.
+
+# TODO
+#
+# Improvements noted in the article.
+# Scale better with subspaces and sqlite or other data structure.
+# And the subspaces could be defined by consistent hashing and smarter
+# partitioning.
+# Also, need to handle crashed process and lost tuple (as lease.rb maybe).
+# Would be nice to have remote option as well.
+# Abstract out domain-specific code from generic framework code.
+# Option to compare result with that of pagerank.rb using same seed.
+
+require 'tupelo/app'
+
+NUM_WORKERS = 4
+NUM_VERTICES = 10
+PRNG_SEED = 1234
+
+def update vertex, incoming_messages, vs_dst
+  vertex = vertex.dup
+  incoming_messages ||= []
+  outgoing_messages = []
+  v_me = vertex["id"]
+  rank = vertex["rank"]
+  step = vertex["step"]
+  active = true
+
+  if step < 50
+    rank = 0.15 / NUM_VERTICES + 0.85 * incoming_messages.inject(0.0) {|sum, m|
+      sum + m["rank"]}
+    outgoing_rank = rank / vs_dst.size
+    outgoing_messages = vs_dst.map {|v_dst|
+      {src: v_me, dst: v_dst, step: step + 1, rank: outgoing_rank}}
+  else
+    active = false
+  end
+
+  vertex["rank"] = rank
+  vertex["active"] = active
+  vertex["step"] += 1
+
+  [vertex, outgoing_messages]
+end
+
+require 'tupelo/app/remote'
+def host i
+  case i % 2
+  when 0; "od1"
+  when 1; "od2"
+  end
+end
+
+Tupelo.tcp_application do
+  
+  NUM_WORKERS.times do |i|
+#    child passive: true do
+    remote host: host(i), log: true, passive: true, eval: %{
+    log "hello"
+      def update vertex, incoming_messages, vs_dst
+        vertex = vertex.dup
+        incoming_messages ||= []
+        outgoing_messages = []
+        v_me = vertex["id"]
+        rank = vertex["rank"]
+        step = vertex["step"]
+        active = true
+
+        if step < 50
+          rank = 0.15 / #{NUM_VERTICES} + 0.85 * incoming_messages.inject(0.0) {|sum, m|
+            sum + m["rank"]}
+          outgoing_rank = rank / vs_dst.size
+          outgoing_messages = vs_dst.map {|v_dst|
+            {src: v_me, dst: v_dst, step: step + 1, rank: outgoing_rank}}
+        else
+          active = false
+        end
+
+        vertex["rank"] = rank
+        vertex["active"] = active
+        vertex["step"] += 1
+
+        [vertex, outgoing_messages]
+      end
+
+      log.progname = "worker #{i}"
+
+      loop do
+        step = vertex = nil
+        transaction do
+          step = read(step: Numeric)["step"]
+          vertex = take id: nil, step: step, rank: nil, active: true
+          # wait for a vertex to be available on current step, but retry if
+          # step changes
+        end
+
+        v_me = vertex["id"]
+        vs_dst = read_all(src: v_me, dst: nil).map {|h| h["dst"]}
+        vs_src = read_all(src: nil, dst: v_me).map {|h| h["src"]}
+        
+        incoming_messages = transaction do
+          vs_src.inject([]) do |ms, v_src|
+            while m=take_nowait(src: v_src, dst: v_me, step: step, rank: nil)
+              ms << m
+            end
+            ms
+          end
+        end
+
+        vertex, outgoing_messages = update(vertex, incoming_messages, vs_dst)
+        write vertex, *outgoing_messages
+        
+        transaction do
+          n = take(count: Integer, active: vertex["active"])["count"]
+          write count: n + 1, active: vertex["active"]
+        end
+      end
+    }
+  end
+  
+  child do
+    log.progname = "coordinator"
+
+    step = 0
+    v_ids = (0...NUM_VERTICES).to_a
+
+    v_ids.each do |v_id|
+      write id: v_id, step: step, rank: 1.0/NUM_VERTICES, active: true
+    end
+
+    srand PRNG_SEED
+    v_ids.each do |v_src|
+      v_ids.sample(4).each do |v_dst|
+        write src: v_src, dst: v_dst
+      end
+    end
+
+    write count: 0, active: true
+    write count: 0, active: false
+    loop do
+      log "step: #{step}"
+      transaction do
+        write step: step
+        take count: nil, active: true
+        write count: 0, active: true
+      end
+      
+      # wait for all vertices to finish step and maybe become inactive
+      done = transaction do
+        n_active = read(count: nil, active: true)["count"]
+        read(count: NUM_VERTICES - n_active, active: false)
+        n_active == 0
+      end
+      
+      if done
+        vs = read_all(id: nil, step: nil, rank: nil, active: nil)
+        log vs.sort_by {|v| v["id"]}.map {|v| v["rank"]}
+        exit
+      end
+      
+      take step: step
+      step += 1
+    end
+  end
+end

data/example/pulse.rb

@@ -2,25 +2,25 @@
 
 require 'tupelo/app'
 
-Tupelo.application do |app|
-  app.child do |client|
-    client.take ['start']
+Tupelo.application do
+  child do
+    take ['start']
     10.times do |i|
-      client.pulse [i]
+      pulse [i]
       sleep 0.1
     end
-    client.write ['finish']
+    write ['finish']
   end
   
-  app.child do |client|
+  child do
     Thread.new do
       loop do
-        client.log client.read [Integer]
-        client.log client.read_all [Integer]
+        log read [Integer]
+        log read_all [Integer]
       end
     end
     
-    client.write ['start']
-    client.take ['finish']
+    write ['start']
+    take ['finish']
   end
 end

data/example/read-in-trans.rb

@@ -4,53 +4,53 @@ Thread.abort_on_exception = true
 
 N = 5
 
-Tupelo.application do |app|
+Tupelo.application do
   N.times do |i|
-    app.child do |client|
+    child do
       Thread.new do
         step = 0
         loop do
           step += 1
-          client.transaction do |t|
-            t.read ["enabled", i]
-            t.write ["output", i, step]
+          transaction do
+            read ["enabled", i]
+            write ["output", i, step]
           end
           sleep 0.2
         end
       end
       
-      client.read ["done"]
+      read ["done"]
       exit
     end
   end
   
-  app.child do |client|
+  child do
     t = Thread.new do
       loop do
-        msg, i, step = client.take [nil, nil, nil]
+        msg, i, step = take [nil, nil, nil]
         printf "%20s from %2d at step %3d\n", msg, i, step
       end
     end
 
     puts "Turning on 0 and 4"
-    client.write ["enabled", 0]
-    client.write ["enabled", 4]
+    write ["enabled", 0]
+    write ["enabled", 4]
     sleep 2
     
     puts "Turning off 0"
-    client.take ["enabled", 0]
+    take ["enabled", 0]
     sleep 2
     
     puts "Turning off 4"
-    client.take ["enabled", 4]
+    take ["enabled", 4]
     sleep 2
     
     puts "Turning on 1 and 3"
-    client.write ["enabled", 1]
-    client.write ["enabled", 3]
+    write ["enabled", 1]
+    write ["enabled", 3]
     sleep 2
     
     puts "Bye!"
-    client.write ["done"]
+    write ["done"]
   end
 end

data/example/subspace.rb

@@ -0,0 +1,34 @@
+require 'tupelo/app'
+
+Tupelo.application do
+  local do
+    log [subscribed_all, subscribed_tags]
+
+    use_subspaces!
+
+    define_subspace(
+      tag:          "foo",
+      template:     [
+        {type: "number"}
+      ]
+    )
+
+    write_wait [0]
+
+    log read_all(Object)
+  end
+  
+  cid = child subscribe: ["foo"] do
+    log [subscribed_all, subscribed_tags]
+    write [1]
+    write_wait ["abc"]
+    log read_all(Object)
+  end
+  Process.wait cid
+
+  local do
+    log [subscribed_all, subscribed_tags]
+    log read_all(Object)
+    log read_all(subspace "foo")
+  end
+end

data/example/take-nowait.rb

@@ -1,6 +1,7 @@
 # Run this with --trace to see that, even in the FAIL case, take_nowait never
 # hangs waiting for a match. The "ready" tuple is just to keep the take
 # requests fairly close in time, increasing the chance of transaction failure.
+# Exactly one of the contending clients will take the tuple.
 
 require 'tupelo/app'
 

data/example/tcp.rb

@@ -28,11 +28,11 @@ port = 9901 # Use 0 to let system choose free port
 Tupelo.application servers_file: svr,
             seqd_addr:  [:tcp, '0.0.0.0', port],
             cseqd_addr: [:tcp, '0.0.0.0', port + 1],
-            arcd_addr:  [:tcp, '0.0.0.0', port + 2] do |app|
-  if app.owns_servers
+            arcd_addr:  [:tcp, '0.0.0.0', port + 2] do
+  if owns_servers
     puts "server started; ^C to stop"
     puts "run in another terminal: ../bin/tup tcp.yaml"
-    if app.log.level > Logger::INFO
+    if log.level > Logger::INFO
       puts "(run with --info or --trace to see events)"
     end
     sleep

data/example/timeout-trans.rb

@@ -4,17 +4,17 @@
 
 require 'tupelo/app'
 
-Tupelo.application do |app|
-  app.child do |client|
+Tupelo.application do
+  child do
     result =
       begin
-        client.transaction timeout: 1 do |t|
-          t.take ["foo"]
+        transaction timeout: 1 do
+          take ["foo"]
         end
       rescue TimeoutError => ex
         ex
       end
-    client.log "result = #{result.inspect}"
+    log "This should be a timeout error: #{result.inspect}"
   end
 end
 

data/example/timeout.rb

@@ -3,25 +3,26 @@
 
 require 'tupelo/app'
 
-Tupelo.application do |app|
-  app.child do |client|
+Tupelo.application do
+  child do
     begin
       n_sec = 2
       Timeout.timeout n_sec do
-        client.log "waiting for non-existing tuple #{[0]}"
-        client.read [0]
+        log "waiting for non-existing tuple #{[0]}"
+        read [0]
       end
     rescue TimeoutError
+      log "stopped waiting"
     end
     
-    r = client.read [1]
-    client.log "got #{r}"
+    r = read [1]
+    log "got #{r}"
   end
 
-  app.child do |client|
+  child do
     sleep 1
-    client.log "writing [1]"
-    client.write [1]
+    log "writing [1]"
+    write [1]
   end
 end