tupelo 0.16 → 0.17

data/example/consistent-hash.rb

@@ -0,0 +1,73 @@
+# TODO make more interesting by changing the set of workers over time.
+
+require 'tupelo/app'
+require 'tupelo/util/bin-circle'
+
+N_BINS = 5
+N_REPS = 30 # of each bin, to make distribution more uniform
+N_ITER = 1000
+
+Tupelo.application do
+  local do
+    use_subspaces!
+
+    N_BINS.times do |id|
+      define_subspace id, [id, Numeric, Numeric]
+    end
+
+    define_subspace "sum", ["sum", Numeric, Numeric, Numeric]
+  end
+
+  circle = BinCircle.new
+
+  N_BINS.times do |id|
+    circle.add_bin id, reps: N_REPS
+  end
+
+  N_BINS.times do |id|
+    child subscribe: [id], passive: true do
+      # take things belonging to the process's bin
+      count = 0
+      at_exit {log "load: #{count}"}
+
+      loop do
+        _, n1, n2 = take [id, Numeric, Numeric]
+          # could use take {|| ...} to optimistically start long computation
+        write ["sum", n1, n2, n1+n2]
+        count += 1
+      end
+
+      # the following loop is faster, but not horiz. scalable
+      if false
+        read [id, Numeric, Numeric] do |_, n1, n2|
+          write ["sum", n1, n2, n1+n2]
+          count += 1
+        end
+      end
+    end
+  end
+  
+  local subscribe: ["sum"] do
+    srand(12345)
+
+    Thread.new do
+      N_ITER.times do  |i|
+        ns = [rand(100), rand(100)]
+        bin_id = circle.find_bin(ns)
+        write [bin_id, *ns]
+      end
+    end
+    
+    N_ITER.times do |i|
+      _, n1, n2, sum = take ["sum", Numeric, Numeric, Numeric]
+      unless n1 + n2 == sum
+        log.error "bad sum"
+      end
+      q,r = (100*(i+1)).divmod N_ITER
+      if r == 0
+        printf "\r%3d%", q
+      end
+    end
+    puts
+  end
+end

data/example/map-reduce/prime-factor-balanced.rb

@@ -1,6 +1,5 @@
-# Factor numbers using remote hosts. Run with --trace to see contention.
-# This is more "map" than "map-reduce", though you could aggregate the
-# factored numbers, such as by finding the largest prime factor.
+# Modified prime-factor.rb attempts to balance load better. Improvement
+# varies--typically around 50% faster with 12 remote hosts.
 
 require 'tupelo/app/remote'
 
@@ -9,28 +8,73 @@ hosts = hosts.split(",")
 
 Tupelo.tcp_application do
   hosts.each_with_index do |host, hi|
-    remote host: host, passive: true, eval: %{
+    remote host: host, passive: true, log: true, eval: %{
       require 'prime' # ruby stdlib for prime factorization
       class M
-        def initialize nh, hi
+        def initialize nh, hi, excl = []
           @nh, @hi = nh, hi
+          @excl = excl
         end
         def === x
           Array === x and
             x[0] == "input" and
-            x[1] % @nh == @hi
+            x[1] % @nh == @hi and
+            not @excl.include? x[1]
+        end
+        def exclude *y
+          self.class.new @nh, @hi, @excl + y
         end
       end
       my_pref = M.new(#{hosts.size}, #{hi})
+
       loop do
-        _, input =
+        txn = transaction
+        begin
+          _, input = txn.take_nowait(my_pref)
+        rescue TransactionFailure => ex
+          next
+        end
+
+        if input
           begin
-            take(my_pref, timeout: 1.0) # fewer fails (5.0 -> none at all)
-          rescue TimeoutError
-            take(["input", Integer])
+            txn.commit
+            output = input.prime_division
+            Thread.new do
+              begin
+                txn.wait
+              rescue TransactionFailure
+                # someone else got it
+              else
+                write ["output", input, output]
+              end
+            end
+          rescue TransactionFailure
           end
+          my_pref = my_pref.exclude input
+          next
+        end
+
+        begin
+          txn.cancel
+        rescue TransactionFailure
+        end
+        break
+      end
+        
+      loop do
+        _, input = take(["input", Integer])
         write ["output", input, input.prime_division]
       end
+        
+
+#        _, input =
+#          begin
+#            take(my_pref, timeout: 1.0) # fewer fails (5.0 -> none at all)
+#          rescue TimeoutError
+#            take(["input", Integer])
+#          end
+#        write ["output", input, input.prime_division]
+#      end
     }
   end
 

data/example/multi-tier/kvspace.rb

@@ -4,7 +4,7 @@
 #
 # Unlike in a key-value store, a given key_string may occur more than once.
 # It is up to the application to decide whether to enforce key uniqueness or
-# not (for example, by taking (k,...) before writing (k,v).
+# not (for example, by taking [k,...] before writing [k,v]).
 #
 # This store should be used only by clients that subscribe to a subspace
 # that can be represented as pairs. (See memo2.rb.)

data/example/multi-tier/memo2.rb

@@ -16,14 +16,11 @@ fork do
     local do
       use_subspaces!
 
-      define_subspace(
-        tag:          "memo",
-        template:     [
-          {value: "memo"},    # tag is encoded in each tuple, for recognizing
-          {type:  "string"},  # key in the cache, must be string
-          nil                 # value, can be any object (e.g. JSON object)
-        ]
-      )
+      define_subspace("memo", [
+        "memo",   # tag is encoded in each tuple, for recognizing
+        String,   # key in the cache, must be string
+        nil       # value, can be any object (e.g. JSON object)
+      ])
     end
 
     child tuplespace: [KVSpace, "memo"], subscribe: ["memo"] do |client|

data/example/multi-tier/multi-sinatras.rb

@@ -5,7 +5,7 @@
 #
 # Depends on the sinatra and http gems.
 
-PORTS = [9001, 9002 , 9003]
+PORTS = [9001, 9002, 9003]
 
 fork do
   require 'tupelo/app'

data/example/subspaces/addr-book.rb

@@ -35,43 +35,34 @@ Tupelo.application do
     use_subspaces!
 
     # Subspace for tuples belonging to the addr book.
-    define_subspace(
-      tag:          ab_tag,
-      template:     [
-        {value: ab_tag},
-        {type:  "string"},  # name
-        nil                 # address; can be any object
-      ]
-    )
+    define_subspace(ab_tag, [
+      ab_tag,
+      String,   # name
+      nil       # address; can be any object
+    ])
     
     # Subspace for commands for fetch, delete, first, last, prev, next.
     # We can't use #read and #take for fetch and delete because then the
     # requesting client would have to subscribe to the ab_tag subspace.
-    define_subspace(
-      tag:          cmd_tag,
-      template:     [
-        {value: cmd_tag},
-        nil,                # request id, such as [client_id, uniq_id]
-        {type:  "string"},  # cmd name
-        {type:  "list"}     # arguments
-      ]
-    )
+    define_subspace(cmd_tag, [
+      cmd_tag,
+      nil,      # request id, such as [client_id, uniq_id]
+      String,   # cmd name
+      Array     # arguments
+    ])
 
     # Subspace for responses to commands. A response identifies the command
     # it is responding to in two ways: by copying it and by an id. The
     # former is so that another client can "spy" on one client's query
     # responses, perhaps saving effort. The latter is to distinguish between
     # iterations of the same command (first, first, ...).
-    define_subspace(
-      tag:          resp_tag,
-      template:     [
-        {value: resp_tag},
-        nil,                # in response to this request id
-        {type:  "string"},  # cmd name
-        {type:  "list"},    # arguments
-        nil,                # result of query -- type depends on command
-      ]
-    )
+    define_subspace(resp_tag, [
+      resp_tag,
+      nil,      # in response to this request id
+      String,   # cmd name
+      Array,    # arguments
+      nil,      # result of query -- type depends on command
+    ])
   end
 
   N_REPLICAS.times do |i|

data/example/subspaces/pubsub.rb

@@ -19,22 +19,10 @@ Tupelo.application do
     use_subspaces!
 
     N_CHAN.times do |i|
-      define_subspace(
-        tag:          i,
-        template:     [
-          {value: i},
-          {type: "string"}
-        ]
-      )
+      define_subspace i, [i, String]
     end
 
-    define_subspace(
-      tag:          "control",
-      template:     [
-        {value: "control"},
-        nil
-      ]
-    )
+    define_subspace "control", ["control", nil]
   end
   
   N_PUBS.times do |pi|

data/example/subspaces/ramp.rb

@@ -45,31 +45,24 @@ Tupelo.application do
   local do
     use_subspaces!
     
-    define_subspace(
-      tag:          "x",
-      template:     {
-        x:      {type:  "number"},    # data payload
-        id:     {type:  "list"},      # [client_id, local_id]
-        final:  {type:  "boolean"}    # false means pending
-      }
-    )
+    bool = PortableObjectTemplate::BOOLEAN
     
-    define_subspace(
-      tag:          "y",
-      template:     {
-        y:      {type:  "number"},    # data payload
-        id:     {type:  "list"},      # [client_id, local_id]
-        final:  {type:  "boolean"}    # false means pending
-      }
-    )
-
-    define_subspace(
-      tag:          "ack", # could make this per-client
-      template:     {
-        ack:    {type:  "string"},    # state ack-ed: "pending"
-        id:     {type:  "list"}       # [client_id, local_id]
-      }
-    )
+    define_subspace("x", {
+      x:      Numeric,  # data payload
+      id:     Array,    # [client_id, local_id]
+      final:  bool      # false means pending
+    })
+    
+    define_subspace("y", {
+      y:      Numeric,  # data payload
+      id:     Array,    # [client_id, local_id]
+      final:  bool      # false means pending
+    })
+
+    define_subspace("ack", { # could make this per-client
+      ack:    String,   # state ack-ed: "pending"
+      id:     Array     # [client_id, local_id]
+    })
   end
 
   X_REPLICATIONS.times do |xi|

data/example/subspaces/shop/shop-v2.rb

@@ -12,14 +12,11 @@ Tupelo.application do
   local do
     use_subspaces!
 
-    define_subspace(
-      tag:          "inventory",
-      template:     [
-        {value: "product"},
-        nil,                  # product_id
-        {type:  "number"}     # count
-      ]
-    )
+    define_subspace("inventory", [
+      "product",
+      nil,       # product_id
+      Integer    # count
+    ])
 
     PRODUCT_IDS.each do |product_id|
       count = 10

data/example/subspaces/simple.rb

@@ -9,15 +9,7 @@ Tupelo.application do
     log [subscribed_all, subscribed_tags]
 
     use_subspaces!
-
-    define_subspace(
-      tag:          "foo",
-      template:     [
-        {type: "number"}
-      ]
-    )
-
-    write_wait [0]
+    define_subspace "foo", [Numeric]
 
     log read_all(Object)
   end

data/example/{fish.rb → wip/fish.rb}

@@ -1,3 +1,7 @@
+# This example doesn't work yet because there is no way to indicate that,
+# instead of waiting, it would be better to continue searching the tuplespace,
+# backtracking around the [Integer, String] tuple that did not have a mate.
+
 require 'tupelo/app'
 
 Tupelo.application do

data/example/wip/fish2.rb

@@ -0,0 +1,59 @@
+# This works, but requires a fix-up step to clean up after a race condition
+# during counter initialization.
+
+require 'tupelo/app'
+
+Tupelo.application do
+  2.times do
+    child passive: true do
+      loop do
+        transaction do
+          fish, _ = take([String])
+          n, _ = take_nowait([Integer, fish])
+          if n
+            write [n + 1, fish]
+          else
+            write [1, fish] # another process might also write this, so ...
+            write ["fixup", fish]
+          end
+        end
+      end
+    end
+  end
+
+  child passive: true do
+    loop do
+      transaction do # fix up the two counter tuples
+        _, fish = take ["fixup", String]
+        n1, _ = take_nowait [Integer, fish]
+        if n1
+          n2, _ = take_nowait [Integer, fish]
+          if n2
+            #log "fixing: #{[n1 + n2, fish]}"
+            write [n1 + n2, fish]
+          else
+            write [n1, fish] # partial rollback
+          end
+        end
+      end
+    end
+  end
+
+  local do
+    seed = 3
+    srand seed
+    log "seed = #{seed}"
+    
+    fishes = %w{ trout marlin char salmon }
+
+    a = fishes * 10
+    a.shuffle!
+    a.each do |fish|
+      write [fish]
+    end
+
+    fishes.each do |fish|
+      log take [10, fish]
+    end
+  end
+end

data/lib/tupelo/app/builder.rb

@@ -68,6 +68,8 @@ module Tupelo
     # the passive flag for processes that wait for tuples and respond in some
     # way. Then you do not have to manually interrupt the whole application when
     # the active processes are done. See examples.
+    #
+    # Returns pid of child process.
     def child client_class = Client, passive: false, **opts, &block
       ez.child :seqd, :cseqd, :arcd, passive: passive do |seqd, cseqd, arcd|
         opts = {seq: seqd, cseq: cseqd, arc: arcd, log: log}.merge(opts)

data/lib/tupelo/client/subspace.rb

@@ -0,0 +1,36 @@
+class Tupelo::Client
+  module Api
+    def define_subspace tag, template, addr: nil
+      metatuple = {
+        __tupelo__: "subspace",
+        tag:        tag,
+        template:   PortableObjectTemplate.spec_from(template),
+        addr:       addr
+      }
+      write_wait metatuple
+    end
+
+    # call this just once at start of first client (it's optional to
+    # preserve behavior of non-subspace-aware code)
+    def use_subspaces!
+      return if subspace(TUPELO_SUBSPACE_TAG)
+      define_subspace(TUPELO_SUBSPACE_TAG, {
+        __tupelo__: "subspace",
+        tag:        nil,
+        template:   nil,
+        addr:       nil
+      })
+    end
+
+    def subspace tag
+      tag = tag.to_s
+      worker.subspaces.find {|sp| sp.tag == tag} or begin
+        if subscribed_tags.include? tag
+          read __tupelo__: "subspace", tag: tag, addr: nil, template: nil
+          worker.subspaces.find {|sp| sp.tag == tag}
+        end
+      end
+      ## this impl will not be safe with dynamic subspaces
+    end
+  end
+end

data/lib/tupelo/client/transaction.rb

@@ -71,8 +71,11 @@ class Tupelo::Client
     def take template, timeout: nil
       transaction timeout: timeout do |t|
         tuple = t.take template
-        yield tuple if block_given?
-        tuple
+        if block_given?
+          yield tuple
+        else
+          tuple
+        end
       end
     end
 

data/lib/tupelo/client/tuplespace.rb

@@ -42,8 +42,8 @@ class Tupelo::Client
     def each(*); end
     def delete_once(*); end
     def insert(*); self; end
-    def find_distinct_matches_for(*); raise; end ##?
-    def find_match_for(*); raise; end ##?
+    def find_distinct_matches_for(*); raise; end
+    def find_match_for(*); raise; end
     def clear; end
     
     ## should store space metadata, so outgoing writes can be tagged

data/lib/tupelo/client/worker.rb

@@ -351,12 +351,9 @@ class Tupelo::Client
         end
 
         take_tuples.each do |tuple|
-          ### abstract this out
-          if tuple.kind_of? Hash and tuple.key? "__tupelo__"
-            if tuple["__tupelo__"] == "subspace" # tuple is subspace metatdata
-              ## do some error checking
-              subspaces.delete_if {|sp| sp.tag == tuple["tag"]}
-            end
+          if is_meta_tuple? tuple
+            ## do some error checking
+            subspaces.delete_if {|sp| sp.tag == tuple["tag"]}
           end
         end
       end
@@ -435,13 +432,17 @@ class Tupelo::Client
       end
     end
 
+    # Returns true if tuple is subspace metadata.
+    def is_meta_tuple? tuple
+      tuple.kind_of? Hash and tuple.key? "__tupelo__" and
+        tuple["__tupelo__"] == "subspace"
+    end
+
     def sniff_meta_tuple tuple
-      if tuple.kind_of? Hash and tuple.key? "__tupelo__"
-        if tuple["__tupelo__"] == "subspace" # tuple is subspace metatdata
-          ## do some error checking
-          ## what if subspace already exists?
-          subspaces << Subspace.new(tuple, self)
-        end
+      if is_meta_tuple? tuple
+        ## do some error checking
+        ## what if subspace already exists?
+        subspaces << Subspace.new(tuple, self)
       end
     end
 

data/lib/tupelo/client.rb

@@ -4,6 +4,7 @@ module Tupelo
   class Client < Funl::Client
     require 'tupelo/client/worker'
     require 'tupelo/client/tuplespace'
+    require 'tupelo/client/subspace'
 
     include Api
 
@@ -60,37 +61,5 @@ module Tupelo
         super().unknown *args
       end
     end
-    
-    ## do these belong in API module?
-    def define_subspace metatuple
-      defaults = {__tupelo__: "subspace", addr: nil}
-      write_wait defaults.merge!(metatuple)
-    end
-    
-    # call this just once at start of first client (it's optional to
-    # preserve behavior of non-subspace-aware code)
-    def use_subspaces!
-      return if subspace(TUPELO_SUBSPACE_TAG)
-      define_subspace(
-        tag:          TUPELO_SUBSPACE_TAG,
-        template:     {
-          __tupelo__: {value: "subspace"},
-          tag:        nil,
-          addr:       nil,
-          template:   nil
-        }
-      )
-    end
-
-    def subspace tag
-      tag = tag.to_s
-      worker.subspaces.find {|sp| sp.tag == tag} or begin
-        if subscribed_tags.include? tag
-          read __tupelo__: "subspace", tag: tag, addr: nil, template: nil
-          worker.subspaces.find {|sp| sp.tag == tag}
-        end
-      end
-      ## this impl will not be safe with dynamic subspaces
-    end
   end
 end