tupelo 0.13 → 0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a657d2c56962432faa339d693f6856f409249f6
-  data.tar.gz: 64a58b3ab627151aeb2fc8965de21a39c1ebf3f6
+  metadata.gz: f4eb65df5eb526b3a885d5f0ce41736bb3ccc216
+  data.tar.gz: 96f7e2b11e86b299af62b6598081564e98ceb159
 SHA512:
-  metadata.gz: 7ea7042705c178a855d4d9ad621c2b65433d0adb4ff7dd75ee855f75d4b53d2bc4301ea91f3491bfaf2847294076a9edffa176290192d004488bdf87caa0b016
-  data.tar.gz: 7e371d515db7f86bd95c353fcd5153efd5d1f34ba6edb0c75b7602994309509615f602ba7463729326f3c8d3aa55ba290e67c0e24fac07e917d61e72d668b886
+  metadata.gz: bd91077b980c38858cb3b4549cec5df43ccb91a58bc209c026fe9db23c9ec6a90486c1cecd8f92c564d420fa26ef00cda93fa6b92e6e9af171d9abd23af78717
+  data.tar.gz: b5de5df9f3e7baf6a02a4440b05e0cbb9a3fa04d37d5e46db3e0950ff5d59e29c23ac49809a9810f94fdd1a54916c47c98ce05a651167e72c9cfe8d60bd227ea

data/example/map-reduce/prime-factor-balanced.rb

@@ -0,0 +1,55 @@
+# Factor numbers using remote hosts. Run with --trace to see contention.
+# This is more "map" than "map-reduce", though you could aggregate the
+# factored numbers, such as by finding the largest prime factor.
+
+require 'tupelo/app/remote'
+
+hosts = ARGV.shift or abort "usage: #$0 <ssh-hostname>,<ssh-hostname>,..."
+hosts = hosts.split(",")
+
+Tupelo.tcp_application do
+  hosts.each_with_index do |host, hi|
+    remote host: host, passive: true, eval: %{
+      require 'prime' # ruby stdlib for prime factorization
+      class M
+        def initialize nh, hi
+          @nh, @hi = nh, hi
+        end
+        def === x
+          Array === x and
+            x[0] == "input" and
+            x[1] % @nh == @hi
+        end
+      end
+      my_pref = M.new(#{hosts.size}, #{hi})
+      loop do
+        _, input =
+          begin
+            take(my_pref, timeout: 1.0) # fewer fails (5.0 -> none at all)
+          rescue TimeoutError
+            take(["input", Integer])
+          end
+        write ["output", input, input.prime_division]
+      end
+    }
+  end
+
+  local do
+    t0 = Time.now
+    inputs = 1_000_000_000_000 .. 1_000_000_000_050
+
+    inputs.each do |input|
+      write ["input", input]
+    end
+
+    inputs.size.times do
+      _, input, outputs = take ["output", Integer, nil]
+      output_str = outputs.map {|prime, exp|
+        exp == 1 ? prime : "#{prime}**#{exp}"}.join(" * ")
+      log "#{input} == #{output_str}"
+    end
+    
+    t1 = Time.now
+    log "elapsed: %6.2f seconds" % (t1-t0)
+  end
+end

data/example/map-reduce/prime-factor.rb

@@ -0,0 +1,39 @@
+# Factor numbers using remote hosts. Run with --trace to see contention.
+# This is more "map" than "map-reduce", though you could aggregate the
+# factored numbers, such as by finding the largest prime factor.
+
+require 'tupelo/app/remote'
+
+hosts = ARGV.shift or abort "usage: #$0 <ssh-hostname>,<ssh-hostname>,..."
+hosts = hosts.split(",")
+
+Tupelo.tcp_application do
+  hosts.each do |host|
+    remote host: host, passive: true, eval: %{
+      require 'prime' # ruby stdlib for prime factorization
+      loop do
+        _, input = take(["input", Integer])
+        write ["output", input, input.prime_division]
+      end
+    }
+  end
+
+  local do
+    t0 = Time.now
+    inputs = 1_000_000_000_000 .. 1_000_000_000_050
+
+    inputs.each do |input|
+      write ["input", input]
+    end
+
+    inputs.size.times do
+      _, input, outputs = take ["output", Integer, nil]
+      output_str = outputs.map {|prime, exp|
+        exp == 1 ? prime : "#{prime}**#{exp}"}.join(" * ")
+      log "#{input} == #{output_str}"
+    end
+    
+    t1 = Time.now
+    log "elapsed: %6.2f seconds" % (t1-t0)
+  end
+end

data/example/multi-tier/kvspace.rb

@@ -32,7 +32,7 @@ class KVSpace
   def each
     hash.each do |k, vs|
       vs.each do |v|
-        yield tag, k, v
+        yield [tag, k, v]
       end
     end
     metas.each do |tuple|
@@ -67,7 +67,7 @@ class KVSpace
 
     else
       if i=metas.index(tuple)
-        delete_at i
+        metas.delete_at i
       end
     end
   end

data/example/subspaces/addr-book.rb

@@ -0,0 +1,145 @@
+# Example of attaching a data structure to a subspace. In this case, we
+# use an in-memory structure, a red-black tree, to maintain the tuples in
+# sorted order. (For a simpler example, with a hash instead of a tree,
+# see [memo example using subspaces](../multi-tier/memo2.rb). The process(es)
+# that manages the rbtree needs to subscribe to this subspace, so it can
+# apply writes to the rbtree.
+#
+# We also have subspaces for query commands and responses so that other clients
+# can access the sorted structure. The process(es) that host the rbtree also
+# subscribe to the command subspace (and write to, but not subscribe to, the
+# response subspace.) The process that query
+# do so by writing to the command subspace and subscribing to the response
+# subspace.
+#
+# This is kinda like redis, but the data is distributed, not stored on the same
+# process that is managing concurrency. Multiple replicas increase concurrency.
+# Run this example with --show-handlers to see which replicas are responding.
+#
+# Note that a subspace can be sharded to different clients, and different
+# clients can each use their own data structure for these tuples.
+
+require 'tupelo/app'
+require_relative 'sorted-set-space'
+
+SHOW_HANDLERS = ARGV.delete("--show-handlers")
+
+N_REPLICAS = 3
+
+ab_tag = "my address book"
+cmd_tag = "#{ab_tag} commands"
+resp_tag = "#{ab_tag} responses"
+
+Tupelo.application do
+  local do
+    use_subspaces!
+
+    # Subspace for tuples belonging to the addr book.
+    define_subspace(
+      tag:          ab_tag,
+      template:     [
+        {value: ab_tag},
+        {type:  "string"},  # name
+        nil                 # address; can be any object
+      ]
+    )
+    
+    # Subspace for commands for fetch, delete, first, last, prev, next.
+    # We can't use #read and #take for fetch and delete because then the
+    # requesting client would have to subscribe to the ab_tag subspace.
+    define_subspace(
+      tag:          cmd_tag,
+      template:     [
+        {value: cmd_tag},
+        nil,                # request id, such as [client_id, uniq_id]
+        {type:  "string"},  # cmd name
+        {type:  "list"}     # arguments
+      ]
+    )
+
+    # Subspace for responses to commands. A response identifies the command
+    # it is responding to in two ways: by copying it and by an id. The
+    # former is so that another client can "spy" on one client's query
+    # responses, perhaps saving effort. The latter is to distinguish between
+    # iterations of the same command (first, first, ...).
+    define_subspace(
+      tag:          resp_tag,
+      template:     [
+        {value: resp_tag},
+        nil,                # in response to this request id
+        {type:  "string"},  # cmd name
+        {type:  "list"},    # arguments
+        nil,                # result of query -- type depends on command
+      ]
+    )
+  end
+
+  N_REPLICAS.times do |i|
+    # Inserts are just writes, which are handled by Worker and SortedSetSpace,
+    # so this child's app loop only needs to handle the special commands.
+    child tuplespace: [SortedSetSpace, ab_tag],
+          subscribe: [ab_tag, cmd_tag], passive: true do
+
+      log.progname = "replica ##{i}"
+
+      loop do
+        _, rqid, cmd, args = take(subspace cmd_tag)
+        if SHOW_HANDLERS
+          log "handling request for #{cmd} #{args}"
+        end
+
+        case cmd
+        when "delete"                # handled by one replica
+          args.each do |name|
+            take [ab_tag, name, nil] # propagates to all replicas
+          end
+
+        when "fetch"
+          _, _, addr = read_nowait [ab_tag, args[0], nil] # addr might be nil
+          write [resp_tag, rqid, cmd, args, addr]
+
+        when "next", "prev", "first", "last"
+          _, name, addr = read_nowait SortedSetTemplate[ab_tag, cmd, *args]
+          write [resp_tag, rqid, cmd, args, [name, addr]]
+
+        else # maybe write an error message in a tuple
+          log.error "bad command: #{cmd}"
+        end
+      end
+    end
+  end
+  
+  child subscribe: resp_tag do
+    log.progname = "user agent"
+
+    counter = 0 # this is a bit hacky -- could use prev txn's global tick
+    next_rqid = proc { [client_id, counter+=1] }
+      # Protect this with a mutex if other threads need it.
+
+    # write some ab entries
+    write [ab_tag, "Eliza", "100 E St."]
+    write [ab_tag, "Alice", "100 A St."]
+    write [ab_tag, "Daisy", "100 D St."]
+    write [ab_tag, "Bob", "100 B St."]
+    write [ab_tag, "Charles", "100 C St."]
+
+    # make some queries
+    rqid = next_rqid.call
+    name = "Daisy"
+    write [cmd_tag, rqid, "fetch", [name]]
+    addr = take( [resp_tag, rqid, nil, nil, nil] ).last
+    log "found: #{name} => #{addr}"
+    
+    rqid = next_rqid.call
+    write [cmd_tag, rqid, "first", []]
+    name, addr = take( [resp_tag, rqid, nil, nil, nil] ).last
+    log "first entry: #{name} => #{addr}"
+    
+    5.times do
+      rqid = next_rqid.call
+      write [cmd_tag, rqid, "next", [name]]
+      name, addr = take( [resp_tag, rqid, nil, nil, nil] ).last
+      log( name ? "next entry: #{name} => #{addr}" : "no more entries" )
+    end
+  end
+end

data/example/subspaces/ramp.rb

@@ -0,0 +1,146 @@
+# Read-atomic multipartition transactions, as per:
+#   http://www.youtube.com/watch?v=_rAdJkAbGls (around minutes 28-30)
+#   http://www.bailis.org/blog/non-blocking-transactional-atomicity
+#
+# Example of transacting separately on two subspaces (i.e. shardable subsets of
+# the tuplespace), but hiding intermediate tuples so that the results show up
+# atomically when readers look for them. (Note that this is different from
+# atomically transacting on replicas of the same shard, which is inherent in
+# tupelo transactions.)
+#
+# In tupelo, we could use the classic tuplespace technique of taking a lock
+# tuple to protect the sequence ot two transactions on the two subspaces, but
+# that would reduce concurrency and require a lease mechanism in case the lock
+# holder dies. That's possible, but not scalable. So we use transactions with a
+# trick...
+#
+# Tupelo doesn't allow transactions to cross subspace boundaries (except in the
+# special case of writes outside of a subspace--see
+# [doc/subspace.md](doc/subspace.md)). We can get around this at the application
+# level, with a few extra steps. This adds latency, but preserves effective
+# atomicity from the application's point of view and does not introduce any
+# fragile locks or blocking. The main trick (as in Bailis's talk) is to use a
+# globally unique value -- in his talk he used a transaction id. We could use
+# the global_tick of a successful transaction (same idea) or a unique id based
+# on client_id -- see [example/uniq-id.rb](example/uniq-id.rb).
+
+# todo: use a smarter data structure for the x and y subspaces
+
+require 'tupelo/app'
+
+N_ITER = 6
+X_REPLICATIONS = 1 # number of copies of the shard of X data
+Y_REPLICATIONS = 1 # number of copies of the shard of Y data
+
+def next_local_id
+  @counter = 0
+  @counter += 1
+    # Protect this with a mutex or queue if other threads need it, or
+    # use the atomic gem. It's ok in a multiprocess app without mutex,
+    # because each process has its own copy.
+end
+
+Tupelo.application do
+
+  local do
+    use_subspaces!
+    
+    define_subspace(
+      tag:          "x",
+      template:     {
+        x:      {type:  "number"},    # data payload
+        id:     {type:  "list"},      # [client_id, local_id]
+        final:  {type:  "boolean"}    # false means pending
+      }
+    )
+    
+    define_subspace(
+      tag:          "y",
+      template:     {
+        y:      {type:  "number"},    # data payload
+        id:     {type:  "list"},      # [client_id, local_id]
+        final:  {type:  "boolean"}    # false means pending
+      }
+    )
+
+    define_subspace(
+      tag:          "ack", # could make this per-client
+      template:     {
+        ack:    {type:  "string"},    # state ack-ed: "pending"
+        id:     {type:  "list"}       # [client_id, local_id]
+      }
+    )
+  end
+
+  X_REPLICATIONS.times do |xi|
+    child subscribe: ["x"], passive: true do
+      log.progname = "x#{xi}"
+
+      read x: nil, id: nil, final: nil do |t|
+        log t
+        if t["final"]
+          # co-writes are at least pending at this point in global time
+          # ("stable"), so remove pending tuple when final tuple exists.
+          # First responding replica wins, and the take propagates to others.
+          take_nowait t.merge(final: false)
+        else
+          write ack: "pending", id: t["id"]
+        end
+      end
+    end
+  end
+
+  Y_REPLICATIONS.times do |yi|
+    child subscribe: ["y"], passive: true do
+      log.progname = "y#{yi}"
+
+      read y: nil, id: nil, final: nil do |t|
+        log t
+        if t["final"]
+          # co-writes are at least pending at this point in global time
+          # ("stable"), so remove pending tuple when final tuple exists.
+          # First responding replica wins, and the take propagates to others.
+          take_nowait t.merge(final: false)
+        else
+          write ack: "pending", id: t["id"]
+        end
+      end
+    end
+  end
+
+  child subscribe: ["ack"] do
+    # Does not subscribe to x or y, so can only write to those spaces.
+    log.progname = "writer"
+    
+    N_ITER.times do |i|
+      uniq_id = [client_id, next_local_id]
+
+      x = {x: i, id: uniq_id}
+      y = {y: i, id: uniq_id}
+
+      write x.merge(final: false), y.merge(final: false) # pending
+      (X_REPLICATIONS + Y_REPLICATIONS).times do
+        take ack: "pending", id: uniq_id  # wait for one to be pending
+      end
+      write x.merge(final: true), y.merge(final: true)
+
+      # Note that each of the two above writes is a multi-space transaction
+      # which is allowed because it is purely writes (no reads or takes).
+      # However, this only guarantees read atomicity for tupelo clients (because
+      # of the global transaction ordering). If some processes are accessing the
+      # x and y data stores through protocols other than tupelo (such as sql
+      # over sockets), this is not enough--they could see inconsistent state.
+      # Hence the explicit wait for an ack to truly synchronize the state.
+    end
+  end
+
+  # This doesn't test that RAMP is working -- it will always see a consistent
+  # view because of tupelo, even without the pending/ack trick. It is more
+  # informative to look at the log output from the x and y clients.
+  child subscribe: ["x", "y"], passive: true do
+    log.progname = "reader"
+    read do |t|
+      log t
+    end
+  end
+end

data/example/subspaces/sorted-set-space.rb

@@ -1,12 +1,17 @@
 require 'rbtree'
 
+## TODO
+##
+## generalize SortedSetSpace to accept params that indicate which fields
+##   are key and value
+
 class SortedSetTemplate
   class << self
     alias [] new
   end
   
   # cmd can be "next", "prev", "first", "last"
-  # for next/prev, args is ["name"]
+  # for next/prev, args is [key]
   # for first/last, args is empty
   def initialize tag, cmd, *args
     @tag = tag
@@ -14,15 +19,38 @@ class SortedSetTemplate
     @args = args
   end
   
-  def === other
-    raise ### should not need this?
-  end
-  
-  def find_in rbtree
+  def find_in tree, distinct_from: []
+    # for simplicity, ignore distinct_from
+    # -- we never take/read multiple keys in the tree
     case @cmd
     when "first"
-      rbtree.first
-    ###
+      k, v = tree.first
+      k && [@tag, k, v.first]
+    when "last"
+      tree.last
+      k && [@tag, k, v.last]
+    when "prev"
+      k = @args[0]
+      (k1,v1),(k2,v2) = tree.bound(tree.first[0], k).last(2)
+        ## Bad rbtree! This will be much less efficient than "next".
+      if k == k2
+        k1 && [@tag, k1, v1.last]
+      else
+        k2 && [@tag, k2, v2.last]
+      end
+      ## anomaly: can't iterate through multivalues
+    when "next"
+      k = @args[0]
+      (k1,v1),(k2,v2) = tree.bound(k, tree.last[0]).first(2)
+        ## Bad rbtree! There is no bounded search with < (rather than <=)
+      if k == k1
+        k2 && [@tag, k2, v2.first]
+      else
+        k1 && [@tag, k1, v1.first]
+      end
+    else
+      raise "bad command"
+    end
   end
 end
 
@@ -30,19 +58,22 @@ end
 # The object may be any serializable object (built up from numbers, booleans,
 # nil, strings, hashes and arrays).
 #
-# Unlike in a key-value store, a given key_string may occur more than once.
+# By default, multiple values per key are allowed. (This differs from a typical
+# key-value store, in wihch a given key_string may occur only once.)
 # It is up to the application to decide whether to enforce key uniqueness or
 # not (for example, by taking (k,...) before writing (k,v).
 #
 # This store should be used only by clients that subscribe to a subspace
-# that can be represented as pairs. (See memo2.rb.)
+# that can be represented as triples (tag, key_string, value), where
+# the tag is a single literal value that is the same for all triples.
+# (See memo2.rb.)
 #
-# This store also manages meta tuples, which it keeps in an array, just like
-# the default Tuplespace class does.
+# This store also manages command and meta tuples, which it keeps in an array,
+# just like the default Tuplespace class does.
 class SortedSetSpace
   include Enumerable
 
-  attr_reader :tag, :hash, :metas
+  attr_reader :tag, :tree, :metas
 
   def initialize tag
     @tag = tag
@@ -50,17 +81,17 @@ class SortedSetSpace
   end
 
   def clear
-    @hash = Hash.new {|h,k| h[k] = []}
-      # It's up to the application to enforce that these arrays have size <=1.
+    @tree = RBTree.new{|t,k| t[k] = []}
+      # It's up to the application to enforce one entry per key.
     @metas = []
       # We are automatically subscribed to tupelo metadata (subspace defs), so
-      # we need to keep them somewhere.
+      # we need to keep them somewhere. Also, the command tuples.
   end
 
   def each
-    hash.each do |k, vs|
+    tree.each do |k, vs|
       vs.each do |v|
-        yield tag, k, v
+        yield [tag, k, v]
       end
     end
     metas.each do |tuple|
@@ -69,11 +100,11 @@ class SortedSetSpace
   end
 
   def insert tuple
-    if tuple.kind_of? Array
-        # and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
-        # This is redundant, because of subscribe.
-      t, k, v = tuple
-      hash[k] << v
+    if tuple.kind_of? Array and tuple.size == 3 and
+       tuple[0] == tag and tuple[1].kind_of? String
+
+      _, k, v = tuple
+      tree[k] << v
 
     else
       metas << tuple
@@ -81,13 +112,13 @@ class SortedSetSpace
   end
 
   def delete_once tuple
-    if tuple.kind_of? Array
-        # and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
-        # This is redundant, because of subscribe.
-      t, k, v = tuple
-      if hash.key?(k) and hash[k].include? v
-        hash[k].delete v
-        hash.delete k if hash[k].empty?
+    if tuple.kind_of? Array and tuple.size == 3 and
+       tuple[0] == tag and tuple[1].kind_of? String
+
+      _, k, v = tuple
+      if tree.key?(k) and tree[k].include? v
+        tree[k].delete v
+        tree.delete k if tree[k].empty?
         true
       else
         false
@@ -95,7 +126,7 @@ class SortedSetSpace
 
     else
       if i=metas.index(tuple)
-        delete_at i
+        metas.delete_at i
       end
     end
   end
@@ -119,7 +150,7 @@ class SortedSetSpace
   def find_match_for template, distinct_from: []
     case template
     when SortedSetTemplate
-      template.find_in rbtree, distinct_from: distinct_from ###
+      template.find_in tree, distinct_from: distinct_from
     else
       # fall back to linear search
       find do |tuple|

data/example/uniq-id.rb

@@ -0,0 +1,34 @@
+# The underlying messaging protocol, funl, keeps track of a unique id per
+# message. This id is awailable in the transaction object, but only after
+# commit has succeeded. We can get the id from the transaction object and
+# use it as a globally unique id.
+#
+# Another source of unique ids is the client id, which is unique per client.
+# You can get an id that is uniq per message by combining it with any value
+# that is unique, in that client, to the message, such as a counter.
+# This has the advantage of not requiring a transaction.
+
+require 'tupelo/app'
+
+Tupelo.application do
+  local do
+    tr = pulse_wait ["noop"] # returns transaction
+    uniq_id = tr.global_tick # available after transaction commits
+    log "unique id is #{uniq_id}"
+
+    # now, we can use that unique id in some other tuples
+    write foo: "bar", id: uniq_id
+    log take foo: nil, id: nil
+    
+    @counter = 0
+    next_local_id = proc { @counter+=1 }
+      # Protect this with a mutex or queue if other threads need it, or
+      # use the atomic gem.
+
+    cid = client_id
+    
+    uniq_id2 = [next_local_id.call, cid]
+    write foo: "baz", id: uniq_id2
+    log take foo: nil, id: nil
+  end
+end

data/lib/tupelo/client/transaction.rb

@@ -6,6 +6,7 @@ class Tupelo::Client
   class TransactionStateError < TransactionError; end
   class TransactionAbort < TransactionError; end
   class TransactionFailure < TransactionError; end
+  class TransactionSubspaceError < TransactionError; end
 
   module Api
     def trans_class
@@ -166,6 +167,10 @@ class Tupelo::Client
       client.client_id
     end
     
+    def subspace tag
+      client.subspace tag
+    end
+
     def log *args
       if args.empty?
         @log
@@ -345,8 +350,8 @@ class Tupelo::Client
 
     rescue TransactionAbort, Interrupt, TimeoutError => ex ## others?
       worker_push Unwaiter.new(self)
-      raise ex.class,
-        "#{ex.message}: client #{client_id} waiting for #{inspect}"
+      cstr = "client #{client_id} (#{log.progname})"
+      raise ex.class, "#{ex.message}: #{cstr} waiting for #{inspect}"
     end
 
     def value

data/lib/tupelo/client/worker.rb

@@ -527,15 +527,19 @@ class Tupelo::Client
           ## in case there is an optimization
         matcher.fails
       else
-        tuple = tuplespace.find_match_for waiter.template
+        tuple = tuplespace.find_match_for matcher.template
         if tuple
-          waiter.peek tuple
+          matcher.peek tuple
         else
           matcher.fails
         end
       end
     end
 
+    def collect_tags tuple
+      subspaces.select {|subspace| subspace === tuple}.map(&:tag)
+    end
+
     def send_transaction transaction
       msg = message_class.new
       msg.client_id = client_id
@@ -550,14 +554,33 @@ class Tupelo::Client
       reads = transaction.read_tuples_for_remote.compact
       
       unless msg.tags
-        tags = []
-        tuples = [writes, pulses, takes, reads].compact.flatten(1)
-        subspaces.each do |subspace|
-          tuples.each do |tuple|
-            if subspace === tuple
-              tags << subspace.tag
-              break
+        tags = nil
+        [takes, reads].compact.flatten(1).each do |tuple|
+          if tags
+            tuple_tags = collect_tags(tuple)
+            unless tuple_tags == tags
+              d = (tuple_tags - tags) + (tags - tuple_tags)
+              raise TransactionSubspaceError,
+                "tuples crossing subspaces: #{d} in #{transaction.inspect}"
             end
+          else
+            tags = collect_tags(tuple)
+          end
+        end
+        tags ||= []
+
+        write_tags = []
+        [writes, pulses].compact.flatten(1).each do |tuple|
+          write_tags |= collect_tags(tuple)
+        end
+
+        if takes.empty? and reads.empty?
+          tags = write_tags
+        else
+          d = write_tags - tags
+          unless d.empty?
+            raise TransactionSubspaceError,
+              "writes crossing subspaces: #{d} in #{transaction.inspect}"
           end
         end
 

data/lib/tupelo/client.rb

@@ -80,7 +80,13 @@ module Tupelo
 
     def subspace tag
       tag = tag.to_s
-      worker.subspaces.find {|sp| sp.tag == tag} ## should go thru worker queue
+      worker.subspaces.find {|sp| sp.tag == tag} or begin
+        if subscribed_tags.include? tag
+          read __tupelo__: "subspace", tag: tag, addr: nil, template: nil
+          worker.subspaces.find {|sp| sp.tag == tag}
+        end
+      end
+      ## this impl will not be safe with dynamic subspaces
     end
   end
 end

data/lib/tupelo/version.rb

@@ -1,3 +1,3 @@
 module Tupelo
-  VERSION = "0.13"
+  VERSION = "0.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tupelo
 version: !ruby/object:Gem::Version
-  version: '0.13'
+  version: '0.14'
 platform: ruby
 authors:
 - Joel VanderWerf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-04 00:00:00.000000000 Z
+date: 2013-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: atdo
@@ -84,119 +84,119 @@ files:
 - lib/tupelo/app/trace.rb
 - lib/tupelo/app/builder.rb
 - lib/tupelo/client.rb
-- lib/tupelo/app.rb
-- lib/tupelo/archiver.rb
-- lib/tupelo/client/worker.rb
-- lib/tupelo/client/common.rb
-- lib/tupelo/client/tuplespace.rb
-- lib/tupelo/client/transaction.rb
-- lib/tupelo/client/atdo.rb
-- lib/tupelo/client/reader.rb
+- lib/tupelo/tuplets/persistent-archiver.rb
 - lib/tupelo/tuplets/persistent-archiver/worker.rb
 - lib/tupelo/tuplets/persistent-archiver/tuplespace.rb
-- lib/tupelo/tuplets/persistent-archiver.rb
+- lib/tupelo/util/boolean.rb
+- lib/tupelo/app.rb
+- lib/tupelo/archiver.rb
 - lib/tupelo/archiver/persister.rb
 - lib/tupelo/archiver/worker.rb
 - lib/tupelo/archiver/tuplespace.rb
 - lib/tupelo/archiver/persistent-tuplespace.rb
-- lib/tupelo/util/boolean.rb
+- lib/tupelo/client/transaction.rb
+- lib/tupelo/client/worker.rb
+- lib/tupelo/client/reader.rb
+- lib/tupelo/client/tuplespace.rb
+- lib/tupelo/client/atdo.rb
+- lib/tupelo/client/common.rb
 - lib/tupelo/version.rb
 - bench/pipeline.rb
 - bugs/read-take.rb
 - bugs/take-write.rb
-- example/pubsub.rb
-- example/timeout-trans.rb
-- example/fish01.rb
-- example/tiny-client.rb
-- example/add.rb
-- example/parallel.rb
-- example/socket-broker.rb
-- example/multi-tier/memo2.rb
-- example/multi-tier/drb.rb
-- example/multi-tier/memo.rb
-- example/multi-tier/kvspace.rb
-- example/multi-tier/http.rb
-- example/multi-tier/multi-sinatras.rb
-- example/app-and-tup.rb
-- example/small.rb
-- example/bounded-retry.rb
-- example/fish.rb
-- example/zk/lock.rb
-- example/concurrent-transactions.rb
-- example/cancel.rb
-- example/map-reduce/map-reduce-v2.rb
-- example/map-reduce/remote-map-reduce.rb
-- example/map-reduce/map-reduce.rb
-- example/tiny-server.rb
-- example/write-wait.rb
-- example/tcp.rb
 - example/timeout.rb
-- example/read-in-trans.rb
-- example/subspaces/simple.rb
-- example/subspaces/pubsub.rb
-- example/subspaces/addr-book-v1.rb
-- example/subspaces/addr-book-v2.rb
-- example/subspaces/shop/shop-v2.rb
-- example/subspaces/shop/shop-v1.rb
-- example/subspaces/sorted-set-space.rb
-- example/balance-xfer-retry.rb
-- example/take-nowait-caution.rb
-- example/lock-mgr-with-queue.rb
-- example/hash-tuples.rb
-- example/pulse.rb
-- example/transaction-logic.rb
-- example/lease.rb
-- example/chat/chat.rb
-- example/chat/chat-nohistory.rb
-- example/balance-xfer.rb
 - example/add-dsl.rb
-- example/lock-mgr.rb
-- example/broker-locking.rb
-- example/dphil-optimistic.rb
+- example/remote.rb
+- example/increment.rb
+- example/matching.rb
+- example/dphil.rb
+- example/broker-optimistic.rb
 - example/fail-and-retry.rb
-- example/fish0.rb
+- example/load-balancer.rb
+- example/read-in-trans.rb
+- example/bounded-retry.rb
+- example/pregel/remote.rb
+- example/pregel/pagerank.rb
 - example/pregel/pregel.rb
 - example/pregel/distributed.rb
-- example/pregel/pagerank.rb
 - example/pregel/update.rb
-- example/pregel/remote.rb
-- example/pregel/dist-opt.rb
-- example/dphil-optimistic-v2.rb
-- example/broker-optimistic-v2.rb
-- example/remote.rb
 - example/take-nowait.rb
-- example/wait-interrupt.rb
-- example/optimist.rb
+- example/boolean-match.rb
+- example/lease.rb
+- example/broker-locking.rb
+- example/transaction-logic.rb
 - example/message-bus.rb
+- example/small-simplified.rb
+- example/small.rb
+- example/lock-mgr.rb
+- example/take-nowait-caution.rb
+- example/concurrent-transactions.rb
+- example/tcp.rb
+- example/notify.rb
+- example/pulse.rb
+- example/chat/chat.rb
+- example/chat/chat-nohistory.rb
+- example/hash-tuples.rb
 - example/balance-xfer-locking.rb
-- example/increment.rb
-- example/child-of-child.rb
-- example/custom-class.rb
-- example/matching.rb
+- example/balance-xfer-retry.rb
 - example/custom-search.rb
-- example/broker-optimistic.rb
-- example/notify.rb
-- example/small-simplified.rb
-- example/broker-queue.rb
-- example/async-transaction.rb
-- example/boolean-match.rb
-- example/load-balancer.rb
+- example/app-and-tup.rb
+- example/multi-tier/memo2.rb
+- example/multi-tier/http.rb
+- example/multi-tier/multi-sinatras.rb
+- example/multi-tier/kvspace.rb
+- example/multi-tier/memo.rb
+- example/multi-tier/drb.rb
 - example/take-many.rb
+- example/subspaces/ramp.rb
+- example/subspaces/sorted-set-space.rb
+- example/subspaces/addr-book.rb
+- example/subspaces/simple.rb
+- example/subspaces/shop/shop-v2.rb
+- example/subspaces/shop/shop-v1.rb
+- example/subspaces/pubsub.rb
+- example/subspaces/addr-book-v2.rb
+- example/dphil-optimistic.rb
+- example/async-transaction.rb
+- example/wait-interrupt.rb
+- example/fish0.rb
+- example/zk/lock.rb
 - example/deadlock.rb
-- example/dphil.rb
-- test/lib/testable-worker.rb
+- example/fish.rb
+- example/add.rb
+- example/dphil-optimistic-v2.rb
+- example/parallel.rb
+- example/tiny-client.rb
+- example/map-reduce/map-reduce.rb
+- example/map-reduce/remote-map-reduce.rb
+- example/map-reduce/map-reduce-v2.rb
+- example/map-reduce/prime-factor.rb
+- example/map-reduce/prime-factor-balanced.rb
+- example/lock-mgr-with-queue.rb
+- example/balance-xfer.rb
+- example/cancel.rb
+- example/socket-broker.rb
+- example/timeout-trans.rb
+- example/uniq-id.rb
+- example/optimist.rb
+- example/tiny-server.rb
+- example/pubsub.rb
+- example/broker-optimistic-v2.rb
+- example/write-wait.rb
+- example/custom-class.rb
+- test/stress/archiver-load.rb
+- test/stress/concurrent-transactions.rb
+- test/system/test-archiver.rb
+- test/lib/mock-client.rb
+- test/lib/time-fuzz.rb
+- test/lib/mock-queue.rb
 - test/lib/mock-seq.rb
+- test/lib/testable-worker.rb
 - test/lib/mock-msg.rb
-- test/lib/mock-queue.rb
-- test/lib/time-fuzz.rb
-- test/lib/mock-client.rb
-- test/system/test-archiver.rb
-- test/unit/test-ops.rb
-- test/unit/test-mock-client.rb
 - test/unit/test-mock-seq.rb
 - test/unit/test-mock-queue.rb
-- test/stress/concurrent-transactions.rb
-- test/stress/archiver-load.rb
+- test/unit/test-ops.rb
+- test/unit/test-mock-client.rb
 - bin/tup
 - bin/tspy
 homepage: https://github.com/vjoel/tupelo
@@ -231,8 +231,8 @@ signing_key:
 specification_version: 4
 summary: Distributed tuplespace
 test_files:
-- test/unit/test-ops.rb
-- test/unit/test-mock-client.rb
 - test/unit/test-mock-seq.rb
 - test/unit/test-mock-queue.rb
+- test/unit/test-ops.rb
+- test/unit/test-mock-client.rb
 has_rdoc: 

data/example/broker-queue.rb

@@ -1,35 +0,0 @@
-# more like how you would do it in redis, except that the queue is not stored in
-# the central server, so operations on it are not a bottleneck, FWIW
-
-require 'tupelo/app'
-
-N_PLAYERS = 10
-
-Tupelo.application do
-  N_PLAYERS.times do
-    # sleep rand / 10 # reduce contention -- could also randomize inserts
-    child do
-      me = client_id
-      write name: me
-      
-      you = transaction do
-        game = read_nowait(
-          player1: nil,
-          player2: me)
-        break game["player1"] if game
-      
-        unless take_nowait name: me
-          raise Tupelo::Client::TransactionFailure
-        end
-
-        you = take(name: nil)["name"]
-        write(
-          player1: me,
-          player2: you)
-        you
-      end
-
-      log "now playing with #{you}"
-    end
-  end
-end

data/example/child-of-child.rb

@@ -1,34 +0,0 @@
-require 'tupelo/app'
-
-### need a programmatic way to start up clients
-
-Tupelo.application do |app|
-
-  app.child do ## local still hangs
-    3.times do |i|
-      app.child do
-        write [i]
-        log "wrote #{i}"
-      end
-    end
-
-    3.times do
-      log take [nil]
-    end
-  end
-end
-
-__END__
-
-this hangs sometimes but not always:
-
-  tick    cid status operation
-A: client 3: wrote 0
-A: client 4: wrote 1
-     1      3        batch write [0]
-     2      4        batch write [1]
-A: client 2: [0]
-     3      2        atomic take [0]
-     4      2        atomic take [1]
-A: client 2: [1]
-A: client 5: wrote 2

data/example/fish01.rb

@@ -1,48 +0,0 @@
-# This works, but requires a fix-up step.
-
-require 'tupelo/app'
-
-Tupelo.application do
-  2.times do
-    child passive: true do
-      loop do
-        fish = nil
-
-        transaction do
-          fish, _ = take([String])
-          n, _ = take_nowait([Integer, fish])
-          if n
-            write [n + 1, fish]
-          else
-            write [1, fish] # another process might also write this, so ...
-          end
-        end
-        ### what if both processes die here?
-        transaction do # ... fix up the two tuples.
-          n1, _ = take_nowait [Integer, fish]; abort unless n1
-          n2, _ = take_nowait [Integer, fish]; abort unless n2
-          #log "fixing: #{[n1 + n2, fish]}"
-          write [n1 + n2, fish]
-        end
-      end
-    end
-  end
-
-  local do
-    seed = 3
-    srand seed
-    log "seed = #{seed}"
-    
-    fishes = %w{ trout marlin char salmon }
-
-    a = fishes * 10
-    a.shuffle!
-    a.each do |fish|
-      write [fish]
-    end
-
-    fishes.each do |fish|
-      log take [10, fish]
-    end
-  end
-end

data/example/pregel/dist-opt.rb

@@ -1,15 +0,0 @@
-#
-# Minor optimization:
-
-class KeyMatcher
-  def initialize i, n
-    @i = i
-    @n = n
-  end
-
-  def === id
-    id % @n == @i
-  end
-end
-
-vertex = take id: v_id_matcher, step: step, rank: nil, active: true

data/example/subspaces/addr-book-v1.rb

@@ -1,106 +0,0 @@
-## TODO
-##
-## scaling params
-
-require 'tupelo/app'
-
-ab_tag = "my address book"
-ab_sort_field = 1
-ab_val_field = 2
-cmd_tag = "#{ab_tag} commands"
-resp_tag = "#{ab_tag} responses"
-
-Tupelo.application do
-  local do
-    use_subspaces!
-
-    # Subspace for tuples belonging to the addr book.
-    define_subspace(
-      tag:          ab_tag,
-      template:     [
-        {value: ab_tag},
-        {type:  "string"},  # name <-- ab_sort_field references this field
-        nil                 # address; can be any object <-- ab_val_field
-      ]
-    )
-    
-    # Subspace for commands for fetch and delete.
-    # We can't use #read and #take because then the requesting client
-    # would have to subscribe to the ab_tag subspace.
-    define_subspace(
-      tag:          cmd_tag,
-      template:     [
-        {value: cmd_tag},
-        {type:  "string"},  # cmd name
-        {type:  "list"}     # arguments
-      ]
-    )
-
-    # Subspace for responses to commands. Identify the command this is in
-    # response to by copying it (alternately, could use ids).
-    define_subspace(
-      tag:          resp_tag,
-      template:     [
-        {value: resp_tag},
-        {type:  "string"},  # cmd name
-        {type:  "list"},    # arguments
-        nil                 # result of query -- type depends on command
-      ]
-    )
-  end
-
-  ## Could set N_SORTED_SET_SPACE > 1, but lookups are so fast it would
-  ## just lead to contention and redundant computation. Redundancy is useful
-  ## though.
-
-  # Inserts are just writes, which are handled by Worker and SortedSetSpace,
-  # so this child's app loop only needs to handle special commands: fetch and
-  # delete, which are delegated to the SortedSetSpace.
-  child tuplespace: [SortedSetSpace, ab_tag, ab_sort_field, ab_val_field],
-        subscribe: [ab_tag, cmd_tag], passive: true do
-    loop do
-      transaction do
-        _, cmd, args = take(subspace cmd_tag)
-
-        case cmd
-        when "delete"
-          args.each do |name|
-            take [ab_tag, name, nil]
-          end
-
-        when "fetch"
-          name = args[0]
-          _, _, addr = read [ab_tag, name, nil]
-          write [resp_tag, name, args, addr]
-
-        when "next", "prev"
-          name = args[0]
-          _, name2, addr = read SortedSetTemplate[ab_tag, cmd, name]
-          write [resp_tag, name, args, name2, addr]
-
-        when "first", "last"
-          _, name, addr = read SortedSetTemplate[ab_tag, cmd]
-          write [resp_tag, name, args, name, addr]
-
-        else # maybe write an error message in a tuple
-          log.error "bad command: #{cmd}"
-        end
-      end
-    end
-  end
-  
-  child subscribe: resp_tag do
-    # write some ab entries
-    write [ab_tag, "McFirst, Firsty", "123 W. Crescent Terrace"]
-    write [ab_tag, "Secondismus, Deuce", "456 S. West Way"]
-
-    # make some queries
-    write [cmd_tag, "first", []]
-    *, name, addr = take [resp_tag, "first", [], nil, nil]
-    log "first entry: #{name} => #{addr}"
-    
-    write [cmd_tag, "next", [name]]
-    *, name, addr = take [resp_tag, "next", [name], nil, nil]
-    log "next entry: #{name} => #{addr}"
-  end
-end