tupelo 0.9 → 0.10

data/lib/tupelo/client/worker.rb

@@ -24,6 +24,7 @@ class Tupelo::Client
     attr_reader :prep_waiters
     attr_reader :trans_waiters
     attr_reader :notify_waiters
+    attr_reader :subspaces
 
     GET_TUPLESPACE = "get tuplespace"
 
@@ -36,6 +37,8 @@ class Tupelo::Client
         @atomic, @writes, @pulses, @takes, @reads =
           atomic, writes, pulses, takes, reads
       end
+      
+      NOOP = new([], [], [], [], [])
 
       def to_s
         ops = [ ["write", writes], ["pulse", pulses],
@@ -49,6 +52,22 @@ class Tupelo::Client
       end
       alias inspect to_s
     end
+    
+    class Subspace
+      attr_reader :tag
+      
+      def initialize metatuple, worker
+        @metatuple = metatuple
+        @tag = metatuple["tag"]
+
+        spec = Marshal.load(Marshal.dump(metatuple["template"]))
+        @pot = worker.pot_for(spec).optimize!
+      end
+      
+      def === tuple
+        @pot === tuple
+      end
+    end
 
     def initialize client
       @client = client
@@ -72,6 +91,7 @@ class Tupelo::Client
       @trans_waiters = []
       @notify_waiters = []
       @stopping = false
+      @subspaces = []
     end
 
     def log *args
@@ -202,9 +222,11 @@ class Tupelo::Client
       end
     end
 
-    def update_to_tick tick
-      # at this point we know that the seq messages now accumulating in
+    def update_to_tick tick: nil, tags: nil, all: false
+      # At this point we know that the seq messages now accumulating in
       # cmd_queue are tick+1, tick+2, ....
+      # (or a subset of this sequence if not subscribed_all).
+      # Some of these might get discarded later if archiver is more current.
       log.debug {"update_to_tick #{tick}"}
 
       unless arc
@@ -218,10 +240,20 @@ class Tupelo::Client
       end
 
       log.info "requesting tuplespace from arc"
-      arc << [GET_TUPLESPACE, nil, tick]
-        ## replace nil with tags, if any
+      subscription_delta = {
+        request_all: all,
+        request_tags: tags,
+        subscribed_all: client.subscribed_all,
+        subscribed_tags: client.subscribed_tags
+      }
+      arc << [GET_TUPLESPACE, subscription_delta, tick]
 
       begin
+        tuplespace.clear
+          ## in some cases, we can keep some of it, but the current
+          ## archiver is not smart enough to send exactly the delta
+          ### abort all current transactions???
+
         arc_tick = arc.read[0]
         log.info "arc says global_tick = #{arc_tick}"
 
@@ -232,6 +264,7 @@ class Tupelo::Client
             done = true
           else
             raise "bad object stream from archiver" if done
+            sniff_meta_tuple tuple
             tuplespace.insert tuple
             count += 1
           end
@@ -255,9 +288,18 @@ class Tupelo::Client
 
       if msg.control?
         client.handle_ack msg
-        op_type, *args = msg.control_op
-        if op_type == Funl::SUBSCRIBE_ALL
-          update_to_tick msg.global_tick
+        op_type, tags = msg.control_op
+        case op_type
+        when Funl::SUBSCRIBE_ALL
+          update_to_tick tick: msg.global_tick, all: true
+        when Funl::SUBSCRIBE
+          update_to_tick tick: msg.global_tick,
+            tags: (client.subscribed_tags + tags)
+        when Funl::UNSUBSCRIBE_ALL
+          update_to_tick tick: msg.global_tick, all: false
+        when Funl::UNSUBSCRIBE
+          update_to_tick tick: msg.global_tick,
+            tags: (client.subscribed_tags - tags)
         else
           raise "Unimplemented: #{msg.inspect}"
         end
@@ -278,7 +320,8 @@ class Tupelo::Client
       @delta = 0
 
       record_history msg
-      op = Operation.new(*blobber.load(msg.blob)) ## op.freeze_deeply
+      op = msg.blob ? Operation.new(*blobber.load(msg.blob)) : Operation::NOOP
+        ## op.freeze_deeply
       log.debug {"applying #{op} from client #{msg.client_id}"}
 
       notify_waiters.each do |waiter|
@@ -295,6 +338,20 @@ class Tupelo::Client
         log.debug {"inserting #{op.writes}; deleting #{actual_tuples}"}
         tuplespace.transaction inserts: op.writes, deletes: actual_tuples,
           tick: @global_tick
+      
+        op.writes.each do |tuple|
+          sniff_meta_tuple tuple
+        end
+
+        actual_tuples.each do |tuple|
+          ### abstract this out
+          if tuple.kind_of? Hash and tuple.key? "__tupelo__"
+            if tuple["__tupelo__"] == "subspace" # tuple is subspace metatdata
+              ## do some error checking
+              subspaces.delete_if {|sp| sp.tag == tuple["tag"]}
+            end
+          end
+        end
       end
 
       notify_waiters.each do |waiter|
@@ -368,6 +425,16 @@ class Tupelo::Client
       end
     end
 
+    def sniff_meta_tuple tuple
+      if tuple.kind_of? Hash and tuple.key? "__tupelo__"
+        if tuple["__tupelo__"] == "subspace" # tuple is subspace metatdata
+          ## do some error checking
+          ## what if subspace already exists?
+          subspaces << Subspace.new(tuple, self)
+        end
+      end
+    end
+
     def record_history msg; end
 
     def handle_client_request req
@@ -449,15 +516,42 @@ class Tupelo::Client
       msg.local_tick = local_tick + 1
       msg.global_tick = global_tick
       msg.delta = delta + 1 # pipelined write/take
-      ##msg.tags = nil
+      msg.tags = transaction.tags
+
+      writes = transaction.writes
+      pulses = transaction.pulses
+      takes = transaction.take_tuples_for_remote.compact
+      reads = transaction.read_tuples_for_remote.compact
+      
+      unless msg.tags
+        tags = []
+        tuples = [writes, pulses, takes, reads].compact.flatten(1)
+        subspaces.each do |subspace|
+          tuples.each do |tuple|
+            if subspace === tuple
+              tags << subspace.tag
+              break
+            end
+          end
+        end
+
+        will_get_this_msg = client.subscribed_all ||
+          tags.any? {|tag| client.subscribed_tags.include? tag} ## optimize
+
+        unless will_get_this_msg
+          tags << true # reflect
+        end
+
+        if not tags.empty?
+          msg.tags = tags
+          log.debug {"tagged transaction: #{tags}"}
+        end
+      end
 
       begin
         msg.blob = blobber.dump([
           transaction.atomic,
-          transaction.writes,
-          transaction.pulses,
-          transaction.take_tuples,
-          transaction.read_tuples
+          writes, pulses, takes, reads
         ])
         ## optimization: use bitfields to identify which ops are present
         ## (instead of nils), and combine this with atomic flag in one int

data/lib/tupelo/client.rb

@@ -10,10 +10,13 @@ module Tupelo
     attr_reader :worker
     attr_reader :tuplespace
 
-    def initialize(tuplespace: SimpleTuplespace, **opts)
+    TUPELO_SUBSPACE_TAG = "tupelo subspace"
+
+    def initialize(tuplespace: SimpleTuplespace, subscribe: :all, **opts)
       super **opts
       @tuplespace = tuplespace
       @worker = make_worker
+      @initial_subscriptions = subscribe
     end
 
     def make_worker
@@ -27,7 +30,13 @@ module Tupelo
     def start
       super
       worker.start
-      subscribe_all ## for now, but eventually should start without subs
+
+      case @initial_subscriptions
+      when :all
+        subscribe_all
+      when Array
+        subscribe @initial_subscriptions | [Tupelo::Client::TUPELO_SUBSPACE_TAG]
+      end
     end
 
     def stop
@@ -41,5 +50,30 @@ module Tupelo
         super().unknown *args
       end
     end
+    
+    ## do these belong in API module?
+    def define_subspace metatuple
+      defaults = {__tupelo__: "subspace", addr: nil}
+      write_wait defaults.merge!(metatuple)
+    end
+    
+    # call this just once at start of first client (it's optional to
+    # preserve behavior of non-subspace-aware code)
+    def use_subspaces!
+      define_subspace(
+        tag:          TUPELO_SUBSPACE_TAG,
+        template:     {
+          __tupelo__: {value: "subspace"},
+          tag:        nil,
+          addr:       nil,
+          template:   nil
+        }
+      )
+    end
+
+    def subspace tag
+      tag = tag.to_s
+      worker.subspaces.find {|sp| sp.tag == tag}
+    end
   end
 end

data/lib/tupelo/version.rb

@@ -1,3 +1,3 @@
 module Tupelo
-  VERSION = "0.9"
+  VERSION = "0.10"
 end

data/test/lib/mock-client.rb

@@ -105,4 +105,8 @@ class MockClient
     end
     val
   end
+  
+  def subscribed_all
+    true
+  end
 end

data/test/lib/testable-worker.rb

@@ -8,7 +8,7 @@ class TestableWorker < Tupelo::Client::Worker
     @cmd_queue = MockQueue.new
     
     observe_started_client
-    update_to_tick 0
+    update_to_tick tick: 0
   end
 
   def in_thread?

data/test/stress/concurrent-transactions.rb

@@ -6,35 +6,35 @@ N = 100
 client_class = Tupelo::TimeFuzz::Client
 Tupelo::TimeFuzz.sleep_max = 0.01
 
-Tupelo.application do |app|
-  app.child(client_class) do |client|
+Tupelo.application do
+  child(client_class) do
     N.times do
-      client.transaction do |t|
-        x, y = t.take [nil, nil]
-        t.write [x+1, y]
+      transaction do
+        x, y = take [nil, nil]
+        write [x+1, y]
       end
     end
-    client.write ["done"]
+    write ["done"]
   end
 
-  app.child(client_class) do |client|
+  child(client_class) do
     N.times do
-      client.transaction do |t|
-        x, y = t.take [nil, nil]
-        t.write [x, y+1]
+      transaction do
+        x, y = take [nil, nil]
+        write [x, y+1]
       end
     end
-    client.write ["done"]
+    write ["done"]
   end
 
-  app.local(client_class) do |client|
-    client.write [0, 0]
+  local(client_class) do
+    write [0, 0]
 
     2.times do
-      client.take ["done"]
+      take ["done"]
     end
 
-    x, y = client.read [nil, nil]
+    x, y = read [nil, nil]
     if x == N and y == N
       puts "OK"
     else

data/test/system/test-archiver.rb

@@ -1,24 +1,24 @@
 require 'tupelo/app'
 
-Tupelo.application do |app|
+Tupelo.application do
   expected = [[1], [2], [3]]
 
-  app.local do |client|
-    client.write *expected
+  local do
+    write *expected
   end
       
-  child_pid = app.child do |client|
+  child_pid = child do
     # Test that tuples written before this client started are readable.
-    a = client.read_all [Integer]
-    client.write_wait result: a
+    a = read_all [Integer]
+    write_wait result: a
   end
 
   # Normally we would wait using tuples, but in this case we want more
   # isolation in the test case, so we wait in terms of the PID.
   Process.waitpid child_pid
 
-  app.local do |client|
-    h = client.read_all result: Array
+  local do
+    h = read_all result: Array
     begin
       a = h.first["result"]
     rescue => ex

data/test/unit/test-ops.rb

@@ -164,6 +164,62 @@ class TestOps < Minitest::Test
     end
   end
   
+  def test_transaction_take_two
+    x = [0]; y = [1]
+    c = make_client(1)
+
+    w_op = c.now {write x, y}
+
+    c.will do
+      transaction do
+        [take([nil]), take([nil])]
+      end
+    end
+
+    assert_equal [x, y].sort, c.run.sort
+  end
+  
+  def test_transaction_take_read
+    x = [0]
+    c = make_client(1)
+
+    w_op = c.now {write x}
+
+    c.will do
+      transaction do
+        [take(x), read_nowait(x)]
+      end
+    end
+
+    assert_equal [x, nil], c.run
+  end
+  
+  def test_transaction_write_read
+    x = [0]
+    c = make_client(1)
+
+    c.will do
+      transaction do
+        write(x); read(x)
+      end
+    end
+
+    assert_equal x, c.run
+  end
+
+  def test_transaction_write_take
+    x = [0]
+    c = make_client(1)
+
+    c.will do
+      transaction do
+        write(x); take(x)
+      end
+    end
+
+    assert_equal x, c.run
+  end
+
   def test_transaction_empty
     transactor = make_client(0)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tupelo
 version: !ruby/object:Gem::Version
-  version: '0.9'
+  version: '0.10'
 platform: ruby
 authors:
 - Joel VanderWerf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-01 00:00:00.000000000 Z
+date: 2013-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: atdo
@@ -83,92 +83,96 @@ files:
 - lib/tupelo/app/irb-shell.rb
 - lib/tupelo/app/trace.rb
 - lib/tupelo/client.rb
-- lib/tupelo/app.rb
-- lib/tupelo/archiver.rb
-- lib/tupelo/client/worker.rb
-- lib/tupelo/client/common.rb
-- lib/tupelo/client/tuplespace.rb
-- lib/tupelo/client/transaction.rb
-- lib/tupelo/client/atdo.rb
-- lib/tupelo/client/reader.rb
+- lib/tupelo/tuplets/persistent-archiver.rb
 - lib/tupelo/tuplets/persistent-archiver/worker.rb
 - lib/tupelo/tuplets/persistent-archiver/tuplespace.rb
-- lib/tupelo/tuplets/persistent-archiver.rb
+- lib/tupelo/util/boolean.rb
+- lib/tupelo/app.rb
+- lib/tupelo/archiver.rb
 - lib/tupelo/archiver/persister.rb
 - lib/tupelo/archiver/worker.rb
 - lib/tupelo/archiver/tuplespace.rb
 - lib/tupelo/archiver/persistent-tuplespace.rb
-- lib/tupelo/util/boolean.rb
+- lib/tupelo/client/transaction.rb
+- lib/tupelo/client/worker.rb
+- lib/tupelo/client/reader.rb
+- lib/tupelo/client/tuplespace.rb
+- lib/tupelo/client/atdo.rb
+- lib/tupelo/client/common.rb
 - lib/tupelo/version.rb
 - bench/pipeline.rb
-- bugs/write-read.rb
+- bugs/read-take.rb
 - bugs/take-write.rb
-- example/pubsub.rb
-- example/timeout-trans.rb
-- example/tiny-client.rb
-- example/add.rb
-- example/parallel.rb
-- example/app-and-tup.rb
-- example/small.rb
-- example/bounded-retry.rb
-- example/map-reduce-v2.rb
-- example/zk/lock.rb
-- example/concurrent-transactions.rb
-- example/cancel.rb
-- example/tiny-server.rb
-- example/write-wait.rb
-- example/tcp.rb
 - example/timeout.rb
+- example/add-dsl.rb
+- example/remote.rb
+- example/increment.rb
+- example/matching.rb
+- example/dphil.rb
+- example/broker-optimistic.rb
+- example/fail-and-retry.rb
+- example/load-balancer.rb
 - example/read-in-trans.rb
-- example/balance-xfer-retry.rb
-- example/lock-mgr-with-queue.rb
-- example/hash-tuples.rb
-- example/pulse.rb
-- example/transaction-logic.rb
-- example/remote-map-reduce.rb
 - example/map-reduce.rb
+- example/bounded-retry.rb
+- example/pregel/remote.rb
+- example/pregel/pagerank.rb
+- example/pregel/pregel.rb
+- example/pregel/distributed.rb
+- example/take-nowait.rb
+- example/boolean-match.rb
 - example/lease.rb
-- example/balance-xfer.rb
-- example/add-dsl.rb
-- example/lock-mgr.rb
+- example/remote-map-reduce.rb
 - example/broker-locking.rb
-- example/dphil-optimistic.rb
-- example/fail-and-retry.rb
-- example/dphil-optimistic-v2.rb
-- example/broker-optimistic-v2.rb
-- example/remote.rb
-- example/take-nowait.rb
-- example/optimist.rb
+- example/transaction-logic.rb
 - example/message-bus.rb
+- example/small-simplified.rb
+- example/small.rb
+- example/lock-mgr.rb
+- example/concurrent-transactions.rb
+- example/tcp.rb
+- example/notify.rb
+- example/pulse.rb
+- example/hash-tuples.rb
 - example/balance-xfer-locking.rb
-- example/increment.rb
-- example/child-of-child.rb
-- example/custom-class.rb
-- example/matching.rb
+- example/balance-xfer-retry.rb
 - example/custom-search.rb
-- example/broker-optimistic.rb
-- example/notify.rb
-- example/small-simplified.rb
-- example/broker-queue.rb
-- example/async-transaction.rb
-- example/boolean-match.rb
-- example/load-balancer.rb
+- example/app-and-tup.rb
 - example/take-many.rb
+- example/dphil-optimistic.rb
+- example/async-transaction.rb
+- example/wait-interrupt.rb
+- example/zk/lock.rb
+- example/subspace.rb
+- example/map-reduce-v2.rb
 - example/deadlock.rb
-- example/dphil.rb
-- test/lib/testable-worker.rb
+- example/add.rb
+- example/dphil-optimistic-v2.rb
+- example/parallel.rb
+- example/tiny-client.rb
+- example/lock-mgr-with-queue.rb
+- example/balance-xfer.rb
+- example/cancel.rb
+- example/timeout-trans.rb
+- example/optimist.rb
+- example/tiny-server.rb
+- example/pubsub.rb
+- example/broker-optimistic-v2.rb
+- example/write-wait.rb
+- example/custom-class.rb
+- test/stress/archiver-load.rb
+- test/stress/concurrent-transactions.rb
+- test/system/test-archiver.rb
+- test/lib/mock-client.rb
+- test/lib/time-fuzz.rb
+- test/lib/mock-queue.rb
 - test/lib/mock-seq.rb
+- test/lib/testable-worker.rb
 - test/lib/mock-msg.rb
-- test/lib/mock-queue.rb
-- test/lib/time-fuzz.rb
-- test/lib/mock-client.rb
-- test/system/test-archiver.rb
-- test/unit/test-ops.rb
-- test/unit/test-mock-client.rb
 - test/unit/test-mock-seq.rb
 - test/unit/test-mock-queue.rb
-- test/stress/concurrent-transactions.rb
-- test/stress/archiver-load.rb
+- test/unit/test-ops.rb
+- test/unit/test-mock-client.rb
 - bin/tup
 - bin/tspy
 homepage: https://github.com/vjoel/tupelo
@@ -198,13 +202,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.4
+rubygems_version: 2.1.10
 signing_key: 
 specification_version: 4
 summary: Distributed tuplespace
 test_files:
-- test/unit/test-ops.rb
-- test/unit/test-mock-client.rb
 - test/unit/test-mock-seq.rb
 - test/unit/test-mock-queue.rb
+- test/unit/test-ops.rb
+- test/unit/test-mock-client.rb
 has_rdoc: 

data/bugs/write-read.rb

@@ -1,15 +0,0 @@
-require 'tupelo/app'
-
-Tupelo.application do |app|
-  app.local do |client|
-    begin
-      val = client.transaction timeout: 0.1 do |t|
-        t.write [1]
-        t.read [1] # similarly for take
-      end
-      p val # should be [1]
-    rescue TimeoutError => ex
-      puts ex
-    end
-  end
-end

data/example/broker-queue.rb

@@ -1,35 +0,0 @@
-# more like how you would do it in redis, except that the queue is not stored in
-# the central server, so operations on it are not a bottleneck, FWIW
-
-require 'tupelo/app'
-
-N_PLAYERS = 10
-
-Tupelo.application do
-  N_PLAYERS.times do
-    # sleep rand / 10 # reduce contention -- could also randomize inserts
-    child do
-      me = client_id
-      write name: me
-      
-      you = transaction do
-        game = read_nowait(
-          player1: nil,
-          player2: me)
-        break game["player1"] if game
-      
-        unless take_nowait name: me
-          raise Tupelo::Client::TransactionFailure
-        end
-
-        you = take(name: nil)["name"]
-        write(
-          player1: me,
-          player2: you)
-        you
-      end
-
-      log "now playing with #{you}"
-    end
-  end
-end