tupelo 0.19 → 0.20

data/example/sqlite/poi.rb

@@ -0,0 +1,40 @@
+# POI -- Points Of Interest
+#
+# This example creates a sqlite db in memory with a table of locations and
+# descriptions of points of interest, and attaches the db to a subspace of the
+# tuplespace. The process which manages that subspace can now do two things:
+#
+# 1. accept inserts (via write)
+#
+# 2. custom queries, accessed by write to a different subspace
+#
+# You can have redundant instances of this, and that will distribute load
+# in #2 above.
+#
+# gem install sequel sqlite3
+
+require 'tupelo/app'
+require_relative 'poi-store'
+
+Tupelo.application do
+  local do
+    POISPACE = PoiStore.define_poispace(self)
+  end
+
+  child tuplespace: [PoiStore, POISPACE], subscribe: "poi", passive: true do
+    log.progname = "poi-store"
+    # handle custom queries here, using poi template
+    read do
+      log read_all # just show everything for each new tuple
+    end
+  end
+  
+  child subscribe: nil do
+    write_wait lat: 12, lng: 34, desc: "foo"
+    sleep 0.5 # give poi store time to store and log
+    write_wait lat: 56, lng: 78, desc: "bar"
+    sleep 0.5 # give poi store time to store and log
+    write_wait lat: 12, lng: 34, desc: "foo" # dup is ok
+    sleep 0.5 # give poi store time to store and log
+  end
+end

data/example/sqlite/tmp/poi-sqlite.rb

@@ -0,0 +1,33 @@
+require 'sequel'
+
+db = Sequel.sqlite
+db.create_table "poi" do
+  primary_key   :id
+  float         :lat, null: false
+  float         :lng, null: false
+  text          :desc
+end
+
+#p db
+#p db.schema :poi
+
+poi = db[:poi]
+
+#10.times do |i|
+#  poi << {
+#    lat:  rand,
+#    lng:  rand,
+#    desc: "point #{i}"
+#  }
+#end
+
+#p poi.all
+#exit
+
+poi << {lat: 1, lng: 2, desc: "a"}
+poi << {lat: 1, lng: 2, desc: "b"}
+
+#p poi.count
+
+p poi.select(:lat, :lng, :desc).
+  where(lat: 0..1, lng: 0..1).limit(5).all

data/example/subspaces/addr-book-v1.rb

@@ -12,8 +12,6 @@ resp_tag = "#{ab_tag} responses"
 
 Tupelo.application do
   local do
-    use_subspaces!
-
     # Subspace for tuples belonging to the addr book.
     define_subspace(
       tag:          ab_tag,

data/example/subspaces/addr-book-v2.rb

@@ -2,8 +2,6 @@ require 'tupelo/app'
 
 Tupelo.application do
   local do
-    use_subspaces!
-
     define_subspace(
       tag:          "address books",
       template:     {

data/example/subspaces/addr-book.rb

@@ -32,8 +32,6 @@ resp_tag = "#{ab_tag} responses"
 
 Tupelo.application do
   local do
-    use_subspaces!
-
     # Subspace for tuples belonging to the addr book.
     define_subspace(ab_tag, [
       ab_tag,

data/example/subspaces/pubsub.rb

@@ -16,8 +16,6 @@ N_CHAN = 3
 
 Tupelo.application do
   local do
-    use_subspaces!
-
     N_CHAN.times do |i|
       define_subspace i, [i, String]
     end

data/example/subspaces/ramp.rb

@@ -43,8 +43,6 @@ end
 Tupelo.application do
 
   local do
-    use_subspaces!
-    
     bool = PortableObjectTemplate::BOOLEAN
     
     define_subspace("x", {

data/example/subspaces/shop/shop-v2.rb

@@ -10,8 +10,6 @@ CUSTOMER_IDS = 1..10
 
 Tupelo.application do
   local do
-    use_subspaces!
-
     define_subspace("inventory", [
       "product",
       nil,       # product_id

data/example/subspaces/simple.rb

@@ -8,7 +8,6 @@ Tupelo.application do
     log.progname = "before"
     log [subscribed_all, subscribed_tags]
 
-    use_subspaces!
     define_subspace "foo", [Numeric]
 
     log read_all(Object)

data/example/subspaces/sorted-set-space.rb

@@ -4,7 +4,12 @@ require 'rbtree'
 ##
 ## generalize SortedSetSpace to accept params that indicate which fields
 ##   are key and value
+##
+## unify with space used in ../riemann v2, generalize
 
+# This is a template class, but it doesn't just match tuples. It can
+# be used to find the *next* tuple after a given one, using the rbtree
+# ordering.
 class SortedSetTemplate
   class << self
     alias [] new

data/lib/tupelo/app.rb

@@ -127,6 +127,14 @@ module Tupelo
         app.start_trace
       end
 
+      if owns_services
+        ## optimize this away -- need lightweight client
+        ## or (better) make subspaces a default even without using app
+        app.local subscribe: nil do
+          use_subspaces!
+        end
+      end
+
       if block
         if block.arity == 0
           app.instance_eval &block

data/lib/tupelo/archiver/persistent-tuplespace.rb

@@ -135,8 +135,8 @@ class Tupelo::Archiver
       end
     end
 
-    def find_match_for tuple
-      @tuple_rec[tuple].count > 0 && tuple
+    def find_match_for tuple, distinct_from: []
+      @tuple_rec[tuple].count > distinct_from.count(tuple)
     end
   end
 end

data/lib/tupelo/archiver/tuplespace.rb

@@ -69,8 +69,8 @@ class Tupelo::Archiver
       end
     end
 
-    def find_match_for tuple
-      @counts[tuple] > 0 && tuple
+    def find_match_for tuple, distinct_from: []
+      @counts[tuple] > distinct_from.count(tuple)
     end
   end
 end

data/lib/tupelo/client/reader.rb

@@ -3,14 +3,18 @@ require 'tupelo/client/common'
 class Tupelo::Client
   # Include into class that defines #worker and #log.
   module Api
+    NOT_META = proc {|t| not defined? t.key? or not t.key? TUPELO_META_KEY}
+
     # If no block given, return one matching tuple, blocking if necessary.
     # If block given, yield each matching tuple that is found
     # locally and then yield each new match as it is written to the space.
     # Guaranteed not to miss tuples, even if they arrive and are immediately
     # taken. (Note that simply doing read(template) in a loop would not
     # have this guarantee.)
-    # The template defaults to Object, which matches any tuple.
-    def read_wait template = Object
+    # The template defaults to NOT_META, which matches any tuple except metas.
+    # The first phase of this method, reading existing tuples, is essentially
+    # the same as read_all, and subject to the same warnings.
+    def read_wait template = NOT_META
       waiter = Waiter.new(worker.make_template(template), self, !block_given?)
       worker << waiter
       if block_given?
@@ -27,17 +31,23 @@ class Tupelo::Client
     end
     alias read read_wait
 
-    # The template defaults to Object, which matches any tuple.
-    def read_nowait template = Object
+    # The template defaults to NOT_META, which matches any tuple except metas.
+    def read_nowait template = NOT_META
       matcher = Matcher.new(worker.make_template(template), self)
       worker << matcher
       matcher.wait
     end
 
-    # Returns all matching tuples currently in the space. The template defaults
-    # to Object, which matches any tuple. Does not wait for more tuples to
-    # arrive.
-    def read_all template = Object
+    # Returns all matching tuples currently in the space. Does not wait for more
+    # tuples to arrive. The template defaults to NOT_META, which matches any
+    # tuple except metas. To read all matches of more than one template, use
+    # the #or method from util/boolean.rb.
+    # Matches are guaranteed to exist at the same tick (even if they no longer
+    # exist after that tick). To take a snapshot like this, the worker is
+    # blocked from all other activity for some time, so be careful about
+    # using read_all when large numbers of tuples match. If a block is given,
+    # it runs after the worker has unblocked.
+    def read_all template = NOT_META
       matcher = Matcher.new(worker.make_template(template), self, :all => true)
       worker << matcher
       a = []

data/lib/tupelo/client/subspace.rb

@@ -1,5 +1,8 @@
 class Tupelo::Client
   module Api
+    TUPELO_SUBSPACE_TAG = "tupelo subspace".freeze
+    TUPELO_META_KEY = "__tupelo__".freeze
+
     def define_subspace tag, template, addr: nil
       metatuple = {
         TUPELO_META_KEY => "subspace",
@@ -11,9 +14,10 @@ class Tupelo::Client
     end
 
     # call this just once at start of first client (it's optional to
-    # preserve behavior of non-subspace-aware code)
+    # preserve behavior of non-subspace-aware code); this is done automatically
+    # in the app framework
     def use_subspaces!
-      return if subspace(TUPELO_SUBSPACE_TAG)
+      return if find_subspace_by_tag(TUPELO_SUBSPACE_TAG)
       define_subspace(TUPELO_SUBSPACE_TAG, {
         TUPELO_META_KEY => "subspace",
         tag:        nil,
@@ -24,16 +28,20 @@ class Tupelo::Client
 
     def subspace tag
       tag = tag.to_s
-      worker.subspaces.find {|sp| sp.tag == tag} or begin
+      find_subspace_by_tag(tag) or begin
         if subscribed_tags.include? tag
           read TUPELO_META_KEY => "subspace",
             tag:      tag,
             template: nil,
             addr:     nil
-          worker.subspaces.find {|sp| sp.tag == tag}
+          find_subspace_by_tag tag
         end
       end
       ## this impl will not be safe with dynamic subspaces
     end
+
+    def find_subspace_by_tag tag
+      worker.find_subspace_by_tag tag
+    end
   end
 end

data/lib/tupelo/client/transaction.rb

@@ -175,6 +175,10 @@ class Tupelo::Client
       end
     end
 
+    def read_only?
+      @writes.empty? && @pulses.empty? && @take_templates.empty?
+    end
+
     def inspect
       stat_extra =
         case
@@ -266,6 +270,10 @@ class Tupelo::Client
     
     # transaction applies only if template has a match
     def read template_spec
+      if block_given?
+        raise ArgumentError,
+          "Transaction#read with block (streaming read) not allowed"
+      end
       check_open
       template = worker.make_template(template_spec)
       @read_templates << template
@@ -476,6 +484,9 @@ class Tupelo::Client
       
       ## convert cancelling write/take to pulse
       ## convert cancelling take/write to read
+      ## remove redundant pulse after read
+      ## remove redundant read before take
+      ## remove redundant read after write
       
       if take_tuples_for_local.all? and read_tuples_for_local.all?
         @queue << true
@@ -520,7 +531,8 @@ class Tupelo::Client
     end
     
     def done global_tick, granted_tuples
-      raise TransactionStateError, "must be pending" unless pending?
+      raise TransactionStateError, "must be pending or read_only" unless
+        pending? or (closed? and read_only?)
       raise unless in_worker_thread?
       raise if @global_tick or @exception
 

data/lib/tupelo/client/worker.rb

@@ -320,6 +320,10 @@ class Tupelo::Client
       @delta = 0
 
       record_history msg
+      execute_transaction msg
+    end
+
+    def execute_transaction msg
       op = msg.blob ? Operation.new(*blobber.load(msg.blob)) : Operation::NOOP
         ## op.freeze_deeply
       log.debug {"applying #{op} from client #{msg.client_id}"}
@@ -329,7 +333,8 @@ class Tupelo::Client
       end
 
       take_tuples = tuplespace.find_distinct_matches_for(op.takes)
-      read_tuples = op.reads.map {|t| tuplespace.find_match_for(t)}
+      read_tuples = op.reads.map {|t| tuplespace.find_match_for(t,
+        distinct_from: take_tuples)}
       succeeded = take_tuples.all? && read_tuples.all?
 
       if client.subscribed_all
@@ -433,10 +438,23 @@ class Tupelo::Client
       end
     end
 
+    def find_subspace_by_tag tag
+      subspaces.find {|sp| sp.tag == tag}
+    end
+
+    def meta_subspace
+      @meta_subspace ||= find_subspace_by_tag(Api::TUPELO_SUBSPACE_TAG)
+    end
+
     # Returns true if tuple is subspace metadata.
     def is_meta_tuple? tuple
-      tuple.kind_of? Hash and tuple.key? TUPELO_META_KEY and
-        tuple[TUPELO_META_KEY] == "subspace"
+      if meta_subspace
+        meta_subspace === tuple
+      else
+        # meta_subspace hasn't arrived yet, so use approximation
+        tuple.kind_of? Hash and tuple.key? Api::TUPELO_META_KEY and
+          tuple[Api::TUPELO_META_KEY] == "subspace"
+      end
     end
 
     def sniff_meta_tuple tuple
@@ -489,7 +507,11 @@ class Tupelo::Client
         t.prepare
         prep_waiters << t unless prep_waiters.include? t
       when t.closed?
-        t.submit
+        if t.read_only?
+          t.done global_tick, nil
+        else
+          t.submit
+        end
         prep_waiters.delete t
       when t.failed?
       else
@@ -600,6 +622,7 @@ class Tupelo::Client
         msg.blob = blobber.dump([writes, pulses, takes, reads])
         ## optimization: use bitfields to identify which ops are present
         ## (instead of nils), in one int
+        ## other optimization: remove trailing nil/[]
       rescue => ex
         raise ex, "cannot serialize #{transaction.inspect}: #{ex}"
       end

data/lib/tupelo/client.rb

@@ -2,18 +2,16 @@ require 'funl/client'
 
 module Tupelo
   class Client < Funl::Client
-    require 'tupelo/client/worker'
-    require 'tupelo/client/tuplespace'
     require 'tupelo/client/subspace'
 
     include Api
 
+    require 'tupelo/client/worker'
+    require 'tupelo/client/tuplespace'
+
     attr_reader :worker
     attr_reader :tuplespace
 
-    TUPELO_SUBSPACE_TAG = "tupelo subspace".freeze
-    TUPELO_META_KEY = "__tupelo__".freeze
-
     def initialize(tuplespace: SimpleTuplespace, subscribe: :all, **opts)
       super **opts
       @tuplespace = tuplespace

data/lib/tupelo/tuplets/persistent-archiver/tuplespace.rb

@@ -82,5 +82,10 @@ class Tupelo::PersistentArchiver
     def find_match_for tuple
       @counts[tuple] > 0 && tuple
     end
+
+###    def find_match_for tuple, distinct_from: []
+###      tuple && @tuple_rec[tuple].count > distinct_from.count(tuple)
+###      ## is 'tuple &&' necessary?
+###    end
   end
 end

data/lib/tupelo/version.rb

@@ -1,3 +1,3 @@
 module Tupelo
-  VERSION = "0.19"
+  VERSION = "0.20"
 end

data/test/lib/mock-client.rb

@@ -2,6 +2,7 @@ require 'fiber'
 
 require 'tupelo/client/reader'
 require 'tupelo/client/transaction'
+require 'tupelo/client/subspace'
 
 require 'mock-queue.rb'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tupelo
 version: !ruby/object:Gem::Version
-  version: '0.19'
+  version: '0.20'
 platform: ruby
 authors:
 - Joel VanderWerf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-26 00:00:00.000000000 Z
+date: 2014-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: atdo
@@ -95,19 +95,27 @@ files:
 - example/balance-xfer.rb
 - example/barrier.rb
 - example/bingo/bingo-v1.rb
+- example/bingo/bingo-v2.rb
 - example/boolean-match.rb
 - example/bounded-retry.rb
 - example/broker-locking.rb
 - example/broker-optimistic-v2.rb
 - example/broker-optimistic.rb
+- example/broker-queue.rb
 - example/cancel.rb
 - example/chat/chat-nohistory.rb
 - example/chat/chat.rb
+- example/child-of-child.rb
 - example/concurrent-transactions.rb
 - example/consistent-hash.rb
+- example/counters/lock.rb
+- example/counters/merge.rb
+- example/counters/optimistic.rb
 - example/custom-class.rb
 - example/custom-search.rb
+- example/dataflow.rb
 - example/deadlock.rb
+- example/dedup.rb
 - example/dphil-optimistic-v2.rb
 - example/dphil-optimistic.rb
 - example/dphil.rb
@@ -118,6 +126,7 @@ files:
 - example/load-balancer.rb
 - example/lock-mgr-with-queue.rb
 - example/lock-mgr.rb
+- example/map-reduce/ex.rb
 - example/map-reduce/map-reduce-v2.rb
 - example/map-reduce/map-reduce.rb
 - example/map-reduce/prime-factor-balanced.rb
@@ -136,6 +145,7 @@ files:
 - example/observer.rb
 - example/optimist.rb
 - example/parallel.rb
+- example/pregel/dist-opt.rb
 - example/pregel/distributed.rb
 - example/pregel/pagerank.rb
 - example/pregel/pregel.rb
@@ -146,13 +156,22 @@ files:
 - example/read-in-trans.rb
 - example/remote.rb
 - example/riemann/event-subspace.rb
-- example/riemann/expirer-v1.rb
+- example/riemann/expiration-dbg.rb
 - example/riemann/producer.rb
-- example/riemann/riemann-v1.rb
-- example/riemann/riemann-v2.rb
+- example/riemann/v1/expirer.rb
+- example/riemann/v1/riemann.rb
+- example/riemann/v2/expirer.rb
+- example/riemann/v2/hash-store.rb
+- example/riemann/v2/http-mode.rb
+- example/riemann/v2/ordered-event-store.rb
+- example/riemann/v2/riemann.rb
 - example/small-simplified.rb
 - example/small.rb
 - example/socket-broker.rb
+- example/sqlite/poi-store.rb
+- example/sqlite/poi-v2.rb
+- example/sqlite/poi.rb
+- example/sqlite/tmp/poi-sqlite.rb
 - example/subspaces/addr-book-v1.rb
 - example/subspaces/addr-book-v2.rb
 - example/subspaces/addr-book.rb
@@ -248,8 +267,8 @@ signing_key:
 specification_version: 4
 summary: Distributed tuplespace
 test_files:
-- test/unit/test-mock-seq.rb
-- test/unit/test-mock-queue.rb
 - test/unit/test-ops.rb
 - test/unit/test-mock-client.rb
+- test/unit/test-mock-seq.rb
+- test/unit/test-mock-queue.rb
 has_rdoc: 

data/example/riemann/expirer-v1.rb

@@ -1,25 +0,0 @@
-class Tupelo::Client
-  # Expire old events.
-  def run_expirer_v1
-    scheduler = make_scheduler
-    
-    read subspace("event") do |event|
-      next if event["state"] == "expired"
-
-      event_exp = event["time"] + event["ttl"]
-      scheduler.at event_exp do
-        transaction do
-          take event
-          pulse event.merge("state" => "expired")
-            # Not sure if this is riemann semantics. Using #pulse rather
-            # that #write means that the expired event exists in the
-            # space only while the transaction is executing, but that is
-            # enough to trigger any client that is waiting on a template
-            # that matches the event. Use the --debug-expiration switch
-            # to see this happening (and use -v to make log messages
-            # verbose, showing timestamps).
-        end
-      end
-    end
-  end
-end