tupelo 0.16 → 0.17

data/lib/tupelo/util/bin-circle.rb

@@ -0,0 +1,123 @@
+require 'rbtree' # gem install rbtree
+require 'set'
+require 'digest/md5'
+
+# An implementation of consistent hashing
+# (https://en.wikipedia.org/wiki/Consistent_hashing) using a circular structure
+# of bins.
+#
+# Synopsis:
+#
+#    circle = BinCircle.new reps: 20
+#
+#    100.times do |id|
+#      circle.add_bin id
+#    end
+#
+#    circle.find_bin "foo"
+#
+# The #find_bin method determines which bin a given object is associated with.
+# For example: use it to look up which distributed cache the object is stored
+# in.
+#
+# BinCircle Does not store or cache any data. The state of a BinCircle instance
+# depends entirely on two things: the replications number (either passed to
+# BinCircle#new or set per bin in #add_bin), and the current set of bins, which
+# is managed with #add_bin and #delete_bin. Note: like rbtree, not thread-safe
+# (for #delete and #show_bins).
+class BinCircle
+  KEY_BITS = 30
+  KEY_MAX = 2**KEY_BITS - 1
+  DEFAULT_REPS = 10
+  
+  attr_accessor :default_reps
+
+  def initialize reps: DEFAULT_REPS
+    @tree = MultiRBTree.new # hashed_bin_id => bin_id
+    @default_reps = reps
+  end
+  
+  # +id+ should be an identifier for your bin, typically a number or string.
+  # Uses id.to_s, so +id+ should not contain a hash (which would not determine
+  # a unique string).
+  def add_bin id, reps: default_reps
+    rep_id = "#{id} 0000"
+    reps.times do |i|
+      key = key_for_string(rep_id)
+      @tree[key] = id
+      rep_id.succ!
+    end
+  end
+  
+  # Returns the set of bin ids now in the circle.
+  def bins
+    Set.new @tree.values
+  end
+  
+  def inspect
+    "#<#{self.class} bins=#{bins.to_a}>"
+  end
+
+  # This hashing fn is applied to both bin ids and object identifiers (in
+  # principle, these could be two different function with the same range of
+  # outputs.
+  def key_for_string str
+    Digest::MD5.hexdigest(str).to_i(16) & KEY_MAX
+  end
+  
+  # Delete the bin specified by +id+ (same as argument to #add_bin).
+  def delete_bin id
+    @tree.delete_if {|k,v| v == id}
+  end
+  
+  # +id+ should be an identifier for your object, typically a number or string.
+  # Uses id.to_s, so +id+ should not contain a hash (which would not determine
+  # a unique string). It is not necessary for +id+ to be unique across objects.
+  def find_bin obj
+    find_bin_by_key(key_for_string(obj.to_s))
+  end
+  
+  def find_bin_by_key key
+    _, id = @tree.lower_bound(key) || @tree.first
+    id
+  end
+
+  def bin_size
+    Hash.new(0).tap do |bs|
+      bs[@tree.first[1]] = 2**30 - @tree.last[0] + @tree.first[0]
+      @tree.each_cons(2) do |(prev, _), (key, id)|
+        bs[id] += key - prev
+      end
+    end
+  end
+
+  # To help in tuning the replication counts, you can look at the
+  # distribution of bin sizes, plus mean and variance.
+  def show_bins
+    bs = bin_size
+    p bs
+
+    mean = bs.inject(0.0) {|sum, (id,n)| sum + n} / N_BINS
+    variance = bs.inject(0.0) {|sum_sqerr, (id,n)|
+      sum_sqerr + (n - mean)**2} / (N_BINS-1)
+    printf "mean : %14d\n" % mean
+    printf "stdev: %14d\n" % Math.sqrt(variance)
+  end
+end
+
+if __FILE__ == $0
+  N_BINS = 100
+  N_REPS = 20
+
+  circle = BinCircle.new reps: N_REPS
+
+  N_BINS.times do |id|
+    circle.add_bin id
+  end
+
+  p circle
+  circle.show_bins
+
+  p circle.find_bin "foo"
+  p circle.find_bin "bar"
+end

data/lib/tupelo/version.rb

@@ -1,3 +1,3 @@
 module Tupelo
-  VERSION = "0.16"
+  VERSION = "0.17"
 end

metadata

@@ -1,72 +1,73 @@
 --- !ruby/object:Gem::Specification
 name: tupelo
 version: !ruby/object:Gem::Version
-  version: '0.16'
+  version: '0.17'
 platform: ruby
 authors:
 - Joel VanderWerf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-22 00:00:00.000000000 Z
+date: 2014-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: atdo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: easy-serve
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: funl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: object-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-description: Distributed tuplespace.
+description: A distributed, transactional tuplespace for polyglot computation and
+  storage.
 email: vjoel@users.sourceforge.net
 executables:
 - tup
@@ -76,130 +77,133 @@ extra_rdoc_files:
 - README.md
 - COPYING
 files:
-- README.md
 - COPYING
+- README.md
 - Rakefile
-- lib/tupelo/app/remote.rb
-- lib/tupelo/app/irb-shell.rb
-- lib/tupelo/app/trace.rb
-- lib/tupelo/app/builder.rb
-- lib/tupelo/client.rb
-- lib/tupelo/tuplets/persistent-archiver.rb
-- lib/tupelo/tuplets/persistent-archiver/worker.rb
-- lib/tupelo/tuplets/persistent-archiver/tuplespace.rb
-- lib/tupelo/util/boolean.rb
-- lib/tupelo/app.rb
-- lib/tupelo/archiver.rb
-- lib/tupelo/archiver/persister.rb
-- lib/tupelo/archiver/worker.rb
-- lib/tupelo/archiver/tuplespace.rb
-- lib/tupelo/archiver/persistent-tuplespace.rb
-- lib/tupelo/client/transaction.rb
-- lib/tupelo/client/worker.rb
-- lib/tupelo/client/reader.rb
-- lib/tupelo/client/tuplespace.rb
-- lib/tupelo/client/atdo.rb
-- lib/tupelo/client/common.rb
-- lib/tupelo/version.rb
 - bench/pipeline.rb
+- bin/tspy
+- bin/tup
 - bugs/read-take.rb
 - bugs/take-write.rb
-- example/timeout.rb
 - example/add-dsl.rb
-- example/remote.rb
-- example/increment.rb
-- example/matching.rb
-- example/dphil.rb
+- example/add.rb
+- example/app-and-tup.rb
+- example/async-transaction.rb
+- example/balance-xfer-locking.rb
+- example/balance-xfer-retry.rb
+- example/balance-xfer.rb
+- example/boolean-match.rb
+- example/bounded-retry.rb
+- example/broker-locking.rb
+- example/broker-optimistic-v2.rb
 - example/broker-optimistic.rb
+- example/cancel.rb
+- example/chat/chat-nohistory.rb
+- example/chat/chat.rb
+- example/concurrent-transactions.rb
+- example/consistent-hash.rb
+- example/custom-class.rb
+- example/custom-search.rb
+- example/deadlock.rb
+- example/dphil-optimistic-v2.rb
+- example/dphil-optimistic.rb
+- example/dphil.rb
 - example/fail-and-retry.rb
+- example/hash-tuples.rb
+- example/increment.rb
+- example/lease.rb
 - example/load-balancer.rb
-- example/read-in-trans.rb
-- example/bounded-retry.rb
-- example/pregel/remote.rb
+- example/lock-mgr-with-queue.rb
+- example/lock-mgr.rb
+- example/map-reduce/map-reduce-v2.rb
+- example/map-reduce/map-reduce.rb
+- example/map-reduce/prime-factor-balanced.rb
+- example/map-reduce/prime-factor.rb
+- example/map-reduce/remote-map-reduce.rb
+- example/matching.rb
+- example/message-bus.rb
+- example/multi-tier/drb.rb
+- example/multi-tier/http.rb
+- example/multi-tier/kvspace.rb
+- example/multi-tier/memo.rb
+- example/multi-tier/memo2.rb
+- example/multi-tier/multi-sinatras.rb
+- example/notify.rb
+- example/optimist.rb
+- example/parallel.rb
+- example/pregel/distributed.rb
 - example/pregel/pagerank.rb
 - example/pregel/pregel.rb
-- example/pregel/distributed.rb
+- example/pregel/remote.rb
 - example/pregel/update.rb
-- example/take-nowait.rb
-- example/boolean-match.rb
-- example/lease.rb
-- example/broker-locking.rb
-- example/transaction-logic.rb
-- example/message-bus.rb
+- example/pubsub.rb
+- example/pulse.rb
+- example/read-in-trans.rb
+- example/remote.rb
 - example/small-simplified.rb
 - example/small.rb
-- example/lock-mgr.rb
-- example/take-nowait-caution.rb
-- example/concurrent-transactions.rb
-- example/tcp.rb
-- example/notify.rb
-- example/pulse.rb
-- example/chat/chat.rb
-- example/chat/chat-nohistory.rb
-- example/hash-tuples.rb
-- example/balance-xfer-locking.rb
-- example/balance-xfer-retry.rb
-- example/custom-search.rb
-- example/app-and-tup.rb
-- example/multi-tier/memo2.rb
-- example/multi-tier/http.rb
-- example/multi-tier/multi-sinatras.rb
-- example/multi-tier/kvspace.rb
-- example/multi-tier/memo.rb
-- example/multi-tier/drb.rb
-- example/take-many.rb
-- example/subspaces/ramp.rb
-- example/subspaces/sorted-set-space.rb
+- example/socket-broker.rb
+- example/subspaces/addr-book-v2.rb
 - example/subspaces/addr-book.rb
-- example/subspaces/simple.rb
-- example/subspaces/shop/shop-v2.rb
-- example/subspaces/shop/shop-v1.rb
 - example/subspaces/pubsub.rb
-- example/subspaces/addr-book-v2.rb
-- example/dphil-optimistic.rb
-- example/async-transaction.rb
-- example/wait-interrupt.rb
-- example/fish0.rb
-- example/zk/lock.rb
-- example/deadlock.rb
-- example/fish.rb
-- example/add.rb
-- example/dphil-optimistic-v2.rb
-- example/tiny-service.rb
-- example/parallel.rb
-- example/tiny-client.rb
-- example/map-reduce/map-reduce.rb
-- example/map-reduce/remote-map-reduce.rb
-- example/map-reduce/map-reduce-v2.rb
-- example/map-reduce/prime-factor.rb
-- example/map-reduce/mr.rb
-- example/map-reduce/prime-factor-balanced.rb
-- example/lock-mgr-with-queue.rb
-- example/balance-xfer.rb
-- example/cancel.rb
-- example/socket-broker.rb
+- example/subspaces/ramp.rb
+- example/subspaces/shop/shop-v1.rb
+- example/subspaces/shop/shop-v2.rb
+- example/subspaces/simple.rb
+- example/subspaces/sorted-set-space.rb
+- example/take-many.rb
+- example/take-nowait-caution.rb
+- example/take-nowait.rb
+- example/tcp.rb
 - example/timeout-trans.rb
+- example/timeout.rb
+- example/tiny-client.rb
+- example/tiny-service.rb
+- example/transaction-logic.rb
 - example/uniq-id.rb
-- example/optimist.rb
-- example/pubsub.rb
-- example/broker-optimistic-v2.rb
+- example/wait-interrupt.rb
+- example/wip/fish.rb
+- example/wip/fish1.rb
+- example/wip/fish2.rb
 - example/write-wait.rb
-- example/custom-class.rb
-- test/stress/archiver-load.rb
-- test/stress/concurrent-transactions.rb
-- test/system/test-archiver.rb
+- example/zk/lock.rb
+- lib/tupelo/app.rb
+- lib/tupelo/app/builder.rb
+- lib/tupelo/app/irb-shell.rb
+- lib/tupelo/app/remote.rb
+- lib/tupelo/app/trace.rb
+- lib/tupelo/archiver.rb
+- lib/tupelo/archiver/persistent-tuplespace.rb
+- lib/tupelo/archiver/persister.rb
+- lib/tupelo/archiver/tuplespace.rb
+- lib/tupelo/archiver/worker.rb
+- lib/tupelo/client.rb
+- lib/tupelo/client/atdo.rb
+- lib/tupelo/client/common.rb
+- lib/tupelo/client/reader.rb
+- lib/tupelo/client/subspace.rb
+- lib/tupelo/client/transaction.rb
+- lib/tupelo/client/tuplespace.rb
+- lib/tupelo/client/worker.rb
+- lib/tupelo/tuplets/persistent-archiver.rb
+- lib/tupelo/tuplets/persistent-archiver/tuplespace.rb
+- lib/tupelo/tuplets/persistent-archiver/worker.rb
+- lib/tupelo/util/bin-circle.rb
+- lib/tupelo/util/boolean.rb
+- lib/tupelo/version.rb
 - test/lib/mock-client.rb
-- test/lib/time-fuzz.rb
+- test/lib/mock-msg.rb
 - test/lib/mock-queue.rb
 - test/lib/mock-seq.rb
 - test/lib/testable-worker.rb
-- test/lib/mock-msg.rb
-- test/unit/test-mock-seq.rb
+- test/lib/time-fuzz.rb
+- test/stress/archiver-load.rb
+- test/stress/concurrent-transactions.rb
+- test/system/test-archiver.rb
+- test/unit/test-mock-client.rb
 - test/unit/test-mock-queue.rb
+- test/unit/test-mock-seq.rb
 - test/unit/test-ops.rb
-- test/unit/test-mock-client.rb
-- bin/tup
-- bin/tspy
 homepage: https://github.com/vjoel/tupelo
 licenses:
 - BSD
@@ -227,7 +231,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: Distributed tuplespace

data/example/map-reduce/mr.rb

@@ -1,61 +0,0 @@
-# see also ../parallel.rb and ../remote.rb
-
-#require 'tupelo/app'
-#require 'tupelo/app/remote'
-require 'easy-serve/remote'
-
-tunnel = !!ARGV.delete("--tunnel")
-host = ARGV.shift
-
-#Tupelo.tcp_application do
-## This is fine:
-##  ez.remote host: host, tunnel: tunnel, eval: %{
-##  }
-#
-## These break:
-#
-#  ez.remote :seqd, :cseqd, host: host, tunnel: tunnel, log: true, eval: %{
-#    require 'funl/client'
-#
-#    log.progname = "client <starting>"
-#
-#    seqd, cseqd = conns
-#    client = Funl::Client.new(seq: seqd, cseq: cseqd, log: log)
-#    client.cseq_read_client_id
-#  }
-#
-##  remote host: host, tunnel: tunnel, eval: %{
-##  }
-#end
-
-EasyServe.start do |ez|
-  log = ez.log
-  log.level = Logger::INFO
-  log.progname = File.basename($0)
-
-  ez.start_servers do
-    svhost = tunnel ? "localhost" : nil # no need to expose port if tunnelled
-
-    ez.server :seqd, :tcp, svhost, 0 do |svr|
-      require 'funl/message-sequencer'
-      seq = Funl::MessageSequencer.new svr, log: log
-      seq.start
-    end
-
-    ez.server :cseqd, :tcp, svhost, 0 do |svr|
-      require 'funl/client-sequencer'
-      cseq = Funl::ClientSequencer.new svr, log: log
-      cseq.start
-    end
-  end
-
-  ez.remote :seqd, :cseqd, host: host, tunnel: tunnel, log: true, eval: %{
-    require 'funl/client'
-
-    log.progname = "client <starting>"
-
-    seqd, cseqd = conns
-    client = Funl::Client.new(seq: seqd, cseq: cseqd, log: log)
-    client.cseq_read_client_id
-  }
-end