tupelo 0.4 → 0.5

data/example/map-reduce.rb

@@ -1,23 +1,8 @@
 require 'tupelo/app'
 
 N = 2 # how many cpus do you want to use for mappers?
-VERBOSE = ARGV.delete "-v"
 
 Tupelo.application do |app|
-  if VERBOSE
-    app.child do |client| # a debugger client, to see what's happening
-      note = client.notifier
-      puts "%4s %4s %10s %s" % %w{ tick cid status operation }
-      loop do
-        status, tick, cid, op = note.wait
-        unless status == :attempt
-          s = status == :failure ? "FAILED" : ""
-          puts "%4d %4d %10s %p" % [tick, cid, s, op]
-        end
-      end
-    end
-  end
-
   app.child do |client|
     document = "I will not map reduce in class\n" * 10
     lineno = 0
@@ -51,12 +36,11 @@ Tupelo.application do |app|
       end
     end
 
-    client.log "DONE. results = #{results}"
-    client.log "Press ^C to exit"
+    client.log "results = #{results}"
   end
   
   N.times do |i|
-    app.child do |client|
+    app.child passive: true do |client|
       client.log.progname = "mapper #{i}"
       
       loop do

data/example/small-simplified.rb

@@ -2,17 +2,17 @@
 
 require 'tupelo/app'
 
-Tupelo.application do |app|
-  app.child do |client|
-    client.write [2, 3, "frogs"]
-    _, s = client.take ["animals", nil]
+Tupelo.application do
+  child do
+    write [2, 3, "frogs"]
+    _, s = take ["animals", nil]
     puts s
   end
 
-  app.child do |client|
-    x, y, s = client.take [Numeric, Numeric, String]
+  child do
+    x, y, s = take [Numeric, Numeric, String]
     s2 = ([s] * (x + y)).join(" ")
-    client.write ["animals", s2]
+    write ["animals", s2]
   end
 end
 

data/example/small.rb

@@ -39,8 +39,8 @@ EasyServe.start(servers_file: "small-servers.yaml") do |ez|
 
     ez.server :arcd do |svr|
       require 'tupelo/archiver'
-      arc = Archiver.new svr, seq: arc_to_seq_sock, cseq: arc_to_cseq_sock,
-              log: log
+      arc = Tupelo::Archiver.new svr,
+        seq: arc_to_seq_sock, cseq: arc_to_cseq_sock, log: log
       arc.start
     end
   end
@@ -49,7 +49,7 @@ EasyServe.start(servers_file: "small-servers.yaml") do |ez|
     log = opts[:log]
     log.progname = "client <starting in #{log.progname}>"
     require 'tupelo/client'
-    client = Client.new opts
+    client = Tupelo::Client.new opts
     client.start do
       log.progname = "client #{client.client_id}"
     end

data/example/tcp.rb

@@ -5,6 +5,10 @@
 #
 #  ruby tcp.rb --info
 #
+# or
+#
+#  ruby tcp.rb --trace
+#
 # You can test with a local client:
 #
 #   ../bin/tup tcp.yaml
@@ -27,7 +31,10 @@ Tupelo.application servers_file: svr,
             arcd_addr:  [:tcp, '0.0.0.0', port + 2] do |app|
   if app.owns_servers
     puts "server started; ^C to stop"
-    puts "(run with --info to see events)" if app.log.level > Logger::INFO
+    puts "run in another terminal: ../bin/tup tcp.yaml"
+    if app.log.level > Logger::INFO
+      puts "(run with --info or --trace to see events)"
+    end
     sleep
   else
     abort "server seems to be running already; check file #{svr.inspect}"

data/lib/tupelo/app/trace.rb

@@ -0,0 +1,23 @@
+require 'tupelo/app'
+
+# displays every transaction in sequence, specially marking failed ones,
+# until INT signal
+class Tupelo::AppBuilder
+  def start_trace
+    child passive: true do |client|
+      trap :INT do
+        exit!
+      end
+
+      note = client.notifier
+      log << ( "%6s %6s %6s %s\n" % %w{ tick cid status operation } )
+      loop do
+        status, tick, cid, op = note.wait
+        unless status == :attempt
+          s = status == :failure ? "FAIL" : ""
+          log << ( "%6d %6d %6s %p\n" % [tick, cid, s, op] )
+        end
+      end
+    end
+  end
+end

data/lib/tupelo/app.rb

@@ -22,21 +22,42 @@ module Tupelo
     end
 
     # Yields a client that runs in this process.
-    def local client_class = Client
+    def local client_class = Client, &block
       ez.local :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
         run_client client_class,
                    seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
-          yield client
+          if block
+            if block.arity == 0
+              client.instance_eval &block
+            else
+              yield client
+            end
+          else
+            client
+          end
         end
       end
     end
 
     # Yields a client that runs in a subprocess.
-    def child client_class = Client
-      ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
+    #
+    # A passive client will be forced to stop after all active clients exit. Use
+    # the passive flag for processes that wait for tuples and respond in some
+    # way. Then you do not have to manually interrupt the whole application when
+    # the active processes are done. See examples.
+    def child client_class = Client, passive: false, &block
+      ez.client :seqd, :cseqd, :arcd, passive: passive do |seqd, cseqd, arcd|
         run_client client_class,
                    seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
-          yield client
+          if block
+            if block.arity == 0
+              client.instance_eval &block
+            else
+              yield client
+            end
+          else
+            client
+          end
         end
       end
     end
@@ -61,7 +82,7 @@ module Tupelo
 
   def self.application argv: ARGV,
         servers_file: nil, blob_type: nil,
-        seqd_addr: [], cseqd_addr: [], arcd_addr: []
+        seqd_addr: [], cseqd_addr: [], arcd_addr: [], &block
 
     log_level = case
       when argv.delete("--debug"); Logger::DEBUG
@@ -79,6 +100,8 @@ module Tupelo
       end
       blob_type ||= "msgpack"
     end
+    
+    enable_trace = ARGV.delete("--trace")
 
     svrs = servers_file || argv.shift || "servers-#$$.yaml"
 
@@ -115,7 +138,22 @@ module Tupelo
         end
       end
 
-      yield AppBuilder.new(ez, owns_servers: owns_servers)
+      app = AppBuilder.new(ez, owns_servers: owns_servers)
+      
+      if enable_trace
+        require 'tupelo/app/trace'
+        app.start_trace
+      end
+
+      if block
+        if block.arity == 0
+          app.instance_eval &block
+        else
+          yield app
+        end
+      else
+        app
+      end
     end
   end
 end

data/lib/tupelo/archiver/worker.rb

@@ -3,6 +3,11 @@ require 'funl/history-worker'
 class Tupelo::Archiver
   class Worker < Tupelo::Client::Worker
     include Funl::HistoryWorker
+    
+    def initialize *args
+      super
+      @scheduled_actions = Hash.new {|h,k| h[k] = []}
+    end
 
     def handle_client_request req
       case req
@@ -17,7 +22,7 @@ class Tupelo::Archiver
       stream = client.arc_server_stream_for req.io
 
       begin
-        op, args = stream.read
+        op, tags, tick = stream.read
       rescue EOFError
         log.debug {"#{stream.peer_name} disconnected from archiver"}
         return
@@ -26,9 +31,19 @@ class Tupelo::Archiver
       end
 
       log.info {
-        "#{stream.peer_name} requested #{op.inspect}" +
-          (args ? " on #{args.inspect}" : "")}
+        "#{stream.peer_name} requested #{op.inspect} at tick=#{tick}" +
+          (tags ? " on #{tags}" : "")}
+
+      if tick <= global_tick
+        fork_for_op op, tags, tick, stream, req
+      else
+        at_tick tick do
+          fork_for_op op, tags, tick, stream, req
+        end
+      end
+    end
 
+    def fork_for_op op, tags, tick, stream, req
       fork do
         begin
           case op
@@ -37,7 +52,7 @@ class Tupelo::Archiver
           when "get range" ### handle this in Funl::HistoryWorker
             raise "Unimplemented" ###
           when GET_TUPLESPACE
-            send_tuplespace stream, args
+            send_tuplespace stream, tags
           else
             raise "Unknown operation: #{op.inspect}"
           end
@@ -50,6 +65,18 @@ class Tupelo::Archiver
     ensure
       req.io.close
     end
+    
+    def at_tick tick, &action
+      @scheduled_actions[tick] << action
+    end
+
+    def handle_message msg
+      super
+      actions = @scheduled_actions.delete(global_tick)
+      actions and actions.each do |action|
+        action.call
+      end
+    end
 
     def send_tuplespace stream, templates
       log.info {

data/lib/tupelo/client/reader.rb

@@ -3,7 +3,6 @@ require 'tupelo/client/common'
 class Tupelo::Client
   # include into class that defines #worker and #log
   module Api
-    ## need read with more complex predicates: |, &, etc
     def read_wait template
       waiter = Waiter.new(worker.make_template(template), self)
       worker << waiter
@@ -15,7 +14,6 @@ class Tupelo::Client
     end
     alias read read_wait
 
-    ## need nonwaiting reader that accepts 2 or more templates
     def read_nowait template
       matcher = Matcher.new(worker.make_template(template), self)
       worker << matcher

data/lib/tupelo/client/transaction.rb

@@ -8,24 +8,35 @@ class Tupelo::Client
   class TransactionFailure < TransactionError; end
 
   module Api
+    def trans_class
+      Transaction
+    end
+
     # Transactions are atomic by default, and are always isolated. In the
     # non-atomic case, a "transaction" is really a batch op. Without a block,
     # returns the Transaction. In the block form, transaction automatically
     # waits for successful completion and returns the value of the block.
-    def transaction atomic: true, timeout: nil
+    def transaction atomic: true, timeout: nil, &block
       deadline = timeout && Time.now + timeout
-      begin
-        t = Transaction.new self, atomic: atomic, deadline: deadline
-        return t unless block_given?
-        val = yield t
-        t.commit.wait
-        return val
-      rescue TransactionFailure => ex
-        log.info {"retrying #{t.inspect}: #{ex}"}
-        retry
-      rescue TransactionAbort
-        log.info {"aborting #{t.inspect}"}
-      end
+      t = trans_class.new self, atomic: atomic, deadline: deadline
+      return t unless block_given?
+
+      val =
+        if block.arity == 0
+          t.instance_eval &block
+        else
+          yield t
+        end
+
+      t.commit.wait
+      return val
+    rescue TransactionFailure => ex
+      log.info {"retrying #{t.inspect}: #{ex}"}
+      retry
+    rescue TransactionAbort
+      log.info {"aborting #{t.inspect}"}
+    ensure
+      t.cancel if t and t.open? and block_given?
     end
     
     def batch &bl
@@ -81,7 +92,6 @@ class Tupelo::Client
   class Transaction
     attr_reader :client
     attr_reader :worker
-    attr_reader :log
     attr_reader :atomic
     attr_reader :deadline
     attr_reader :status
@@ -134,6 +144,7 @@ class Tupelo::Client
       @granted_tuples = nil
       @missing = nil
       @_take_nowait = nil
+      @_read_nowait = nil
       
       if deadline
         worker.at deadline do
@@ -144,6 +155,14 @@ class Tupelo::Client
       open!
     end
     
+    def log *args
+      if args.empty?
+        @log
+      else
+        @log.unknown *args
+      end
+    end
+
     def inspect
       stat_extra =
         case
@@ -207,7 +226,7 @@ class Tupelo::Client
       template = worker.make_template(template_spec)
       @take_templates << template
       log.debug {"asking worker to take #{template_spec.inspect}"}
-      worker << self
+      worker_push self
       wait
       return take_tuples.last
     end
@@ -223,7 +242,7 @@ class Tupelo::Client
       @_take_nowait[i] = true
       @take_templates << template
       log.debug {"asking worker to take_nowait #{template_spec.inspect}"}
-      worker << self
+      worker_push self
       wait
       return take_tuples[i]
     end
@@ -237,23 +256,52 @@ class Tupelo::Client
       template = worker.make_template(template_spec)
       @read_templates << template
       log.debug {"asking worker to read #{template_spec.inspect}"}
-      worker << self
+      worker_push self
       wait
       return read_tuples.last
     end
-    
+
+    def read_nowait template_spec
+      raise "cannot read in batch" unless atomic
+      raise exception if failed?
+      raise TransactionStateError, "not open: #{inspect}" unless open? or
+        failed?
+      template = worker.make_template(template_spec)
+      @_read_nowait ||= {}
+      i = @read_templates.size
+      @_read_nowait[i] = true
+      @read_templates << template
+      log.debug {"asking worker to read #{template_spec.inspect}"}
+      worker_push self
+      wait
+      return read_tuples[i]
+    end
+
+    # Client may call this before commit. In transaction do...end block,
+    # this causes transaction to be re-executed.
+    def fail!
+      raise if in_worker_thread?
+      raise unless open?
+      failed!
+      raise TransactionFailure
+    end
+
     # idempotent
     def commit
       if open?
         closed!
         log.info {"committing #{inspect}"}
-        worker << self
+        worker_push self
       else
         raise exception if failed?
       end
       return self
     end
-    
+
+    def worker_push event=Proc.new
+      worker << event
+    end
+
     def wait
       return self if done?
       raise exception if failed?
@@ -268,7 +316,7 @@ class Tupelo::Client
       raise "bug: #{inspect}"
 
     rescue TransactionAbort, Interrupt, TimeoutError => ex ## others?
-      worker << Unwaiter.new(self)
+      worker_push Unwaiter.new(self)
       raise ex.class,
         "#{ex.message}: client #{client.client_id} waiting for #{inspect}"
     end
@@ -353,13 +401,24 @@ class Tupelo::Client
           @_take_nowait.delete i
         end
         
+        skip = nil
         (read_tuples.size...read_templates.size).each do |i|
           read_tuples[i] = worker.tuplespace.find_match_for(
             read_templates[i])
           if read_tuples[i]
             log.debug {"prepared #{inspect} with #{read_tuples[i]}"}
+          else
+            if @_read_nowait and @_read_nowait[i]
+              (skip ||= []) << i
+            end
           end
         end
+
+        skip and skip.reverse_each do |i|
+          read_tuples.delete_at i
+          read_templates.delete_at i
+          @_read_nowait.delete i
+        end
       end
       
       ## convert cancelling write/take to pulse
@@ -440,9 +499,9 @@ class Tupelo::Client
 
     # Called by another thread to cancel a waiting transaction.
     def cancel err = TransactionAbort
-      worker << proc do
+      worker_push do
         raise unless in_worker_thread?
-        if @global_tick or @exception
+        if not open? or @global_tick or @exception
           log.info {"cancel was applied too late: #{inspect}"}
         else
           @exception = err.new
@@ -450,6 +509,7 @@ class Tupelo::Client
           @queue << false
         end
       end
+      nil
     end
   end
 end

data/lib/tupelo/client/worker.rb

@@ -9,7 +9,6 @@ class Tupelo::Client
     attr_reader :client
     attr_reader :seq
     attr_reader :arc
-    attr_reader :log
     attr_reader :client_id
     attr_reader :local_tick
     attr_reader :global_tick
@@ -80,6 +79,14 @@ class Tupelo::Client
       @stopping = false
     end
 
+    def log *args
+      if args.empty?
+        @log
+      else
+        @log.unknown *args
+      end
+    end
+
     def start
       return if @worker_thread
 
@@ -223,7 +230,7 @@ class Tupelo::Client
       end
 
       log.info "requesting tuplespace from arc"
-      arc << [GET_TUPLESPACE, nil]
+      arc << [GET_TUPLESPACE, nil, tick]
         ## replace nil with template tuples, if any
 
       begin
@@ -231,19 +238,21 @@ class Tupelo::Client
         log.info "arc says global_tick = #{arc_tick}"
 
         done = false
+        count = 0
         arc.each do |tuple|
           if tuple.nil?
             done = true
           else
             raise "bad object stream from archiver" if done
             tuplespace.insert tuple
+            count += 1
           end
         end
         unless done
           raise "did not get all of tuplespace from archiver" ## roll back?
         end
 
-        log.info "received tuplespace from arc"
+        log.info "received tuplespace from arc: #{count} tuples"
 
         @global_tick = arc_tick
         log.info "global_tick = #{global_tick}"
@@ -262,9 +271,9 @@ class Tupelo::Client
             # due to archiver timing, for example
           return
         elsif msg.global_tick > global_tick + 1
-          log.error "message out of order: #{msg.inspect}, " +
+          log.fatal "message out of order: #{msg.inspect}, " +
             "received at global_tick=#{global_tick}"
-          ## exit? wait? in udp case, get history
+          raise "fatal error"
         end
       end
 

data/lib/tupelo/util/boolean.rb

@@ -0,0 +1,25 @@
+class Tupelo::Client
+  class Or
+    attr_reader :templates
+
+    def initialize worker, templates
+      @templates = templates.map {|template| worker.make_template(template)}
+    end
+
+    def === obj
+      templates.any? {|template| template === obj}
+    end
+  end
+  
+  def or *templates
+    Or.new(worker, templates)
+  end
+  alias match_any or
+end
+
+class Tupelo::Client::Transaction
+  def or *templates
+    client.or *templates
+  end
+  alias match_any or
+end

data/lib/tupelo/version.rb

@@ -1,3 +1,3 @@
 module Tupelo
-  VERSION = "0.4"
+  VERSION = "0.5"
 end

data/test/lib/time-fuzz.rb

@@ -0,0 +1,36 @@
+module Tupelo
+  module TimeFuzz
+    DEFAULT_SLEEP_MAX = 0.01
+    @sleep_max = DEFAULT_SLEEP_MAX
+
+    class << self
+      attr_accessor :sleep_max
+    end
+
+    module Api
+      def trans_class
+        TimeFuzz::Transaction
+      end
+    end
+
+    class Transaction < Tupelo::Client::Transaction
+      def worker_push event=Proc.new
+        sleep sleep_duration
+        super
+      end
+
+      def wait
+        super
+        sleep sleep_duration
+      end
+
+      def sleep_duration
+        rand * TimeFuzz.sleep_max
+      end
+    end
+    
+    class Client < Tupelo::Client
+      include TimeFuzz::Api
+    end
+  end
+end

data/test/stress/archiver-load.rb

@@ -0,0 +1,16 @@
+# Stresses the ability of the archiver to handle highl load, and in particular
+# situations where the client asks for data more recent than what the archiver
+# has received.
+
+require 'tupelo/app'
+
+N = 100
+
+Tupelo.application do
+  N.times do |i|
+    sleep i/1000.0
+    child do
+      write [1]
+    end
+  end
+end