tupelo 0.1

data/example/map-reduce-v2.rb

@@ -0,0 +1,96 @@
+require 'tupelo/app'
+
+N = 2 # how many cpus do you want to use for mappers?
+VERBOSE = ARGV.delete "-v"
+
+class Tupelo::Client
+  class Or
+    attr_reader :templates
+
+    def initialize worker, templates
+      @templates = templates.map {|template| worker.make_template(template)}
+    end
+
+    def === obj
+      templates.any? {|template| template === obj}
+    end
+  end
+  
+  def or *templates
+    Or.new(worker, templates)
+  end
+end
+
+Tupelo.application do |app|
+  if VERBOSE
+    app.child do |client| # a debugger client, to see what's happening
+      note = client.notifier
+      puts "%4s %4s %10s %s" % %w{ tick cid status operation }
+      loop do
+        status, tick, cid, op = note.wait
+        unless status == :attempt
+          s = status == :failure ? "FAILED" : ""
+          puts "%4d %4d %10s %p" % [tick, cid, s, op]
+        end
+      end
+    end
+  end
+
+  app.child do |client|
+    document = "I will not map reduce in class\n" * 10
+    lineno = 0
+    document.each_line do |line|
+      lineno += 1
+      client.write line: line, lineno: lineno
+        # Note that tuples should be small, so if the data is large, the line
+        # should be a reference, not the actual data.
+        # Also, in a complex application you might want to add another
+        # key/value pair to avoid collisions (owner: ..., for example).
+    end
+
+    results = Hash.new(0)
+    lines_remaining = lineno
+    results_remaining = 0
+    result_template = client.or(
+      {word: String, count: Integer},
+      {lineno: Integer, result_count: Integer}
+    )
+
+    until lines_remaining == 0 and results_remaining == 0 do
+      result = client.take result_template
+
+      if result["word"]
+        results[result["word"]] += result["count"]
+        results_remaining -= 1
+      elsif result["lineno"]
+        lines_remaining -= 1
+        results_remaining += result["result_count"]
+      else
+        log.error "bad keys in result: #{result}"
+      end
+    end
+
+    client.log "DONE. results = #{results}"
+    client.log "Press ^C to exit"
+  end
+  
+  N.times do |i|
+    app.child do |client|
+      client.log.progname = "mapper #{i}"
+      
+      loop do
+        input = client.take line: String, lineno: Integer
+
+        h = Hash.new(0)
+        input["line"].split.each do |word|
+          h[word] += 1
+        end
+
+        h.each do |word, count|
+          client.write word: word, count: count
+        end
+        client.write lineno: input["lineno"], result_count: h.size
+      end
+    end
+  end
+end

data/example/map-reduce.rb

@@ -0,0 +1,77 @@
+require 'tupelo/app'
+
+N = 2 # how many cpus do you want to use for mappers?
+VERBOSE = ARGV.delete "-v"
+
+Tupelo.application do |app|
+  if VERBOSE
+    app.child do |client| # a debugger client, to see what's happening
+      note = client.notifier
+      puts "%4s %4s %10s %s" % %w{ tick cid status operation }
+      loop do
+        status, tick, cid, op = note.wait
+        unless status == :attempt
+          s = status == :failure ? "FAILED" : ""
+          puts "%4d %4d %10s %p" % [tick, cid, s, op]
+        end
+      end
+    end
+  end
+
+  app.child do |client|
+    document = "I will not map reduce in class\n" * 10
+    lineno = 0
+    document.each_line do |line|
+      lineno += 1
+      client.write ["wc input", lineno, line]
+        # Note that tuples should be small, so if the data is large, the line
+        # should be a reference, not the actual data.
+    end
+
+    results = Hash.new(0)
+    lines_remaining = lineno
+    results_remaining = 0
+    until lines_remaining == 0 and results_remaining == 0 do
+      event, *a = client.take [/wc (?:output|done)/, nil, nil]
+        # Using a regex is hacky. Better to use an "or" template. See
+        # boolean-match.rb. That's not a standard feature yet.
+        # Also, in real use it might be better to use hash tuples rather than
+        # arrays.
+        # See map-reduce-v2.rb.
+
+      case event
+      when "wc output"
+        word, count = a
+        results[word] += count
+        results_remaining -= 1
+      when "wc done"
+        lineno, result_count = a
+        lines_remaining -= 1
+        results_remaining += result_count
+      end
+    end
+
+    client.log "DONE. results = #{results}"
+    client.log "Press ^C to exit"
+  end
+  
+  N.times do |i|
+    app.child do |client|
+      client.log.progname = "mapper #{i}"
+      
+      loop do
+        _, lineno, line = client.take ["wc input", Integer, String]
+
+        h = Hash.new(0)
+        line.split.each do |word|
+          h[word] += 1
+        end
+
+        h.each do |word, count|
+          client.write ["wc output", word, count]
+        end
+        client.write ["wc done", lineno, h.size]
+      end
+    end
+  end
+end

data/example/matching.rb

@@ -0,0 +1,9 @@
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.local do |client|
+    client.write_wait ["foo", 42.5]
+    p client.read_all [/oo/, nil]
+    p client.read_all [nil, 5..95]
+  end
+end

data/example/notify.rb

@@ -0,0 +1,35 @@
+# See also bin/tspy
+
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    Thread.new do
+      note = client.notifier
+      client.write ["start"]
+
+      client.log "%10s %10s %10s %s" % %w{ status tick client operation }
+      loop do
+        status, global_tick, client_id, op = note.wait
+        client.log "%10s %10d %10d %p" % [status, global_tick, client_id, op]
+      end
+    end
+
+    client.take ["finish"]
+  end
+  
+  app.child do |client|
+    client.take ["start"]
+
+    client.write [1, 2]
+    client.write [3, 4]
+    client.write foo: "bar", baz: ["zap"]
+    
+    client.transaction do |t|
+      x, y = t.take [Numeric, Numeric]
+      t.write [x, y, x + y]
+    end
+    
+    client.write ["finish"]
+  end
+end

data/example/optimist.rb

@@ -0,0 +1,20 @@
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    client.write [1]
+    sleep 0.1
+    client.take [1]
+    client.write [2]
+  end
+  
+  app.child do |client|
+    final_i =
+      client.take([Integer]) do |optimistic_i|
+        client.log "optimistic_i = #{optimistic_i}"
+        sleep 0.2
+      end
+    
+    client.log "final_i = #{final_i}"
+  end
+end

data/example/pulse.rb

@@ -0,0 +1,24 @@
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    client.take ['start']
+    10.times do |i|
+      client.pulse [i]
+      sleep 0.1
+    end
+    client.write ['finish']
+  end
+  
+  app.child do |client|
+    Thread.new do
+      loop do
+        client.log client.read [Integer]
+        client.log client.read_all [Integer]
+      end
+    end
+    
+    client.write ['start']
+    client.take ['finish']
+  end
+end

data/example/read-in-trans.rb

@@ -0,0 +1,56 @@
+require 'tupelo/app'
+
+Thread.abort_on_exception = true
+
+N = 5
+
+Tupelo.application do |app|
+  N.times do |i|
+    app.child do |client|
+      Thread.new do
+        step = 0
+        loop do
+          step += 1
+          client.transaction do |t|
+            t.read ["enabled", i]
+            t.write ["output", i, step]
+          end
+          sleep 0.2
+        end
+      end
+      
+      client.read ["done"]
+      exit
+    end
+  end
+  
+  app.child do |client|
+    t = Thread.new do
+      loop do
+        msg, i, step = client.take [nil, nil, nil]
+        printf "%20s from %2d at step %3d\n", msg, i, step
+      end
+    end
+
+    puts "Turning on 0 and 4"
+    client.write ["enabled", 0]
+    client.write ["enabled", 4]
+    sleep 2
+    
+    puts "Turning off 0"
+    client.take ["enabled", 0]
+    sleep 2
+    
+    puts "Turning off 4"
+    client.take ["enabled", 4]
+    sleep 2
+    
+    puts "Turning on 1 and 3"
+    client.write ["enabled", 1]
+    client.write ["enabled", 3]
+    sleep 2
+    
+    puts "Bye!"
+    client.write ["done"]
+  end
+end

data/example/small-simplified.rb

@@ -0,0 +1,18 @@
+# Same as small.rb, but with the generic setup code factored out.
+
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    client.write [2, 3, "frogs"]
+    _, s = client.take ["animals", nil]
+    puts s
+  end
+
+  app.child do |client|
+    x, y, s = client.take [Numeric, Numeric, String]
+    s2 = ([s] * (x + y)).join(" ")
+    client.write ["animals", s2]
+  end
+end
+

data/example/small.rb

@@ -0,0 +1,76 @@
+# An example of using EasyServe directly (the hard way) rather than tupelo/app.
+# See small-simplified.rb for the difference.
+
+require 'easy-serve'
+
+log_level = case
+  when ARGV.delete("--debug"); Logger::DEBUG
+  when ARGV.delete("--info");  Logger::INFO
+  when ARGV.delete("--warn");  Logger::WARN
+  when ARGV.delete("--error"); Logger::ERROR
+  when ARGV.delete("--fatal"); Logger::FATAL
+  else Logger::WARN
+end
+
+EasyServe.start(servers_file: "small-servers.yaml") do |ez|
+  log = ez.log
+  log.level = log_level
+  log.progname = "parent"
+
+  ez.start_servers do
+    arc_to_seq_sock, seq_to_arc_sock = UNIXSocket.pair
+    arc_to_cseq_sock, cseq_to_arc_sock = UNIXSocket.pair
+    
+    ez.server :seqd do |svr|
+      require 'funl/message-sequencer'
+      seq = Funl::MessageSequencer.new svr, seq_to_arc_sock, log: log,
+        blob_type: 'msgpack' # the default
+        #blob_type: 'marshal' # if you need to pass general ruby objects
+        #blob_type: 'yaml' # less general ruby objects, but cross-language
+        #blob_type: 'json' # more portable than yaml, but more restrictive
+      seq.start
+    end
+    
+    ez.server :cseqd do |svr|
+      require 'funl/client-sequencer'
+      cseq = Funl::ClientSequencer.new svr, cseq_to_arc_sock, log: log
+      cseq.start
+    end
+
+    ez.server :arcd do |svr|
+      require 'tupelo/archiver'
+      arc = Archiver.new svr, seq: arc_to_seq_sock, cseq: arc_to_cseq_sock,
+              log: log
+      arc.start
+    end
+  end
+  
+  def run_client opts
+    log = opts[:log]
+    log.progname = "client <starting in #{log.progname}>"
+    require 'tupelo/client'
+    client = Client.new opts
+    client.start do
+      log.progname = "client #{client.client_id}"
+    end
+    yield client
+  ensure
+    client.stop if client # gracefully exit the tuplespace management thread
+  end
+
+  ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
+    run_client seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
+      client.write [2, 3, "frogs"]
+      _, s = client.take ["animals", nil]
+      puts s
+    end
+  end
+
+  ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
+    run_client seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
+      x, y, s = client.take [Numeric, Numeric, String]
+      s2 = ([s] * (x + y)).join(" ")
+      client.write ["animals", s2]
+    end
+  end
+end

data/example/tcp.rb

@@ -0,0 +1,35 @@
+# Most examples in this dir use unix sockets for simplicity. With a little
+# more effort, you can switch to tcp for remote access.
+#
+# Run this file like:
+#
+#  ruby tcp.rb --info
+#
+# You can test with a local client:
+#
+#   ../bin/tup tcp.yaml
+#
+# Copy tcp.yaml to a remote host, and in the remote copy edit the
+# addr field to a hostname (or ip addr) isntead of 0.0.0.0.
+#
+# Then run a client like this:
+#
+#   bin/tup remote-copy-of-tcp.yaml
+
+require 'tupelo/app'
+
+svr = "tcp.yaml" # copy this file to remote clients, setting host as needed
+port = 9901 # Use 0 to let system choose free port
+
+Tupelo.application servers_file: svr,
+            seqd_addr:  [:tcp, '0.0.0.0', port],
+            cseqd_addr: [:tcp, '0.0.0.0', port + 1],
+            arcd_addr:  [:tcp, '0.0.0.0', port + 2] do |app|
+  if app.owns_servers
+    puts "server started; ^C to stop"
+    puts "(run with --info to see events)" if app.log.level > Logger::INFO
+    sleep
+  else
+    abort "server seems to be running already; check file #{svr.inspect}"
+  end
+end

data/example/timeout-trans.rb

@@ -0,0 +1,21 @@
+# Use tupelo's scheduler to time out a transaction. See also timeout.rb.
+# This has the advantage of using just one extra thread for all timeouts,
+# rather than one thread per timeout.
+
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    result =
+      begin
+        client.transaction timeout: 1 do |t|
+          t.take ["foo"]
+        end
+      rescue TimeoutError => ex
+        ex
+      end
+    client.log "result = #{result.inspect}"
+  end
+end
+
+

data/example/timeout.rb

@@ -0,0 +1,27 @@
+# Use ruby's timeout lib to time out a read. For transactions there is also
+# the timeout parameter (which is more efficient). See timeout-trans.rb.
+
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    begin
+      n_sec = 2
+      Timeout.timeout n_sec do
+        client.log "waiting for non-existing tuple #{[0]}"
+        client.read [0]
+      end
+    rescue TimeoutError
+    end
+    
+    r = client.read [1]
+    client.log "got #{r}"
+  end
+
+  app.child do |client|
+    sleep 1
+    client.log "writing [1]"
+    client.write [1]
+  end
+end
+

data/example/tiny-client.rb

@@ -0,0 +1,14 @@
+require 'tupelo/app'
+
+svr = "tiny-server.yaml"
+
+Tupelo.application servers_file: svr do |app|
+  if app.owns_servers
+    abort "server not running"
+  end
+
+  app.child do |client|
+    client.write ["Hello", "world!"]
+    p client.take [nil, nil]
+  end
+end

data/example/tiny-server.rb

@@ -0,0 +1,12 @@
+require 'tupelo/app'
+
+svr = "tiny-server.yaml"
+
+Tupelo.application servers_file: svr do |app|
+  if app.owns_servers
+    puts "server started"
+    sleep
+  else
+    abort "server seems to be running already; check file #{svr.inspect}"
+  end
+end

data/example/transaction-logic.rb

@@ -0,0 +1,40 @@
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.child do |client|
+    done = false
+    until done do
+      client.transaction do |t|
+        op, x, y = t.take [String, Numeric, Numeric]
+        # further operations within this transaction can depend on the above.
+        
+        case op
+        when "+"
+          t.write [op, x, y, x + y]
+        when "*"
+          t.write [op, x, y, x * y]
+        when "select"
+          _, _, _, z = t.take [nil, nil, nil, x..y]
+          t.write [op, x, y, z]
+        when "stop"
+          done = true
+        end
+      end
+    end
+  end
+
+  app.child do |client|
+    client.write ["+", 1, 2]
+    results = client.read ["+", 1, 2, nil]
+    p results
+
+    client.write ["*", 3, 4]
+    results = client.read ["*", 3, 4, nil]
+    p results
+
+    client.write ["select", 10, 20]
+    p client.read ["select", 10, 20, nil]
+    
+    client.write ["stop", 0, 0]
+  end
+end

data/example/write-wait.rb

@@ -0,0 +1,17 @@
+require 'tupelo/app'
+
+Tupelo.application do |app|
+  app.local do |client|
+    client.write [1]
+    client.write [2]
+    w = client.write [3]
+    p client.read_all [nil]
+    w.wait # wait for the write to come back and be applied to the client
+    p client.read_all [nil]
+
+    client.write [4]
+    client.write [5]
+    client.write_wait [6]
+    p client.read_all [nil]
+  end
+end