tupelo 0.1

data/lib/tupelo/app.rb

@@ -0,0 +1,121 @@
+require 'easy-serve'
+require 'tupelo/client'
+
+## this could be unified with the implementation of bin/tup, which is similar
+
+module Tupelo
+  # Not an essential part of the library, but used to build up groups of
+  # processes for use in examples, tests, benchmarks, etc.
+  class AppBuilder
+    attr_reader :ez
+
+    # Does this app own (as child processes) the seq, cseq, and arc servers?
+    attr_reader :owns_servers
+
+    def initialize ez, owns_servers: nil
+      @ez = ez
+      @owns_servers = owns_servers
+    end
+
+    def log
+      ez.log
+    end
+
+    # Yields a client that runs in this process.
+    def local client_class = Client
+      ez.local :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
+        run_client client_class,
+                   seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
+          yield client
+        end
+      end
+    end
+
+    # Yields a client that runs in a subprocess.
+    def child client_class = Client
+      ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
+        run_client client_class,
+                   seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
+          yield client
+        end
+      end
+    end
+
+    def run_client client_class, opts
+      log = opts[:log]
+      log.progname = "client <starting in #{log.progname}>"
+      client = client_class.new opts
+      client.start do
+        log.progname = "client #{client.client_id}"
+      end
+      yield client
+    ensure
+      client.stop if client # gracefully exit the tuplespace management thread
+    end
+  end
+
+  #blob_type: 'msgpack' # the default
+  #blob_type: 'marshal' # if you need to pass general ruby objects
+  #blob_type: 'yaml' # less general ruby objects, but cross-language
+  #blob_type: 'json' # more portable than yaml, but more restrictive
+
+  def self.application argv: ARGV,
+        servers_file: nil, blob_type: nil,
+        seqd_addr: [], cseqd_addr: [], arcd_addr: []
+
+    log_level = case
+      when argv.delete("--debug"); Logger::DEBUG
+      when argv.delete("--info");  Logger::INFO
+      when argv.delete("--warn");  Logger::WARN
+      when argv.delete("--error"); Logger::ERROR
+      when argv.delete("--fatal"); Logger::FATAL
+      else Logger::WARN
+    end
+
+    unless blob_type
+      %w{--marshal --yaml --json --msgpack}.each do |switch|
+        s = argv.delete(switch) and
+          blob_type ||= s.delete("--")
+      end
+      blob_type ||= "msgpack"
+    end
+
+    svrs = servers_file || argv.shift || "servers-#$$.yaml"
+
+    EasyServe.start(servers_file: svrs) do |ez|
+      log = ez.log
+      log.level = log_level
+      log.progname = "parent"
+      owns_servers = false
+
+      ez.start_servers do
+        owns_servers = true
+
+        arc_to_seq_sock, seq_to_arc_sock = UNIXSocket.pair
+        arc_to_cseq_sock, cseq_to_arc_sock = UNIXSocket.pair
+
+        ez.server :seqd, *seqd_addr do |svr|
+          require 'funl/message-sequencer'
+          seq = Funl::MessageSequencer.new svr, seq_to_arc_sock, log: log,
+            blob_type: blob_type
+          seq.start
+        end
+
+        ez.server :cseqd, *cseqd_addr do |svr|
+          require 'funl/client-sequencer'
+          cseq = Funl::ClientSequencer.new svr, cseq_to_arc_sock, log: log
+          cseq.start
+        end
+
+        ez.server :arcd, *arcd_addr do |svr|
+          require 'tupelo/archiver'
+          arc = Archiver.new svr, seq: arc_to_seq_sock,
+            cseq: arc_to_cseq_sock, log: log
+          arc.start
+        end
+      end
+
+      yield AppBuilder.new(ez, owns_servers: owns_servers)
+    end
+  end
+end

data/lib/tupelo/archiver/tuplespace.rb

@@ -0,0 +1,68 @@
+class Tupelo::Archiver
+  class Tuplespace
+    include Enumerable
+    
+    attr_reader :zero_tolerance
+
+    def initialize(zero_tolerance: Tupelo::Archiver::ZERO_TOLERANCE)
+      @counts = Hash.new(0) # tuple => count
+      @nzero = 0
+      @zero_tolerance = zero_tolerance
+    end
+    
+    # note: multiple equal tuples are yielded once
+    def each
+      @counts.each do |tuple, count|
+        yield tuple, count if count > 0
+      end
+    end
+    
+    def insert tuple
+      @counts[tuple] += 1
+    end
+    
+    def delete_once tuple
+      if @counts[tuple] > 0
+        @counts[tuple] -= 1
+        if @counts[tuple] == 0
+          @nzero += 1
+          clear_excess_zeros if @nzero > zero_tolerance
+        end
+        true
+      else
+        false
+      end
+    end
+
+    def transaction inserts: [], deletes: []
+      deletes.each do |tuple|
+        delete_once tuple or raise "bug"
+      end
+
+      inserts.each do |tuple|
+        insert tuple.freeze ## freeze recursively
+      end
+    end
+
+    def clear_excess_zeros
+      nd = (@nzero - zero_tolerance / 2)
+      @counts.delete_if {|tuple, count| count == 0 && (nd-=1) >= 0}
+    end
+
+    def find_distinct_matches_for tuples
+      h = Hash.new(0)
+      tuples.map do |tuple|
+        if @counts[tuple] > h[tuple]
+          h[tuple] += 1
+          tuple
+        else
+          nil
+        end
+      end
+    end
+
+    def find_match_for tuple
+      @counts[tuple] > 0 && tuple
+    end
+  end
+end

data/lib/tupelo/archiver/worker.rb

@@ -0,0 +1,87 @@
+require 'funl/history-worker'
+
+class Tupelo::Archiver
+  class Worker < Tupelo::Client::Worker
+    include Funl::HistoryWorker
+
+    def handle_client_request req
+      case req
+      when Tupelo::Archiver::ForkRequest
+        handle_fork_request req
+      else
+        super
+      end
+    end
+
+    def handle_fork_request req
+      stream = client.arc_server_stream_for req.io
+
+      begin
+        op, args = stream.read
+      rescue EOFError
+        log.debug {"#{stream.peer_name} disconnected from archiver"}
+        return
+      rescue => ex
+        log.error "in fork for #{stream || req.io}: #{ex.inspect}"
+      end
+
+      log.info {
+        "#{stream.peer_name} requested #{op.inspect}" +
+          (args ? " on #{args.inspect}" : "")}
+
+      fork do
+        begin
+          case op
+          when "new client"
+            raise "Unimplemented" ###
+          when "get range" ### handle this in Funl::HistoryWorker
+            raise "Unimplemented" ###
+          when GET_TUPLESPACE
+            send_tuplespace stream, args
+          else
+            raise "Unknown operation: #{op.inspect}"
+          end
+        rescue EOFError
+          log.debug {"#{stream.peer_name} disconnected from archiver"}
+        rescue => ex
+          log.error "in fork for #{stream || req.io}: #{ex.inspect}"
+        end
+      end
+    ensure
+      req.io.close
+    end
+
+    def send_tuplespace stream, templates
+      log.info {
+        "send_tuplespace to #{stream.peer_name} " +
+        "at tick #{global_tick.inspect} " +
+        (templates ? " with templates #{templates.inspect}" : "")}
+      
+      stream << [global_tick]
+
+      if templates
+        templates = templates.map {|t| Tupelo::Client::Template.new t}
+        tuplespace.each do |tuple, count|
+          if templates.any? {|template| template === tuple}
+            count.times do
+              stream << tuple
+              ## optimization: use stream.write_to_buffer
+            end
+          end
+          ## optimize this if templates have simple form, such as
+          ##   [ [str1, nil, ...], [str2, nil, ...], ...]
+        end
+      else
+        tuplespace.each do |tuple, count|
+          count.times do ## just dump and send str * count?
+            stream << tuple ## optimize this, and cache the serial
+            ## optimization: use stream.write_to_buffer
+          end
+        end
+      end
+
+      stream << nil # terminator
+      ## stream.flush or close if write_to_buffer used above
+    end
+  end
+end

data/lib/tupelo/archiver.rb

@@ -0,0 +1,86 @@
+require 'tupelo/client'
+require 'funl/history-client'
+
+class Tupelo::Archiver < Tupelo::Client; end
+
+require 'tupelo/archiver/worker'
+require 'tupelo/archiver/tuplespace'
+
+module Tupelo
+  class Archiver
+    include Funl::HistoryClient
+
+    attr_reader :server
+    attr_reader :server_thread
+
+    # How many tuples with count=0 do we permit before cleaning up?
+    ZERO_TOLERANCE = 1000
+
+    def initialize server, **opts
+      super arc: nil, tuplespace: Tupelo::Archiver::Tuplespace, **opts
+      @server = server
+    end
+
+    # three kinds of requests:
+    #
+    # 1. fork a new client, with given Client class, and subselect
+    #   using given templates
+    #
+    # 2. accept tcp/unix socket connection and fork, and then:
+    #
+    #   a. dump subspace matching given templates OR
+    #
+    #   b. dump all ops in a given range of the global sequence
+    #      matching given templates
+    #
+    # the fork happens when tuplespace is consistent; we
+    # do this by passing cmd to worker thread, with conn
+    class ForkRequest
+      attr_reader :io
+      def initialize io
+        @io = io
+      end
+    end
+
+    def make_worker
+      Tupelo::Archiver::Worker.new self
+    end
+
+    def start
+      ## load from file?
+      super # start worker thread
+      @server_thread = Thread.new do
+        run
+      end
+    end
+
+    def stop
+      server_thread.kill if server_thread
+      super # stop worker thread
+    end
+
+    def run
+      loop do
+        ## nonblock_accept?
+        Thread.new(server.accept) do |conn|
+          handle_conn conn
+        end
+
+        ## periodically send worker request to dump space to file?
+      end
+    rescue => ex
+      log.error ex
+      raise
+    end
+
+    def handle_conn conn
+      log.debug {"accepted #{conn.inspect}"}
+      begin
+        worker << ForkRequest.new(conn)
+      rescue => ex
+        log.error ex
+        raise
+      end
+    end
+  end
+end

data/lib/tupelo/client/common.rb

@@ -0,0 +1,10 @@
+module Tupelo
+  class Client
+    class Unwaiter
+      attr_reader :waiter
+      def initialize waiter
+        @waiter = waiter
+      end
+    end
+  end
+end

data/lib/tupelo/client/reader.rb

@@ -0,0 +1,124 @@
+require 'tupelo/client/common'
+
+class Tupelo::Client
+  # include into class that defines #worker and #log
+  module Api
+    ## need read with more complex predicates: |, &, etc
+    def read_wait template
+      waiter = Waiter.new(worker.make_template(template), self)
+      worker << waiter
+      result = waiter.wait
+      waiter = nil
+      result
+    ensure
+      worker << Unwaiter.new(waiter) if waiter
+    end
+    alias read read_wait
+
+    ## need nonwaiting reader that accepts 2 or more templates
+    def read_nowait template
+      matcher = Matcher.new(worker.make_template(template), self)
+      worker << matcher
+      matcher.wait
+    end
+
+    # By default, reads *everything*.
+    def read_all template = Object
+      matcher = Matcher.new(worker.make_template(template), self, :all => true)
+      worker << matcher
+      a = []
+      while tuple = matcher.wait ## inefficient?
+        yield tuple if block_given?
+        a << tuple
+      end
+      a
+    end
+
+    def notifier
+      NotifyWaiter.new(self).tap {|n| n.toggle}
+    end
+  end
+
+  class WaiterBase
+    attr_reader :template
+    attr_reader :queue
+
+    def initialize template, client
+      @template = template
+      @queue = client.make_queue
+      @client = client
+    end
+
+    def gloms tuple
+      if template === tuple
+        peek tuple
+        true
+      else
+        false
+      end
+    end
+    
+    def peek tuple
+      queue << tuple
+    end
+
+    def wait
+      @client.log.debug {"waiting for #{self}"}
+      r = queue.pop
+      @client.log.debug {"finished waiting for #{self}"}
+      r
+    end
+
+    def inspect
+      "<#{self.class}: #{template.inspect}>"
+    end
+  end
+  
+  class Waiter < WaiterBase
+  end
+  
+  class Matcher < WaiterBase
+    attr_reader :all # this is only cosmetic -- see #inspect
+
+    def initialize template, client, all: false
+      super template, client
+      @all = all
+    end
+
+    def fails
+      queue << nil
+    end
+
+    def inspect
+      e = all ? "all " : ""
+      t = template.inspect
+      "<#{self.class}: #{e}#{t}>"
+    end
+  end
+
+  # Instrumentation.
+  class NotifyWaiter
+    attr_reader :queue
+
+    def initialize client
+      @client = client
+      @queue = client.make_queue
+    end
+
+    def << event
+      queue << event
+    end
+
+    def wait
+      queue.pop
+    end
+
+    def toggle
+      @client.worker << self
+    end
+
+    def inspect
+      to_s
+    end
+  end
+end