tupelo 0.7 → 0.8

data/lib/tupelo/client/worker.rb

@@ -63,12 +63,7 @@ class Tupelo::Client
       @delta = 0
 
       @cmd_queue = client.make_queue
-      @tuplespace =
-        begin
-          client.tuplespace.new
-        rescue NoMethodError
-          client.tuplespace
-        end
+      @tuplespace = nil
       @message_class = client.message_class
       @blobber = nil
 
@@ -86,6 +81,16 @@ class Tupelo::Client
         @log.unknown *args
       end
     end
+    
+    def tuplespace
+      @tuplespace ||= begin
+        if client.tuplespace.respond_to? :new
+          client.tuplespace.new
+        else
+          client.tuplespace
+        end
+      end
+    end
 
     def start
       return if @worker_thread
@@ -119,7 +124,6 @@ class Tupelo::Client
       worker_thread.join if worker_thread ## join(limit)?
       msg_reader_thread.kill if msg_reader_thread
       @atdo.stop if @atdo
-      ## optionally write final state (including global_tick) to disk
     end
 
     # stop without any remote handshaking
@@ -203,27 +207,14 @@ class Tupelo::Client
 
     def update_to_tick tick
       # at this point we know that the seq messages now accumulating in
-      # cmd_queue are tick+1, tick+2, .... In particular, if tick==0, we don't
-      # need to get state from archiver.
+      # cmd_queue are tick+1, tick+2, ....
       log.debug {"update_to_tick #{tick}"}
 
-      if tick == 0
-        @global_tick = 0
-        return
-      end
-
-      if false
-        ## load from file, update @global_tick, and see if equal to tick
-        if @global_tick == tick
-          return
-        elsif @global_tick > tick
-          raise "bad tick: #{@global_tick} > #{tick}"
-        end
-      end
-
       unless arc
-        log.warn "no archiver provided; assuming pubsub mode; " +
-          "some client ops (take and local read) will not work"
+        if tick > 0
+          log.warn "no archiver provided; assuming pubsub mode; " +
+            "some client ops (take and local read) will not work"
+        end
         @global_tick = tick
         log.info "global_tick = #{global_tick}"
         return
@@ -296,7 +287,8 @@ class Tupelo::Client
 
       if succeeded
         log.debug {"inserting #{op.writes}; deleting #{actual_tuples}"}
-        tuplespace.transaction inserts: op.writes, deletes: actual_tuples
+        tuplespace.transaction inserts: op.writes, deletes: actual_tuples,
+          tick: @global_tick
       end
 
       notify_waiters.each do |waiter|

data/lib/tupelo/tuplets/persistent-archiver/tuplespace.rb

@@ -0,0 +1,86 @@
+require 'sequel'
+
+class Tupelo::PersistentArchiver
+  class Tuplespace
+    include Enumerable
+    
+    attr_reader :zero_tolerance
+
+    def initialize(file: ":memory:",
+                   zero_tolerance: Tupelo::Archiver::ZERO_TOLERANCE)
+      @db = Sequel.sqlite(:database => file)
+      @nzero = 0
+      @zero_tolerance = zero_tolerance
+    end
+    
+    # note: multiple equal tuples are yielded once
+    def each
+      @db[:tuples].select(:packed, :count).each do |row| ## select as array?
+        packed, count = row.values_at(:packed, :count)
+        yield packed, count if count > 0
+      end
+    end
+    
+    def insert packed
+      if packed has exact match in data table
+        inc count
+      else
+        let hash = packed_hash(str)
+        select rows with this hash
+
+        if op is insert
+          if rows.count == 0, insert new row, with count=1
+          else find row using packed_compare(str, packed_tuple)
+            if found, increment count
+            else insert new row, with count=1
+
+
+      @db[:tuples].insert 
+      @counts[tuple] += 1
+    end
+    
+    def delete_once tuple
+      if @counts[tuple] > 0
+        @counts[tuple] -= 1
+        if @counts[tuple] == 0
+          @nzero += 1
+          clear_excess_zeros if @nzero > zero_tolerance
+        end
+        true
+      else
+        false
+      end
+    end
+
+    def transaction inserts: [], deletes: []
+      deletes.each do |tuple|
+        delete_once tuple or raise "bug"
+      end
+
+      inserts.each do |tuple|
+        insert tuple.freeze ## freeze recursively
+      end
+    end
+
+    def clear_excess_zeros
+      nd = (@nzero - zero_tolerance / 2)
+      @counts.delete_if {|tuple, count| count == 0 && (nd-=1) >= 0}
+    end
+
+    def find_distinct_matches_for tuples
+      h = Hash.new(0)
+      tuples.map do |tuple|
+        if @counts[tuple] > h[tuple]
+          h[tuple] += 1
+          tuple
+        else
+          nil
+        end
+      end
+    end
+
+    def find_match_for tuple
+      @counts[tuple] > 0 && tuple
+    end
+  end
+end

data/lib/tupelo/tuplets/persistent-archiver/worker.rb

@@ -0,0 +1,114 @@
+require 'funl/history-worker'
+
+class Tupelo::PersistentArchiver
+  class Worker < Tupelo::Client::Worker
+    include Funl::HistoryWorker
+    
+    def initialize *args
+      super
+      @scheduled_actions = Hash.new {|h,k| h[k] = []}
+    end
+
+    def handle_client_request req
+      case req
+      when Tupelo::Archiver::ForkRequest
+        handle_fork_request req
+      else
+        super
+      end
+    end
+
+    def handle_fork_request req
+      stream = client.arc_server_stream_for req.io
+
+      begin
+        op, tags, tick = stream.read
+      rescue EOFError
+        log.debug {"#{stream.peer_name} disconnected from archiver"}
+        return
+      rescue => ex
+        log.error "in fork for #{stream || req.io}: #{ex.inspect}"
+      end
+
+      log.info {
+        "#{stream.peer_name} requested #{op.inspect} at tick=#{tick}" +
+          (tags ? " on #{tags}" : "")}
+
+      if tick <= global_tick
+        fork_for_op op, tags, tick, stream, req
+      else
+        at_tick tick do
+          fork_for_op op, tags, tick, stream, req
+        end
+      end
+    end
+
+    def fork_for_op op, tags, tick, stream, req
+      fork do
+        begin
+          case op
+          when "new client"
+            raise "Unimplemented" ###
+          when "get range" ### handle this in Funl::HistoryWorker
+            raise "Unimplemented" ###
+          when GET_TUPLESPACE
+            send_tuplespace stream, tags
+          else
+            raise "Unknown operation: #{op.inspect}"
+          end
+        rescue EOFError
+          log.debug {"#{stream.peer_name} disconnected from archiver"}
+        rescue => ex
+          log.error "in fork for #{stream || req.io}: #{ex.inspect}"
+        end
+      end
+    ensure
+      req.io.close
+    end
+    
+    def at_tick tick, &action
+      @scheduled_actions[tick] << action
+    end
+
+    def handle_message msg
+      super
+      actions = @scheduled_actions.delete(global_tick)
+      actions and actions.each do |action|
+        action.call
+      end
+    end
+
+    def send_tuplespace stream, templates
+      log.info {
+        "send_tuplespace to #{stream.peer_name} " +
+        "at tick #{global_tick.inspect} " +
+        (templates ? " with templates #{templates.inspect}" : "")}
+      
+      stream << [global_tick]
+
+      if templates
+        templates = templates.map {|t| Tupelo::Client::Template.new t}
+        tuplespace.each do |tuple, count|
+          if templates.any? {|template| template === tuple}
+            count.times do
+              stream << tuple
+              ## optimization: use stream.write_to_buffer
+            end
+          end
+          ## optimize this if templates have simple form, such as
+          ##   [ [str1, nil, ...], [str2, nil, ...], ...]
+        end
+      else
+        tuplespace.each do |tuple, count|
+          count.times do ## just dump and send str * count?
+            stream << tuple ## optimize this, and cache the serial
+            ## optimization: use stream.write_to_buffer
+          end
+        end
+      end
+
+      stream << nil # terminator
+      ## stream.flush or close if write_to_buffer used above
+    end
+  end
+end

data/lib/tupelo/tuplets/persistent-archiver.rb

@@ -0,0 +1,86 @@
+require 'tupelo/client'
+require 'funl/history-client'
+
+class Tupelo::PersistentArchiver < Tupelo::Client; end
+
+require 'tupelo/tuplets/persistent-archiver/worker'
+require 'tupelo/tuplets/persistent-archiver/tuplespace'
+
+module Tupelo
+  class PersistentArchiver
+    include Funl::HistoryClient
+
+    attr_reader :server
+    attr_reader :server_thread
+
+    # How many tuples with count=0 do we permit before cleaning up?
+    ZERO_TOLERANCE = 1000
+
+    def initialize server, **opts
+      super arc: nil, tuplespace: Tupelo::PersistentArchiver::Tuplespace, **opts
+      @server = server
+    end
+
+    # three kinds of requests:
+    #
+    # 1. fork a new client, with given Client class, and subselect
+    #   using given templates
+    #
+    # 2. accept tcp/unix socket connection and fork, and then:
+    #
+    #   a. dump subspace matching given templates OR
+    #
+    #   b. dump all ops in a given range of the global sequence
+    #      matching given templates
+    #
+    # the fork happens when tuplespace is consistent; we
+    # do this by passing cmd to worker thread, with conn
+    class ForkRequest
+      attr_reader :io
+      def initialize io
+        @io = io
+      end
+    end
+
+    def make_worker
+      Tupelo::PersistentArchiver::Worker.new self
+    end
+
+    def start
+      ## load from file?
+      super # start worker thread
+      @server_thread = Thread.new do
+        run
+      end
+    end
+
+    def stop
+      server_thread.kill if server_thread
+      super # stop worker thread
+    end
+
+    def run
+      loop do
+        ## nonblock_accept?
+        Thread.new(server.accept) do |conn|
+          handle_conn conn
+        end
+
+        ## periodically send worker request to dump space to file?
+      end
+    rescue => ex
+      log.error ex
+      raise
+    end
+
+    def handle_conn conn
+      log.debug {"accepted #{conn.inspect}"}
+      begin
+        worker << ForkRequest.new(conn)
+      rescue => ex
+        log.error ex
+        raise
+      end
+    end
+  end
+end

data/lib/tupelo/version.rb

@@ -1,3 +1,3 @@
 module Tupelo
-  VERSION = "0.7"
+  VERSION = "0.8"
 end

data/test/lib/mock-client.rb

@@ -1,3 +1,5 @@
+require 'fiber'
+
 require 'tupelo/client/reader'
 require 'tupelo/client/transaction'
 
@@ -16,23 +18,91 @@ class MockClient
   attr_accessor :arc
   attr_accessor :start_tick
 
-  def updater
-    @updater ||=
-      Fiber.new do
-        loop do
-          log.debug "update begin"
-          worker.update
-          log.debug "update end"
-          Fiber.yield
+  class IsBlocked < RuntimeError; end
+  class IsDone < RuntimeError; end
+
+  def update
+    worker.update
+  end
+  
+  def make_queue
+    MockQueue.new
+  end
+  
+  def will &block
+    (@will_do ||= []) << Fiber.new { instance_eval &block }
+    self
+  end
+  
+  def step
+    loop do
+      fiber = @will_do[0] or raise IsDone, "nothing to do"
+
+      if fiber.alive?
+        update
+        val = fiber.resume
+        update
+        return val
+      end
+
+      @will_do.shift
+    end
+  end
+  
+  def run limit: 100
+    loop do
+      fiber = @will_do[0] or raise IsDone, "nothing to do"
+
+      count = 0
+      while fiber.alive?
+        update
+        val = fiber.resume
+        update
+        if fiber.alive? or @will_do.size > 1
+          if val == :block
+            count += 1
+            if count > limit
+              raise IsBlocked, "exceeded blocking limit"
+            end
+          else
+            count = 0
+            yield val if block_given?
+          end
+        else
+          return val
         end
       end
+
+      @will_do.shift
+    end
   end
   
-  def update
-    updater.resume
+  def run_until_blocked limit: 100, &block
+    begin
+      run limit: limit, &block
+    rescue IsBlocked
+      return
+    end
+    raise IsDone, "run_until_blocked never blocked"
   end
   
-  def make_queue
-    MockQueue.new
+  def now limit: 100, &block
+    fiber = Fiber.new { instance_eval &block }
+    val = nil
+    count = 0
+    update
+    while fiber.alive?
+      val = fiber.resume
+      if val == :block
+        count += 1
+        if count > limit
+          raise IsBlocked, "cannot now do that -- exceeded blocking limit"
+        end
+      else
+        count = 0
+      end
+      update
+    end
+    val
   end
 end

data/test/lib/mock-queue.rb

@@ -28,8 +28,8 @@ class MockQueue
       while @entries.empty?
         Fiber.yield :block
       end
-    rescue FiberError
-      raise QueueEmptyError, "queue empty"
+    rescue FiberError => ex
+      raise QueueEmptyError, "queue empty (#{ex.message})"
     end
     
     val = @entries.shift

data/test/unit/test-mock-client.rb

@@ -0,0 +1,103 @@
+require 'minitest/autorun'
+
+require 'mock-client.rb'
+
+class TestMockClient < Minitest::Test
+  class MockWorker
+    def update
+    end
+  end
+  
+  def test_step
+    c = MockClient.new
+    c.worker = MockWorker.new
+    
+    c.will do
+      3.times do |i|
+        Fiber.yield i
+      end
+      "done"
+    end
+
+    assert_equal 0, c.step
+    assert_equal 1, c.step
+    assert_equal 2, c.step
+    assert_equal "done", c.step
+    assert_raises MockClient::IsDone do
+      c.step
+    end
+  end
+  
+  def test_run
+    c = MockClient.new
+    c.worker = MockWorker.new
+    
+    c.will do
+      3.times do |i|
+        Fiber.yield i
+      end
+      "done"
+    end
+    
+    a = []
+    r = c.run do |val|
+      a << val
+    end
+    
+    assert_equal "done", r
+    assert_equal [0,1,2], a
+  end
+  
+  def test_run_until_blocked
+    c = MockClient.new
+    c.worker = MockWorker.new
+    
+    c.will do
+      3.times do |i|
+        Fiber.yield i
+      end
+      11.times do
+        Fiber.yield :block
+      end
+      "done"
+    end
+    
+    a = []
+    c.run_until_blocked limit:10 do |val|
+      a << val
+    end
+    
+    assert_equal [0,1,2], a
+
+    r = c.run
+    assert_equal "done", r
+  end
+
+  def test_now
+    c = MockClient.new
+    c.worker = MockWorker.new
+    
+    result = c.now do
+      3.times do
+        Fiber.yield :block
+      end
+      "done"
+    end
+    
+    assert_equal "done", result
+  end
+  
+  def test_now_limit
+    c = MockClient.new
+    c.worker = MockWorker.new
+    
+    assert_raises MockClient::IsBlocked do
+      c.now limit: 2 do
+        3.times do
+          Fiber.yield :block
+        end
+        "done"
+      end
+    end
+  end
+end