emb 0.4.1 → 0.4.3.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d30fe18a35d49193f1fc98fed618c871617cd144e5a06a5332106498ffb09836
-  data.tar.gz: 43ec2aa7a9ce174b219688ff0f4e537467ab434c425c3cd90ab0b6cb408fd2bc
+  metadata.gz: 0b8dde2d96792769f263d041d6385fe18968bea14f7366887069df92397473e1
+  data.tar.gz: f18d769881356e77e7c3449be87b0e541272300a71a7b3c427da61e646f05e3c
 SHA512:
-  metadata.gz: 24b32a4a561dfabc2ff1711e28ef11b1f2a9e009e3ae16b6d1b27befb2f412116c20f747043a553c57e805408f17cd0d6197e1560e366fda4382f1c77a468a04
-  data.tar.gz: 2a863ec65770fcd5630855eccd77173e47d6384d8f457b26ae4ed141c237771cac2dca84a2770437020e61b735e4e6c9a64d074d758957a5cdfcd503cf6679b5
+  metadata.gz: a39342b62646c8b9f6c8eedb3f4cb480a1f13ff03c11b682453a4728423dc27bc19602679db26c121ff63a0f609450bd9345a9e48ce210443754c6c661472668
+  data.tar.gz: f870981d417fa513f588ec1394e09e14bf864460b46666cb9d605fba9c2400d88f643345c793430298dfed375dc80828cb1e2f8fb3e96d4e0e522bd9e58bb048

data/README.md

@@ -102,6 +102,11 @@ Emb.setup(url: "redis://localhost:6379", pool: 10)
 The default pool size is **5**. The client keeps `pool` persistent connections per
 emb **instance** and routes commands through them **in round-robin order** —
 consecutive commands use different connections instead of reusing one hot connection.
+Selection is **work-conserving**: a command takes the next *available* connection and
+never waits behind a busy one while another connection is free, so an occasional slow
+inference cannot stall commands that another connection could serve. A released
+connection returns to the tail of the free queue, so sequential commands still rotate
+in order.
 
 `url` also accepts an **array of instances** (interchangeable replicas serving the same
 model set); commands then round-robin across instances first, then across connections
@@ -126,7 +131,7 @@ and queue it there even while other instances sit idle. Round-robin spreading ac
 **at least your expected emb instance count** (e.g. `pool: 10` for 10 instances).
 
 Commands beyond the pool's parallelism do **not** time out: when `pool` commands are
-already in flight, later commands wait on the shared connections until one frees
+already in flight, later commands wait for the first connection to free
 (there is no checkout timeout — unlike the previous pool's 5s
 `ConnectionPool::TimeoutError`). Only the wait for a free pool connection is
 unbounded — once a command runs, `RedisClient`'s `connect_timeout`,
@@ -138,9 +143,10 @@ pool size — every command reaches the same server. The pool size still control
 how many persistent connections the client holds and how many commands run in
 parallel. The pool is usually not the bottleneck for
 inference-bound workloads (small pools are fine); it becomes a knob at high
-concurrency on a multi-model box — see [Performance](#performance). With
-round-robin selection up to `pool` commands run in parallel; beyond that, commands
-share connections and serialize on them.
+concurrency on a multi-model box — see [Performance](#performance). Up to `pool`
+commands run in parallel; beyond that, commands wait for the first connection to
+free rather than blocking on a specific busy one, so size `pool` to your expected
+concurrency.
 
 ### Authentication
 

data/lib/emb/round_robin_pool.rb

@@ -1,17 +1,22 @@
 # frozen_string_literal: true
 
 module Emb
-  # A thread-safe pool of N RedisClient connections with round-robin selection.
-  # Behind connection-level load balancers (AWS Service Connect, an NLB, or an
-  # Envoy TCP proxy) each keep-alive connection is pinned to one upstream
-  # instance, so rotating commands across the pool spreads traffic across every
-  # instance — even single-threaded at zero concurrency. Connections are created
-  # up front but connect lazily on first use.
+  # A thread-safe pool of N RedisClient connections with work-conserving
+  # round-robin selection. Behind connection-level load balancers (AWS Service
+  # Connect, an NLB, or an Envoy TCP proxy) each keep-alive connection is pinned
+  # to one upstream instance, so rotating commands across the pool spreads
+  # traffic across every instance — even single-threaded at zero concurrency.
+  # Connections are created up front but connect lazily on first use.
+  #
+  # Selection is work-conserving: a command takes the next *available*
+  # connection, never waiting behind a busy connection while another is free.
+  # A released connection returns to the tail of the free queue, so sequential
+  # commands still rotate in order.
   #
   # Two behaviors deliberately match the connection_pool gem it replaces: a
   # nested `with` from the same thread re-enters the held connection, and after
   # `fork` (Puma preload_app, unicorn, resque) the pool closes inherited sockets
-  # and rebuilds its mutexes in the child so parent and child never share a
+  # and rebuilds its free queue in the child so parent and child never share a
   # connection.
   class RoundRobinPool
     # Pools are tracked only to reset them in forked children. WeakMap so a
@@ -33,31 +38,30 @@ module Emb
 
       @size = size
       @connections = Array.new(size, &)
-      @locks = Array.new(size) { Mutex.new }
-      @next = 0
-      @index_mutex = Mutex.new
+      @free = free_queue
       INSTANCES&.[]=(self, self)
     end
 
-    # Yields the next connection in rotation order. Safe from multiple threads:
-    # up to `size` commands run in parallel, each on its own connection. A
-    # nested `with` from the same thread re-enters the connection this pool
-    # already holds without re-locking; other pools are unaffected.
+    # Yields an available connection. Safe from multiple threads: up to `size`
+    # commands run in parallel, each on its own connection; a command beyond that
+    # waits for the first connection to free. A nested `with` from the same
+    # thread re-enters the connection this pool already holds; other pools are
+    # unaffected.
     def with(&)
       held = Thread.current[THREAD_KEY]
       if held&.key?(self)
         yield @connections[held[self]]
       else
-        take(held, &)
+        acquire(held, &)
       end
     end
 
-    # Child side of after_fork(): drop inherited sockets and sync state.
+    # Child side of after_fork(): drop inherited sockets and rebuild the free
+    # queue, keeping the connection this thread still holds out of it (its
+    # holder re-adds it on release).
     def reload_after_fork!
       @connections.each { |conn| conn.close if conn.respond_to?(:close) }
-      @locks = Array.new(@size) { Mutex.new }
-      @next = 0
-      @index_mutex = Mutex.new
+      @free = free_queue(exclude: Thread.current[THREAD_KEY]&.[](self))
     end
 
     if Process.respond_to?(:fork)
@@ -74,26 +78,30 @@ module Emb
 
     private
 
-    # Acquires the next connection, records it as held by this thread/pool, and
-    # releases both on exit — even when the block raises.
-    def take(held)
-      idx, connection = pick
-      held ||= {}
-      Thread.current[THREAD_KEY] = held
-      held[self] = idx
-      begin
-        @locks[idx].synchronize { yield connection }
-      ensure
-        held.delete(self)
-        Thread.current[THREAD_KEY] = nil if held.empty?
-      end
+    def free_queue(exclude: nil)
+      free = Queue.new
+      @size.times { |idx| free << idx unless idx == exclude }
+      free
     end
 
-    def pick
-      @index_mutex.synchronize do
-        idx = @next % @size
-        @next += 1
-        [idx, @connections[idx]]
+    # Takes the next free connection, records it as held by this thread/pool,
+    # and returns it to the queue on exit — even when the block raises.
+    def acquire(held)
+      idx = nil
+      begin
+        idx = @free.pop
+        held ||= {}
+        Thread.current[THREAD_KEY] = held
+        held[self] = idx
+        yield @connections[idx]
+      ensure
+        # idx stays nil when no connection was taken; otherwise the taken index
+        # is returned even on async interrupt.
+        if idx
+          held&.delete(self)
+          Thread.current[THREAD_KEY] = nil if held&.empty?
+          @free << idx
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: emb
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.3.pre1
 platform: ruby
 authors:
 - elcuervo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-06 00:00:00.000000000 Z
+date: 2026-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: batch-loader