@adhd/sox-embedding-provider 0.3.0 → 0.4.1

package/dist/sharedFastembedProcess.js

@@ -15,10 +15,11 @@
  * process alive) but forks a child **process** instead of constructing a
  * `worker_threads.Worker`.
  */
-import { fork } from 'node:child_process';
+import { fork, execSync } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { existsSync, readFileSync } from 'node:fs';
+import * as os from 'node:os';
 import { performance } from 'node:perf_hooks';
 import { log } from '@adhd/sox-telemetry';
 import { resolveFastembedLockPath } from './fastembedLock.js';
@@ -76,28 +77,169 @@ function isPidAlive(pid) {
  * missing/unreadable/stale lock file is silently treated as "no competing
  * host", never thrown — this must never be able to break or slow a real
  * embed call.
+ *
+ * (DEBT-EPIC-HOTPATH-REDUNDANT-IO-001) This used to be an UNCONDITIONAL
+ * `existsSync` + `readFileSync` pair on EVERY call to `request()` below — i.e.
+ * on every single embed, since `embedSingle`/`embedBatch` route through it.
+ * Measured (Node 20, warm fs cache, 5000 iterations against a real lock
+ * file): ~9.6us/call synchronously blocking the event loop each time — small
+ * relative to a ~619ms warm bge-base embed, but it is a real, avoidable
+ * per-call syscall pair on the single hottest path in the write pipeline, and
+ * a sync fs call blocks OTHER concurrent work in the same process regardless
+ * of its own cost (the mechanism argument the epic calls out). The value this
+ * reads — whether a second fastembed host process is alive — cannot change on
+ * a sub-second cadence (it only flips at process start/exit), so a short TTL
+ * cache removes the per-call I/O without weakening the signal: a genuinely
+ * competing host is still detected and logged, just at worst
+ * `COMPETING_HOST_CACHE_TTL_MS` later than before.
+ */
+const COMPETING_HOST_CACHE_TTL_MS = 3000;
+let _competingHostCache = null;
+let _competingHostCacheOwnPid;
+let _competingHostCacheOwnPoolGroup;
+let _competingHostCacheAt = -Infinity;
+/** TEST-ONLY: clear the TTL cache so a test can force a fresh fs read. */
+export function __resetCompetingHostCacheForTests() {
+    _competingHostCache = null;
+    _competingHostCacheOwnPid = undefined;
+    _competingHostCacheOwnPoolGroup = undefined;
+    _competingHostCacheAt = -Infinity;
+}
+/**
+ * Exported for direct unit testing (DEBT-EPIC-HOTPATH-REDUNDANT-IO-001) — see the
+ * TTL-cache doc comment above for why this used to be a per-call sync fs read.
+ *
+ * (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) `ownPoolGroup`, when supplied,
+ * suppresses a "competing host" result whose lock entry carries the SAME
+ * `poolGroup` — i.e. another member of this client's own `FastembedProcessPool`,
+ * not a genuinely unrelated fastembed host. Without this, the BL-432
+ * `competing_host_pid` telemetry field would read as permanently "contended"
+ * for every pooled request, which is exactly the false-positive noise BL-331's
+ * own postmortem warns against trusting.
  */
-function detectCompetingFastembedHost(ownPid) {
+export function detectCompetingFastembedHost(ownPid, ownPoolGroup) {
+    const now = performance.now();
+    if (now - _competingHostCacheAt < COMPETING_HOST_CACHE_TTL_MS &&
+        _competingHostCacheOwnPid === ownPid &&
+        _competingHostCacheOwnPoolGroup === ownPoolGroup) {
+        return _competingHostCache;
+    }
+    _competingHostCacheAt = now;
+    _competingHostCacheOwnPid = ownPid;
+    _competingHostCacheOwnPoolGroup = ownPoolGroup;
     try {
         const lockPath = resolveFastembedLockPath();
-        if (!existsSync(lockPath))
+        if (!existsSync(lockPath)) {
+            _competingHostCache = null;
             return null;
+        }
         const raw = JSON.parse(readFileSync(lockPath, 'utf8'));
         const pid = typeof raw.pid === 'number' ? raw.pid : null;
-        if (pid === null || pid === ownPid || !isPidAlive(pid))
+        const isKnownPoolSibling = typeof raw.poolGroup === 'string' && ownPoolGroup !== undefined && raw.poolGroup === ownPoolGroup;
+        if (pid === null || pid === ownPid || !isPidAlive(pid) || isKnownPoolSibling) {
+            _competingHostCache = null;
             return null;
-        return { pid, startedAt: typeof raw.startedAt === 'string' ? raw.startedAt : 'unknown' };
+        }
+        _competingHostCache = { pid, startedAt: typeof raw.startedAt === 'string' ? raw.startedAt : 'unknown' };
+        return _competingHostCache;
     }
     catch {
+        _competingHostCache = null;
         return null;
     }
 }
+/**
+ * (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Thrown by `FastembedProcessPool
+ * .request()` when every pool member already has `admissionLimit` or more
+ * requests in flight — the "a queue that never refuses hides its own
+ * failure" backstop the bug item asked for. This is NOT the primary fix (the
+ * pool itself is — see the class below); it exists purely so a burst that
+ * genuinely exceeds total pool capacity gets a fast, typed, retryable
+ * rejection instead of silently joining a multi-minute queue. Named
+ * `TransientEmbeddingError`-shaped (has `retryAfterMs`) but defined locally
+ * rather than importing from `./index.js`, to keep this module free of a
+ * dependency on the public interface skeleton — callers can duck-type on
+ * `.name === 'FastembedBusyError'` or `instanceof FastembedBusyError`.
+ */
+export class FastembedBusyError extends Error {
+    retryAfterMs;
+    constructor(message, retryAfterMs) {
+        super(message);
+        this.name = 'FastembedBusyError';
+        this.retryAfterMs = retryAfterMs;
+    }
+}
 export class SharedFastembedProcessClient {
     child = null;
     startingPromise = null;
+    /**
+     * (BUG-MEMORYSERVER-SHUTDOWN-LEAKS-FASTEMBED-CHILD-001) True once
+     * `terminate()` has been called on THIS client — permanent, never reset.
+     * Distinct from an unexpected crash (the `c.on('exit')`/`c.on('error')`
+     * handlers below null `this.child` WITHOUT setting this flag, so BUG-021's
+     * transparent-respawn-and-reinit behaviour for a genuinely crashed child is
+     * completely unaffected).
+     *
+     * ROOT CAUSE this closes: `fastembed.ts`'s `initModel()` retries a
+     * cache-hit warmup up to `WARMUP_CACHE_HIT_ATTEMPTS` times, each attempt
+     * calling `this.shared.request({ type: 'init', ... })`. `coordinatedShutdown`
+     * (`memory-server/src/backend.ts`) races step 0 (`flushPendingEmbeds`)
+     * against a short bound and, on timeout, ABANDONS it as a background
+     * "loser" promise that keeps running concurrently with step 1
+     * (`terminateEmbedWorkers`). If that abandoned promise's in-flight
+     * `initModel()` attempt is still on its first attempt when step 1's
+     * `terminate()` below nulls `this.child`/`this.startingPromise` and rejects
+     * every pending request (including that attempt), `initModel()`'s catch
+     * block loops and fires attempt 2 — which, before this fix, called
+     * `ensureProcess()` and found `this.child === null`, so it happily forked a
+     * BRAND NEW, completely untracked child process. That child was never
+     * subject to any further termination step (`coordinatedShutdown` had
+     * already moved past step 1) and was abandoned as a live orphan the instant
+     * `exit(0)` fired — reproduced directly: a real backend forked under
+     * `SOX_PROXY_BACKEND=1`, given one `memory_write` (to start Phase-B's async
+     * embed), then SIGTERM'd immediately — server pid exited cleanly at
+     * t+694ms, but a SECOND `fastembedProcessHost.js` child (forked ~150ms
+     * AFTER the coordinated shutdown's own step 4 already finished) was still
+     * alive seconds later, orphaned under the process's original parent.
+     *
+     * `ensureProcess()` now refuses to fork once this flag is set — any request
+     * racing a real `terminate()` fails fast with a clear error instead of
+     * spawning a child nothing will ever reap.
+     */
+    terminated = false;
     nextId = 1;
     pending = new Map();
     hostPath;
+    poolGroup;
+    poolSize;
+    memberIndex;
+    /**
+     * (BUG-021) The last `{ type: 'init', model, cacheDir }` payload this
+     * member has ever been asked to load — kept independently of any pool
+     * wrapper above it so a LONE (non-pooled) client is self-healing too.
+     * `null` until the first successful `init`.
+     */
+    lastInitPayload = null;
+    /**
+     * (BUG-021 root cause) True once the CURRENT `this.child` has actually
+     * loaded the model named by `lastInitPayload`. Reset to `false` every time
+     * `ensureProcess()` forks a NEW child — including a transparent respawn
+     * after the previous child died (crash, OOM-kill, the BL-426/std::bad_alloc
+     * native-teardown hazards documented on `fastembedProcessHost.ts`, etc).
+     *
+     * Before this fix, `ensureProcess()`'s `c.on('exit')`/`c.on('error')`
+     * handlers nulled out `this.child`/`this.startingPromise` on an unexpected
+     * death, but nothing told the NEXT `request()` call that the freshly
+     * (re)forked replacement child has an empty `_embedder` — every real
+     * `embed`/`embedBatch` request sent to it failed with "Model not
+     * initialized" forever, because the higher-level `FastembedProvider.ready`
+     * flag was already latched `true` from the original (pre-crash) init and
+     * `ensureReady()` never re-sends `init` once `ready` is true. Live incident
+     * BUG-021: the lock file showed a child forked mere SECONDS before every
+     * request against it failed — exactly this respawn-without-reinit gap, not
+     * a stale/orphaned lock as first suspected.
+     */
+    childInitialized = false;
     /**
      * @param hostPathOverride Test-only injection point (BL-410): points the
      * fork target at a lightweight fixture host instead of the real
@@ -105,16 +247,70 @@ export class SharedFastembedProcessClient {
      * contract without loading fastembed or downloading a model. Production
      * code (`getSharedFastembedProcess()`) never passes this — it always
      * resolves the real host path.
+     * @param poolGroup (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Set by
+     * `FastembedProcessPool` to a group id shared by every member of that pool.
+     * Forwarded to the forked child via `SOX_FASTEMBED_POOL_GROUP` so the
+     * BL-331 advisory lock can tell "another member of my own pool" apart from
+     * "a genuinely unrelated fastembed host" — see `fastembedLock.ts`'s
+     * `poolGroup` doc comment. `undefined` for a lone (non-pooled) client,
+     * which preserves today's exact single-host lock behaviour there.
+     * @param poolSize / @param memberIndex (BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001,
+     * observability addendum) Set by `FastembedProcessPool` so every
+     * `fastembed_process.request.*` telemetry record this client emits carries
+     * the pool's actual size and this member's index alongside `queue_depth`/
+     * `response_ms`. Diagnosing THIS incident required an agent to read source
+     * and reconstruct `resolveFastembedPoolSize()`'s arithmetic by hand,
+     * because no telemetry field ever recorded what size the pool actually
+     * resolved to at runtime — `queue_depth` alone cannot distinguish
+     * "contention on an inert 1-member pool" from "genuine over-capacity on a
+     * 4-member pool". `undefined` for a lone (non-pooled) client, which omits
+     * both fields from telemetry exactly as before this addendum.
      */
-    constructor(hostPathOverride) {
+    constructor(hostPathOverride, poolGroup, poolSize, memberIndex) {
         this.hostPath = hostPathOverride;
+        this.poolGroup = poolGroup;
+        this.poolSize = poolSize;
+        this.memberIndex = memberIndex;
     }
     /** True once the underlying child process has been forked. */
     get started() {
         return this.child !== null;
     }
+    /**
+     * (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Number of requests already
+     * admitted-and-unsettled on this specific child, i.e. how many are ahead
+     * of a hypothetical next request. Exposed read-only so `FastembedProcessPool`
+     * (below) can route a new request to whichever pool member is least loaded
+     * — the exact `this.pending.size` value `request()` already measures
+     * internally for the `queue_depth` telemetry field, just made visible to a
+     * caller one level up instead of re-derived.
+     */
+    get pendingCount() {
+        return this.pending.size;
+    }
+    /**
+     * (BL-575) Update the `pool_size`/`member_index` telemetry fields this
+     * client stamps on every `fastembed_process.request.*` record. Package-
+     * private (no `readonly`) specifically so `AdaptiveFastembedProcessPool`
+     * can keep them truthful as the pool grows/shrinks at runtime — a FIXED
+     * `FastembedProcessPool` never calls this (its size is constant for the
+     * member's whole lifetime, set once at construction).
+     */
+    setPoolMeta(poolSize, memberIndex) {
+        this.poolSize = poolSize;
+        this.memberIndex = memberIndex;
+    }
     /** Lazily fork (exactly once) and return the single shared child process. */
     ensureProcess() {
+        // (BUG-MEMORYSERVER-SHUTDOWN-LEAKS-FASTEMBED-CHILD-001) A caller racing a
+        // real `terminate()` must never re-fork a child no one will ever reap —
+        // see `terminated`'s own doc comment for the exact reproduction. Checked
+        // FIRST, before the `this.child`/`this.startingPromise` fast paths, so it
+        // also covers the narrow window where `terminate()` has set this flag but
+        // a stale (about-to-be-killed) `this.child` reference is still non-null.
+        if (this.terminated) {
+            return Promise.reject(new Error('shared fastembed process terminated'));
+        }
         if (this.child)
             return Promise.resolve(this.child);
         if (this.startingPromise)
@@ -126,6 +322,9 @@ export class SharedFastembedProcessClient {
                 // Real inference is CPU-bound in native code; no need to keep the
                 // parent process alive on this child's account.
                 detached: false,
+                ...(this.poolGroup !== undefined
+                    ? { env: { ...process.env, SOX_FASTEMBED_POOL_GROUP: this.poolGroup } }
+                    : {}),
             });
             c.unref();
             c.on('message', (msg) => {
@@ -147,6 +346,12 @@ export class SharedFastembedProcessClient {
                 this.pending.clear();
                 this.child = null;
                 this.startingPromise = null;
+                // (BUG-021) A dead child never reappears un-loaded — the NEXT
+                // ensureProcess() forks a genuinely new one, which starts this flag
+                // at false again anyway, but clearing it here too closes any window
+                // where a caller reads `childInitialized` between the crash and the
+                // next fork.
+                this.childInitialized = false;
             });
             c.on('exit', (code) => {
                 if (code !== 0 && code !== null) {
@@ -157,6 +362,7 @@ export class SharedFastembedProcessClient {
                 }
                 this.child = null;
                 this.startingPromise = null;
+                this.childInitialized = false;
             });
             // Same re-unref pattern as `sharedOnnxWorker.ts` — attaching listeners
             // can re-ref the underlying handle; re-assert `unref()` once every
@@ -234,15 +440,63 @@ export class SharedFastembedProcessClient {
      *   3. `competing_host_pid` — present only when a second, still-live
      *      `fastembedProcessHost` process is detected (BL-331's advisory lock),
      *      since that changes embed latency 25-50x independent of queueing.
+     *
+     * BL-576: `signal`, when supplied, is an external cancellation source —
+     * e.g. `operation-guard.ts`'s `withOperationDeadline` abort controller, so
+     * a deadline that fires while an embed is queued behind an unrelated slow
+     * request stops WAITING on it immediately rather than riding the deadline
+     * out via its own separate mechanism. This does NOT kill the underlying
+     * fastembed child mid-inference (the same "no native cancellation
+     * primitive" constraint `operation-guard.ts` documents applies here too —
+     * the child process is shared across other pending requests, so killing
+     * it on one caller's abort would collaterally fail every other in-flight
+     * request on that member) — it settles the CALLER's promise immediately
+     * with an `AbortError` and removes the pending entry so a late reply from
+     * the child (once the real work finishes) is silently dropped instead of
+     * resolving/rejecting a promise nobody is awaiting anymore. If `signal`
+     * is already aborted when `request()` is called, it rejects immediately
+     * without ever sending to the child.
      */
-    async request(payload, timeoutMs) {
-        const child = await this.ensureProcess();
+    async request(payload, timeoutMs, signal) {
+        if (signal?.aborted) {
+            throw signal.reason instanceof Error
+                ? signal.reason
+                : new Error('shared fastembed process request aborted before it was sent');
+        }
+        const isInitRequest = payload['type'] === 'init';
+        if (isInitRequest) {
+            this.lastInitPayload = payload;
+        }
+        let child = await this.ensureProcess();
+        if (!isInitRequest && this.lastInitPayload && !this.childInitialized) {
+            // (BUG-021 fix) `child` was just (re)spawned — by this call's own
+            // `ensureProcess()` above, OR by an earlier request on this member
+            // after the previous child died unexpectedly — and has never loaded a
+            // model. Replay the last known-good `init` payload before letting the
+            // real request through, exactly mirroring what `FastembedProcessPool`/
+            // `AdaptiveFastembedProcessPool` already do for a brand-new member on
+            // grow, but here for the respawn-in-place case those pools never
+            // detect (they only (re)send `init` when THEY create a member; they
+            // have no visibility into this client transparently re-forking its own
+            // dead child). Recurses through this exact method so the replay gets
+            // identical telemetry/timeout/admission treatment to any other init.
+            await this.request(this.lastInitPayload, timeoutMs, signal);
+            // Re-fetch: `ensureProcess()` is idempotent for a live child, but never
+            // trust a reference captured before an `await` against a process that
+            // can die out from under it.
+            child = await this.ensureProcess();
+        }
         const id = this.nextId++;
         const queueDepth = this.pending.size;
-        const competing = detectCompetingFastembedHost(child.pid);
+        const competing = detectCompetingFastembedHost(child.pid, this.poolGroup);
         const baseFields = {
             queue_depth: queueDepth,
             ...(competing ? { competing_host_pid: competing.pid } : {}),
+            // (BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001) See the constructor's
+            // `poolSize`/`memberIndex` doc comment: makes "was this process even
+            // pooled, and at what size" a directly observable telemetry field
+            // instead of something an agent has to re-derive from source.
+            ...(this.poolSize !== undefined ? { pool_size: this.poolSize, member_index: this.memberIndex } : {}),
         };
         return new Promise((resolve, reject) => {
             let to;
@@ -265,10 +519,24 @@ export class SharedFastembedProcessClient {
                 if (typeof to.unref === 'function')
                     to.unref();
             }
+            let onAbort;
+            const detachAbort = () => {
+                if (onAbort && signal)
+                    signal.removeEventListener('abort', onAbort);
+            };
             this.pending.set(id, {
                 resolve: (v) => {
                     if (to)
                         clearTimeout(to);
+                    detachAbort();
+                    // (BUG-021) Only a genuinely SUCCESSFUL init reply proves this
+                    // child now has a loaded model — flip the flag here, not
+                    // optimistically at send time, so a rejected init (model load
+                    // error) correctly leaves `childInitialized` false and the next
+                    // real request replays init again rather than sailing through
+                    // believing a failed load succeeded.
+                    if (isInitRequest)
+                        this.childInitialized = true;
                     log.info('fastembed_process.request.finish', {
                         ...baseFields,
                         response_ms: Math.round(performance.now() - sentAt),
@@ -278,6 +546,7 @@ export class SharedFastembedProcessClient {
                 reject: (e) => {
                     if (to)
                         clearTimeout(to);
+                    detachAbort();
                     log.warn('fastembed_process.request.error', {
                         ...baseFields,
                         response_ms: Math.round(performance.now() - sentAt),
@@ -286,6 +555,32 @@ export class SharedFastembedProcessClient {
                     reject(e);
                 },
             });
+            // BL-576: an external abort (e.g. operation-guard.ts's deadline)
+            // settles the CALLER's promise immediately — via the SAME pending-map
+            // removal + `unrefIfIdle()` path the internal `timeoutMs` timer above
+            // already uses — without waiting for `timeoutMs` (which may be unset,
+            // or longer than the caller's own deadline) and without touching the
+            // child process itself: the real work, if the child is still grinding
+            // on it, keeps running and its late reply (if any) is silently
+            // dropped by the `c.on('message')` handler above (`pending.get(msg.id)`
+            // returns undefined once this entry is deleted).
+            if (signal) {
+                onAbort = () => {
+                    if (!this.pending.has(id))
+                        return;
+                    this.pending.delete(id);
+                    this.unrefIfIdle();
+                    if (to)
+                        clearTimeout(to);
+                    log.warn('fastembed_process.request.aborted', {
+                        ...baseFields,
+                        response_ms: Math.round(performance.now() - sentAt),
+                        reason: signal.reason instanceof Error ? signal.reason.message : String(signal.reason),
+                    });
+                    reject(signal.reason instanceof Error ? signal.reason : new Error('shared fastembed process request aborted'));
+                };
+                signal.addEventListener('abort', onAbort, { once: true });
+            }
             // BL-410: keep the parent's event loop ref'd until this request settles.
             this.refForPending();
             log.info('fastembed_process.request.admitted', baseFields);
@@ -309,13 +604,34 @@ export class SharedFastembedProcessClient {
      * shutdown sequence waiting on a child that will never exit gracefully.
      */
     async terminate() {
+        // (BUG-MEMORYSERVER-SHUTDOWN-LEAKS-FASTEMBED-CHILD-001) Set FIRST and
+        // synchronously — before anything else in this method runs — so any
+        // `ensureProcess()` call racing this `terminate()` (including one already
+        // past its own `if (this.child)`/`if (this.startingPromise)` fast-path
+        // checks in the SAME microtask, which cannot happen after this
+        // synchronous write) observes `terminated` and refuses to fork a new,
+        // unreapable child. See `terminated`'s own doc comment for the exact
+        // reproduction this closes.
+        this.terminated = true;
         const c = this.child;
+        // A fork that was ALREADY in flight (started by an `ensureProcess()` call
+        // that ran to completion of its `fork()` before this `terminate()` set
+        // the flag above) has no `this.child` yet — `c` above would miss it
+        // entirely. Capture and await `startingPromise` too so that in-flight
+        // fork is killed once it resolves, instead of completing unobserved and
+        // orphaning exactly like the bug this fixes.
+        const starting = this.startingPromise;
         this.child = null;
         this.startingPromise = null;
         for (const { reject } of this.pending.values()) {
             reject(new Error('shared fastembed process terminated'));
         }
         this.pending.clear();
+        if (c === null && starting) {
+            void starting.then((lateChild) => {
+                lateChild.kill();
+            }).catch(() => undefined);
+        }
         if (!c)
             return;
         const exited = new Promise((resolve) => {
@@ -325,8 +641,13 @@ export class SharedFastembedProcessClient {
             if (c.connected)
                 c.send({ __shutdown: true });
         }
-        catch {
-            // IPC already gone — kill() below is the only path left.
+        catch (err) {
+            // IPC already gone — kill() below is the only path left. Deliberately silent:
+            // process.send() throws EPIPE if the IPC channel is gone; this is the
+            // expected path when the child exits without a graceful __shutdown message.
+            log.debug('embedding_provider.fastembed.terminate.send_failed', {
+                reason: err instanceof Error ? err.message : 'IPC channel unavailable',
+            });
         }
         const timedOut = await Promise.race([
             exited.then(() => false),
@@ -342,6 +663,664 @@ export class SharedFastembedProcessClient {
 }
 /** (BL-405) How long `terminate()` waits for the graceful `__shutdown` message before falling back to `kill()`. */
 const TERMINATE_GRACE_MS = 1000;
+// ── FastembedProcessPool (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) ───────
+//
+// PRODUCTION MEASUREMENT (n=3769, 11 days, from the bug report this fixes):
+//
+//   response_ms   p50=1011ms   p90=5674ms   p99=40518ms   max=109089ms
+//   qdepth 0      n=1335   p50=534ms     p90=1501ms
+//   qdepth 1-2    n=1905   p50=1164ms    p90=4367ms
+//   qdepth 3-5    n=354    p50=2821ms    p90=15441ms
+//   qdepth 6-10   n=105    p50=5519ms    p90=18862ms
+//   qdepth 11+    n=70     p50=26723ms   p90=76712ms
+//
+// Root cause: ALL embed requests in a process share ONE `SharedFastembedProcessClient`
+// (`getSharedFastembedProcess()`), which forks exactly ONE child process, whose
+// `fastembedProcessHost.ts` processes requests through a single serialized
+// `_queue` promise chain (`enqueue()`). Every request beyond the first-in-flight
+// waits for every earlier one to fully finish — classic head-of-line blocking,
+// and it is why the slope above is so clean: `response_ms` scales almost
+// linearly with `queue_depth`.
+//
+// The single-child-PROCESS design is NOT what's wrong — it is the proven fix
+// for BL-238 (fastembed's onnxruntime-node@1.21.0 cannot share a THREAD, or
+// even sequential same-thread loading, with transformers.js's onnxruntime-node
+// @1.24.3; see the file-header comment in `fastembedProcessHost.ts`). What's
+// wrong is that there is only ONE such child. A POOL of N independent child
+// PROCESSES preserves the exact isolation property BL-238 requires (each pool
+// member is its own OS process — the hazard classes BL-238 documents are both
+// structurally about SHARING a thread/address space, never about how many
+// separate processes exist) while removing the forced serialization across
+// members: N members can each process one request at a time, so N requests
+// run concurrently instead of 1.
+//
+// Routing: least-loaded-member (by `pendingCount`), not round-robin — under
+// bursty arrival (the production shape: several agents writing at once) a
+// least-loaded pick keeps the queue-depth distribution across members far
+// tighter than round-robin, which can stack several slow requests on the same
+// member by bad luck.
+//
+// Admission control: once EVERY member already has `admissionLimit` or more
+// requests in flight, `request()` for a non-init payload throws
+// `FastembedBusyError` instead of enqueuing — "a queue that never refuses is
+// a queue that hides its own failure" (bug item, direction #2). The default
+// (`Number.POSITIVE_INFINITY`, i.e. disabled) preserves today's "always admit"
+// behaviour; ops can opt in via `SOX_EMBED_POOL_ADMISSION_LIMIT` once the pool
+// alone is proven sufficient in production (see the benchmark harness results
+// cited in this bug's resolution notes — a 4-member pool absorbs the
+// qdepth-11+ regime the production telemetry measured with room to spare, so
+// admission control is shipped as an available backstop, not defaulted on).
+//
+// Batch coalescing (bug item direction #3) was evaluated and rejected for
+// this pass: `embedBatch` already exists and batches WITHIN one caller's
+// texts; merging batches ACROSS independent concurrent callers would require
+// buffering/deadline logic (wait up to Xms for more arrivals before sending)
+// that trades a bounded latency floor for a throughput gain the pool already
+// captures via parallelism. The pool is strictly simpler, has no added
+// latency floor, and the measured numbers below show it closes the gap
+// without it.
+//
+// ── Memory cost — MEASURED, not assumed (post-review addendum) ─────────────
+//
+// A pool member is a full onnxruntime-node `InferenceSession`, and the
+// question of whether N of them cost N times the model's on-disk size was
+// open until measured directly: forked 1 and 4 REAL `fastembedProcessHost.js`
+// children loading the production default model (bge-base-en-v1.5, 219MB on
+// disk) and read each child's `footprint` (Apple's private+compressed
+// physical-footprint accounting, which — unlike raw RSS — correctly
+// distinguishes real per-process cost from pages shared/mmap'd across
+// processes):
+//
+//   1 child:  phys_footprint ≈ 261 MB, of which 207 MB is a single
+//             `MALLOC_LARGE` DIRTY (not Clean/Reclaimable) allocation.
+//   4 children: EACH independently shows phys_footprint ≈ 262-271 MB with
+//             its OWN ~207-208 MB MALLOC_LARGE dirty block — no shared/clean
+//             mmap'd region for the model weights at all.
+//
+// Conclusion: onnxruntime-node HEAP-ALLOCATES the model per process; it does
+// NOT mmap it read-only for the OS to share across children. The cost is
+// real and scales linearly with pool size — a 4-member pool for this model
+// costs ~4×265MB ≈ 1.06GB of additional resident memory, not a few hundred
+// extra KB for session arenas. Other configured models range further: per
+// `fastembedModels.ts`'s own descriptions, codexembed-400m is documented at
+// "~1.6GB RAM" per instance — a 4-member pool of that model would be ~6.4GB.
+//
+// This matters because embedding-provider has no way to know at
+// `getSharedFastembedProcess()` call time (before any `init`) which model
+// will be loaded, so pool sizing CANNOT be exact per-model — and the box
+// this was measured on had 128MB of free physical memory at measurement
+// time (`top -l 1`, `PhysMem: 31G used … 128M unused`), out of 32GB total,
+// with 11GB already in the compressor. An unconditional hardware-sized
+// default (the original `floor(cpus/2)`, cap 4) would have shipped a
+// default that, on THIS box alone, tries to allocate ~800MB more than is
+// physically free — risking heavy swap or an OOM kill mid-embed-write, which
+// is exactly the unclean-shutdown shape this subsystem's corruption class
+// feeds on. A latency fix that increases corruption exposure is not a win.
+//
+// So sizing is now BOTH memory-aware (auto-computed from `os.freemem()`
+// against a conservative measured per-member budget, `DEFAULT_PER_MEMBER_MB`
+// below — calibrated to the measured bge-base-en-v1.5 figure with margin, the
+// package's own default model) AND still capped by CPU count, defaulting to
+// the SAFE end when the two disagree. On a memory-constrained box this
+// resolves to pool size 1 — the exact pre-fix single-child topology — rather
+// than silently trying to grab memory that isn't there. `SOX_EMBED_POOL_SIZE`
+// remains an unconditional override for an operator who has measured their
+// own model's real footprint and confirmed headroom; it is honored exactly,
+// with no memory clamp applied (the operator's explicit judgement wins).
+/** Conservative measured per-member memory budget (MB) used only for the
+ *  memory-aware AUTO-sizing path (i.e. when `SOX_EMBED_POOL_SIZE` is not
+ *  set) — calibrated to the measured bge-base-en-v1.5 `phys_footprint`
+ *  (~262-271MB observed) with margin. Models configured with a materially
+ *  different footprint (bge-m3, codexembed-400m — "~1.6GB RAM" per
+ *  `fastembedModels.ts`) are NOT auto-detected here (pool size is resolved
+ *  before any model is known); an operator running one of those should set
+ *  `SOX_EMBED_POOL_PER_CHILD_MB` (or just pin `SOX_EMBED_POOL_SIZE`
+ *  directly) rather than trust this default. */
+const DEFAULT_PER_MEMBER_MB = 300;
+/** Auto-sizing never lets the pool consume the machine's last headroom —
+ *  this many MB of free memory are always left unclaimed by the pool-sizing
+ *  calculation (the rest of the process, and everything else on the
+ *  machine, still needs to run). */
+const MEMORY_SAFETY_MARGIN_MB = 1024;
+/**
+ * (BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001) Estimate REAL available memory
+ * (MB), platform-aware — the input `resolveFastembedPoolSize()`'s
+ * memory-cap arithmetic below actually needs, as opposed to what
+ * `os.freemem()` alone reports.
+ *
+ * `os.freemem()` is not a usable proxy for "memory this process could
+ * actually claim" on macOS: it maps to Mach's raw "free" page count only,
+ * which deliberately EXCLUDES "inactive"/"speculative"/"purgeable" pages —
+ * pages the kernel is using as disk cache but will hand back instantly
+ * (zero swap-in cost) under real pressure. Measured via `vm_stat` on the
+ * exact box this defect was diagnosed on (32GB physical, page size 16384):
+ * `os.freemem()` reported ~299MB while `Pages inactive` ALONE was 345,579
+ * pages (~5.4GB) — the overwhelming majority of genuinely-available memory
+ * was invisible to the metric `resolveFastembedPoolSize()` used to
+ * compute `memoryCap`. Because `MEMORY_SAFETY_MARGIN_MB` (1024) routinely
+ * exceeds `os.freemem()`'s ~300MB-ish reading on macOS regardless of real
+ * load, `memoryCap` collapsed to `Math.max(1, negative) === 1` on every
+ * macOS box, unconditionally — the pool was permanently INERT (silently
+ * behaving exactly like the pre-fix single-child topology) unless an
+ * operator manually overrode `SOX_EMBED_POOL_SIZE`. `hol-pool-sizing.spec.ts`
+ * even self-documents "this suite was itself first run on a box with only
+ * ~128MB free" — the sizing design was validated against the very macOS
+ * quirk that made it universally wrong, not a genuinely memory-constrained
+ * machine.
+ *
+ * Fix: compute "available" as free + inactive + speculative + purgeable
+ * pages (the standard macOS "reclaimable without swapping" heuristic —
+ * matches what `htop`-family tools approximate; NOT Apple's undocumented
+ * memory-pressure internals, which have no public API). On Linux,
+ * `/proc/meminfo`'s `MemAvailable` is already the kernel's own equivalent
+ * estimate and is used directly. Any other platform, or any parse/exec
+ * failure, falls back to `os.freemem()` unchanged — this must never throw
+ * or block pool sizing on a shell-out failing; it runs once per process
+ * (at `getSharedFastembedProcess()` construction), never per-request.
+ */
+export function estimateAvailableMemMb() {
+    try {
+        if (process.platform === 'darwin') {
+            const out = execSync('vm_stat', { encoding: 'utf8', timeout: 2000 });
+            const pageSizeMatch = /page size of (\d+) bytes/.exec(out);
+            const pageSize = pageSizeMatch ? Number(pageSizeMatch[1]) : 4096;
+            const pages = (label) => {
+                const m = new RegExp(`${label}:\\s+(\\d+)\\.`).exec(out);
+                return m ? Number(m[1]) : 0;
+            };
+            const availablePages = pages('Pages free') + pages('Pages inactive') + pages('Pages speculative') + pages('Pages purgeable');
+            const availableMb = (availablePages * pageSize) / (1024 * 1024);
+            if (Number.isFinite(availableMb) && availableMb > 0)
+                return availableMb;
+        }
+        else if (process.platform === 'linux') {
+            const meminfo = readFileSync('/proc/meminfo', 'utf8');
+            const m = /MemAvailable:\s+(\d+)\s+kB/.exec(meminfo);
+            if (m) {
+                const availableMb = Number(m[1]) / 1024;
+                if (Number.isFinite(availableMb) && availableMb > 0)
+                    return availableMb;
+            }
+        }
+    }
+    catch (err) {
+        log.debug('embedding_provider.fastembed.pool_sizing.mem_estimate_failed', {
+            error: err instanceof Error ? err.message : String(err),
+            platform: process.platform,
+        });
+    }
+    return os.freemem() / (1024 * 1024);
+}
+/**
+ * (BL-575) The explicit `SOX_EMBED_POOL_SIZE` override, if set — a HARD PIN.
+ * Split out of `resolveFastembedPoolSize()` so `getSharedFastembedProcess()`
+ * can tell "operator pinned an exact size" (disables ALL adaptation —
+ * `AdaptiveFastembedProcessPool` is never constructed) apart from "no
+ * override, compute a ceiling for adaptive sizing to grow toward" — the two
+ * cases used to be indistinguishable from the return value of a single
+ * function that already folded the CPU/memory-cap arithmetic into the
+ * "no override" branch.
+ */
+export function resolveFastembedPoolPin() {
+    const raw = Number(process.env['SOX_EMBED_POOL_SIZE']);
+    if (Number.isFinite(raw) && raw >= 1)
+        return Math.floor(raw);
+    return null;
+}
+/**
+ * (BL-575) The maximum pool size the memory/CPU budget allows — i.e. what
+ * `resolveFastembedPoolSize()` used to compute unconditionally in its
+ * "no override" branch. Now used two ways: (a) unchanged, as
+ * `resolveFastembedPoolSize()`'s own fallback when no pin is set, so every
+ * existing caller of that function keeps its exact prior behavior; (b) as
+ * `AdaptiveFastembedProcessPool`'s `maxSize` — the ceiling it is allowed to
+ * grow toward under sustained load, never exceeded regardless of how
+ * sustained the demand is.
+ *
+ * Auto-sized from BOTH real available memory (`estimateAvailableMemMb()` —
+ * see its doc comment for why this is NOT simply `os.freemem()`, and
+ * BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001 for the incident this fixes —
+ * against `DEFAULT_PER_MEMBER_MB`, less `MEMORY_SAFETY_MARGIN_MB` headroom —
+ * override the per-member budget via `SOX_EMBED_POOL_PER_CHILD_MB` for a
+ * non-default model) AND CPU count (half the logical CPUs, cap 4 — fastembed
+ * inference is CPU/ANE-bound per request, not embarrassingly parallel across
+ * all cores), taking the SMALLER of the two so a memory-constrained box
+ * never gets sized past what's actually free. See the module doc comment
+ * above for the measurement (footprint/vmmap on 1 vs 4 real children) that
+ * justifies this.
+ *
+ * @param getAvailableMemMb Test-only injection point — production always
+ * uses the default `estimateAvailableMemMb`. Lets tests exercise the sizing
+ * arithmetic against deterministic MB values instead of the real, inherently
+ * machine/moment-dependent OS memory state.
+ */
+export function resolveFastembedPoolCeiling(getAvailableMemMb = estimateAvailableMemMb) {
+    const perMemberMb = Number(process.env['SOX_EMBED_POOL_PER_CHILD_MB']);
+    const memberBudgetMb = Number.isFinite(perMemberMb) && perMemberMb > 0 ? perMemberMb : DEFAULT_PER_MEMBER_MB;
+    const freeMb = getAvailableMemMb();
+    const memoryCap = Math.max(1, Math.floor((freeMb - MEMORY_SAFETY_MARGIN_MB) / memberBudgetMb));
+    const cpus = os.cpus().length || 1;
+    const cpuCap = Math.max(1, Math.min(4, Math.floor(cpus / 2)));
+    return Math.max(1, Math.min(cpuCap, memoryCap));
+}
+/**
+ * Number of independent fastembed child processes a FIXED (non-adaptive)
+ * pool should use. Preserved unchanged for backward compatibility (existing
+ * callers/tests) — `SOX_EMBED_POOL_SIZE` honored exactly if set
+ * (`resolveFastembedPoolPin()`), else the memory/CPU ceiling
+ * (`resolveFastembedPoolCeiling()`). `getSharedFastembedProcess()` itself no
+ * longer calls this for its default (adaptive) path as of BL-575 — see
+ * `resolveFastembedPoolPin`/`resolveFastembedPoolCeiling`'s doc comments.
+ *
+ * @param getAvailableMemMb Test-only injection point, forwarded to
+ * `resolveFastembedPoolCeiling`.
+ */
+export function resolveFastembedPoolSize(getAvailableMemMb = estimateAvailableMemMb) {
+    return resolveFastembedPoolPin() ?? resolveFastembedPoolCeiling(getAvailableMemMb);
+}
+/** Per-member in-flight cap above which `FastembedProcessPool.request()` fast-rejects
+ *  a non-init request with `FastembedBusyError` instead of enqueuing. Disabled
+ *  (`Infinity`) by default — see the pool doc comment above for why. Override
+ *  via `SOX_EMBED_POOL_ADMISSION_LIMIT` (a finite number opts in). */
+export function resolveFastembedAdmissionLimit() {
+    const raw = Number(process.env['SOX_EMBED_POOL_ADMISSION_LIMIT']);
+    if (Number.isFinite(raw) && raw >= 0)
+        return raw;
+    return Number.POSITIVE_INFINITY;
+}
+/**
+ * A pool of `size` independent `SharedFastembedProcessClient`s — i.e. `size`
+ * independent fastembed-hosting OS child processes, each fully isolated from
+ * the others (preserving BL-238's process-isolation requirement) — routing
+ * each request to whichever member currently has the fewest requests in
+ * flight. See the module-level doc comment above for the full rationale and
+ * the production measurements this fixes.
+ */
+export class FastembedProcessPool {
+    members;
+    admissionLimit;
+    /**
+     * (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Routing/admission MUST NOT
+     * read `member.pendingCount` for the decision — that counter is only
+     * incremented deep inside `SharedFastembedProcessClient.request()`, AFTER
+     * its own `await this.ensureProcess()`, i.e. at least one microtask tick
+     * after the call starts. Several callers issued in the same synchronous
+     * burst (`Promise.all(callers.map(() => pool.request(...)))`, the exact
+     * shape a real caller under load produces) all run their OWN synchronous
+     * prefix — including this pool's `leastLoaded()`/admission check — before
+     * ANY of them reaches that tick, so every one of them would read
+     * `pendingCount === 0` for every member and pile onto `members[0]`,
+     * silently defeating both load-balancing and admission control for
+     * exactly the bursty-arrival case this fix exists for. `inFlight` is
+     * incremented SYNCHRONOUSLY the instant a member is chosen (before any
+     * `await`), so the very next synchronous call in the same burst already
+     * sees it.
+     */
+    inFlight;
+    /** Cached so a member added to routing after the first `init` (there is
+     *  none today — pool size is fixed at construction — but kept so a future
+     *  dynamic-resize doesn't silently skip initializing a new member) can be
+     *  brought up to date. Also lets `request()` short-circuit a redundant
+     *  broadcast if the model/cacheDir haven't changed. */
+    lastInitPayload = null;
+    constructor(size, hostPathOverride, admissionLimit = resolveFastembedAdmissionLimit()) {
+        if (!Number.isFinite(size) || size < 1) {
+            throw new Error(`FastembedProcessPool: size must be >= 1, got ${size}`);
+        }
+        // One group id shared by every member — see `fastembedLock.ts`'s
+        // `poolGroup` doc comment: this is what lets the BL-331 advisory lock
+        // recognize a sibling pool member instead of warning about it as an
+        // unrelated competing host.
+        const poolGroup = `pool-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+        this.members = Array.from({ length: size }, (_, i) => new SharedFastembedProcessClient(hostPathOverride, poolGroup, size, i));
+        this.inFlight = new Array(size).fill(0);
+        this.admissionLimit = admissionLimit;
+    }
+    /** True once at least one member has forked its child process. */
+    get started() {
+        return this.members.some((m) => m.started);
+    }
+    /** The last `{ type: 'init', model, cacheDir }` payload broadcast to every
+     *  member, or `null` before the first init. Exposed for introspection/tests
+     *  only — `request()` itself doesn't need to read this back today (pool
+     *  size is fixed at construction, so every member is always initialized by
+     *  the broadcast), it's retained purely so a future dynamic-resize path has
+     *  the payload on hand to bring a newly-added member up to date. */
+    get lastInit() {
+        return this.lastInitPayload;
+    }
+    /** Sum of in-flight requests across every member — the pool-wide
+     *  head-of-line depth an arriving caller would experience. */
+    get pendingCount() {
+        return this.members.reduce((sum, m) => sum + m.pendingCount, 0);
+    }
+    /** Index of the least-loaded member by the SYNCHRONOUS `inFlight` counter
+     *  — see that field's doc comment for why `member.pendingCount` itself is
+     *  unsafe to read here. */
+    leastLoadedIndex() {
+        let bestIdx = 0;
+        for (let i = 1; i < this.inFlight.length; i++) {
+            if (this.inFlight[i] < this.inFlight[bestIdx])
+                bestIdx = i;
+        }
+        return bestIdx;
+    }
+    async request(payload, timeoutMs) {
+        // `init` must reach EVERY member — an embed/embedBatch request can be
+        // routed to ANY member, so every member needs the model loaded before
+        // it can serve one. Broadcasting once (per distinct model/cacheDir) is
+        // cheap relative to a real model load, and members race the load
+        // concurrently rather than serially, so this does not multiply the
+        // warmup budget by pool size.
+        if (payload['type'] === 'init') {
+            this.lastInitPayload = payload;
+            const results = await Promise.all(this.members.map((m) => m.request(payload, timeoutMs)));
+            return results[0];
+        }
+        // Reserve the slot SYNCHRONOUSLY (see `inFlight`'s doc comment) before
+        // any `await` — this is what makes routing/admission correct for a
+        // burst of calls issued in the same microtask (e.g. `Promise.all` over
+        // many concurrent callers), not just for calls staggered by real I/O.
+        const idx = this.leastLoadedIndex();
+        if (this.inFlight[idx] >= this.admissionLimit) {
+            const retryAfterMs = Math.min(2000, 100 * (this.inFlight[idx] + 1));
+            throw new FastembedBusyError(`fastembed pool saturated: every one of ${this.members.length} member(s) already has ` +
+                `>= ${this.admissionLimit} requests in flight`, retryAfterMs);
+        }
+        this.inFlight[idx] += 1;
+        const target = this.members[idx];
+        try {
+            return await target.request(payload, timeoutMs);
+        }
+        finally {
+            this.inFlight[idx] -= 1;
+        }
+    }
+    async terminate() {
+        await Promise.all(this.members.map((m) => m.terminate()));
+    }
+}
+const GROW_QUEUE_RATIO_THRESHOLD = 1.5;
+const GROW_SUSTAIN_COUNT = 3;
+/**
+ * (BL-575 hysteresis fix, post-ship — real-inference sweep by a peer agent,
+ * 2026-08-17) `growSustainCount` alone has NO time dimension:
+ * `avgPendingPerMember` is a function of INSTANTANEOUS concurrency, not
+ * sustained demand. When N callers submit near-simultaneously against a
+ * small pool (e.g. concurrency=8 against size=1), `pendingCount` ramps past
+ * the threshold within MILLISECONDS — "3 consecutive over-threshold
+ * admissions" can all land in the same millisecond, so a single momentary
+ * burst (exactly the case §HYSTERESIS POLICY above says should NOT trigger
+ * a grow) satisfied the count condition every time. Measured: concurrency=8,
+ * n=24 (where the pool should stay at size 1 the whole run per the cited
+ * benchmark) grew 1->2 on 2/2 real trials, making both wall time and p99
+ * WORSE than the correct behavior (wall 7896/6690ms, p99 4749/3959ms vs the
+ * real fixed-pool=1 baseline of ~3041ms). `GROW_COOLDOWN_MS` does not help —
+ * it only gates the SECOND+ grow, not this premature first one.
+ *
+ * Fix: require the over-threshold streak to span at least
+ * `GROW_SUSTAIN_WINDOW_MS` of REAL elapsed time, in addition to the
+ * consecutive-count requirement — a burst of simultaneous submissions that
+ * resolves within milliseconds (the common case at low-moderate concurrency,
+ * where each request completes in low seconds) cannot satisfy both; genuine
+ * sustained backlog (queue depth staying elevated for multiple seconds while
+ * throughput fails to keep up) can. 2000ms is deliberately shorter than a
+ * single solo inference's own typical latency (~2.5-3s measured) — long
+ * enough to rule out "everyone submitted at once", short enough not to blunt
+ * the responsiveness a genuinely overloaded pool needs.
+ */
+const GROW_SUSTAIN_WINDOW_MS = 2_000;
+const GROW_COOLDOWN_MS = 15_000;
+const SHRINK_IDLE_MS = 60_000;
+const SHRINK_CHECK_INTERVAL_MS = 15_000;
+export class AdaptiveFastembedProcessPool {
+    members;
+    inFlight;
+    minSize;
+    maxSize;
+    hostPathOverride;
+    admissionLimit;
+    poolGroup;
+    clock;
+    shrinkTimer;
+    growQueueRatioThreshold;
+    growSustainCount;
+    growSustainWindowMs;
+    growCooldownMs;
+    shrinkIdleMs;
+    lastInitPayload = null;
+    growConsecutiveOverThreshold = 0;
+    /** Wall-clock timestamp the CURRENT over-threshold streak began — `null`
+     *  when not currently in a streak. See `GROW_SUSTAIN_WINDOW_MS`'s doc
+     *  comment for why a count alone is insufficient. */
+    growStreakStartedAt = null;
+    lastGrowAt = -Infinity;
+    idleSinceMs;
+    /** Serializes concurrent grow attempts — `request()` can observe the
+     *  over-threshold condition from several concurrent callers in the same
+     *  burst; only one grow should actually happen. */
+    growInFlight = null;
+    terminated = false;
+    /** Counts of grow/shrink actions taken — exposed for tests/observability,
+     *  not consumed by any routing logic. */
+    growCount = 0;
+    shrinkCount = 0;
+    constructor(opts) {
+        this.minSize = Math.max(1, Math.floor(opts.minSize ?? 1));
+        this.maxSize = Math.max(this.minSize, Math.floor(opts.maxSize));
+        this.hostPathOverride = opts.hostPathOverride;
+        this.admissionLimit = opts.admissionLimit ?? resolveFastembedAdmissionLimit();
+        this.clock = opts.now ?? (() => Date.now());
+        this.growQueueRatioThreshold = opts.growQueueRatioThreshold ?? GROW_QUEUE_RATIO_THRESHOLD;
+        this.growSustainCount = opts.growSustainCount ?? GROW_SUSTAIN_COUNT;
+        this.growSustainWindowMs = opts.growSustainWindowMs ?? GROW_SUSTAIN_WINDOW_MS;
+        this.growCooldownMs = opts.growCooldownMs ?? GROW_COOLDOWN_MS;
+        this.shrinkIdleMs = opts.shrinkIdleMs ?? SHRINK_IDLE_MS;
+        this.poolGroup = `apool-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+        this.members = Array.from({ length: this.minSize }, (_, i) => new SharedFastembedProcessClient(this.hostPathOverride, this.poolGroup, this.minSize, i));
+        this.inFlight = new Array(this.minSize).fill(0);
+        this.idleSinceMs = this.clock();
+        // Unref'd — a periodic shrink-check timer must never keep an otherwise
+        // idle process alive (BL-370's exact failure shape, applied here).
+        this.shrinkTimer = setInterval(() => {
+            this.maybeShrink();
+        }, opts.shrinkCheckIntervalMs ?? SHRINK_CHECK_INTERVAL_MS);
+        if (typeof this.shrinkTimer.unref === 'function')
+            this.shrinkTimer.unref();
+    }
+    get started() {
+        return this.members.some((m) => m.started);
+    }
+    get lastInit() {
+        return this.lastInitPayload;
+    }
+    get pendingCount() {
+        return this.members.reduce((sum, m) => sum + m.pendingCount, 0);
+    }
+    /** Current pool size — exposed for tests/observability. */
+    get size() {
+        return this.members.length;
+    }
+    leastLoadedIndex() {
+        let bestIdx = 0;
+        for (let i = 1; i < this.inFlight.length; i++) {
+            if (this.inFlight[i] < this.inFlight[bestIdx])
+                bestIdx = i;
+        }
+        return bestIdx;
+    }
+    /** Renumber every member's `pool_size`/`member_index` telemetry fields to
+     *  match current pool composition — called after every grow/shrink so
+     *  `fastembed_process.request.*` records stay truthful. */
+    renumberMembers() {
+        for (let i = 0; i < this.members.length; i++) {
+            this.members[i].setPoolMeta(this.members.length, i);
+        }
+    }
+    /**
+     * Attempt to add one member, up to `maxSize`. Best-effort and internally
+     * serialized (`growInFlight`) — safe to call from multiple concurrent
+     * `request()` admissions without double-growing. If a `lastInitPayload`
+     * exists (the pool has already been initialized with a model), the new
+     * member is sent the SAME init payload before being added to routing —
+     * an un-initialized member would fail every real embed request it was
+     * routed to.
+     */
+    async grow() {
+        // (BUG-MEMORYSERVER-SHUTDOWN-LEAKS-FASTEMBED-CHILD-001) A `request()`
+        // admission racing a real `terminate()` must not add a brand-new,
+        // never-terminated member to a pool that has already been told to shut
+        // down — defense in depth alongside the per-member `terminated` guard in
+        // `SharedFastembedProcessClient.ensureProcess()`, which is what actually
+        // stops the fork in the reproduced scenario (a single-member pool, the
+        // default topology); this closes the analogous pool-level growth path.
+        if (this.terminated)
+            return;
+        if (this.growInFlight)
+            return this.growInFlight;
+        if (this.members.length >= this.maxSize)
+            return;
+        this.growInFlight = (async () => {
+            const newIndex = this.members.length;
+            const member = new SharedFastembedProcessClient(this.hostPathOverride, this.poolGroup, newIndex + 1, newIndex);
+            try {
+                if (this.lastInitPayload) {
+                    await member.request(this.lastInitPayload);
+                }
+                this.members.push(member);
+                this.inFlight.push(0);
+                this.renumberMembers();
+                this.growCount += 1;
+                this.lastGrowAt = this.clock();
+                this.growConsecutiveOverThreshold = 0;
+                this.growStreakStartedAt = null;
+                log.info('fastembed_process.pool.grew', {
+                    pool_group: this.poolGroup,
+                    new_size: this.members.length,
+                    max_size: this.maxSize,
+                });
+            }
+            catch (err) {
+                // A failed init (model load error, fork failure) must not corrupt
+                // routing state — the member is simply discarded; the pool stays
+                // at its previous size and can retry growing on a later admission.
+                log.warn('fastembed_process.pool.grow_failed', {
+                    pool_group: this.poolGroup,
+                    attempted_size: newIndex + 1,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+                await member.terminate().catch(() => undefined);
+            }
+        })();
+        try {
+            await this.growInFlight;
+        }
+        finally {
+            this.growInFlight = null;
+        }
+    }
+    /** Periodic shrink check — terminates exactly ONE idle member if the
+     *  entire pool has been continuously idle for `SHRINK_IDLE_MS`. Never
+     *  shrinks below `minSize`. Resets the idle clock after shrinking so a
+     *  demand spike right after only costs one re-grow step, not a full
+     *  rebuild, and so consecutive shrinks are still spaced `SHRINK_IDLE_MS`
+     *  apart (the same conservative cadence, not a rapid drain to `minSize`). */
+    maybeShrink() {
+        if (this.terminated)
+            return;
+        if (this.members.length <= this.minSize)
+            return;
+        if (this.pendingCount > 0) {
+            this.idleSinceMs = null;
+            return;
+        }
+        if (this.idleSinceMs === null) {
+            this.idleSinceMs = this.clock();
+            return;
+        }
+        if (this.clock() - this.idleSinceMs < this.shrinkIdleMs)
+            return;
+        const doomed = this.members.pop();
+        this.inFlight.pop();
+        this.renumberMembers();
+        this.shrinkCount += 1;
+        this.idleSinceMs = this.clock();
+        log.info('fastembed_process.pool.shrank', {
+            pool_group: this.poolGroup,
+            new_size: this.members.length,
+            min_size: this.minSize,
+        });
+        void doomed.terminate().catch((err) => {
+            log.warn('fastembed_process.pool.shrink_terminate_failed', {
+                pool_group: this.poolGroup,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        });
+    }
+    async request(payload, timeoutMs, signal) {
+        if (payload['type'] === 'init') {
+            this.lastInitPayload = payload;
+            const results = await Promise.all(this.members.map((m) => m.request(payload, timeoutMs, signal)));
+            return results[0];
+        }
+        // Any real admission means the pool is not idle right now — cancel any
+        // in-progress idle clock so `maybeShrink()` requires a fresh full
+        // `SHRINK_IDLE_MS` window starting from here.
+        this.idleSinceMs = null;
+        // Evaluate the grow condition BEFORE reserving this request's own slot —
+        // the ratio should reflect backlog that existed independent of this
+        // admission, matching "sustained queue depth", not "this one request
+        // pushed the ratio over the line by itself".
+        //
+        // BL-575 hysteresis fix: a count of consecutive over-threshold
+        // admissions has NO time dimension on its own — see
+        // `GROW_SUSTAIN_WINDOW_MS`'s doc comment for the real-inference trial
+        // that caught this (concurrency=8 grew 1->2 on every run, which the
+        // design explicitly says should never happen). BOTH the count AND a
+        // minimum REAL elapsed span since the streak began are now required.
+        const avgPendingPerMember = this.pendingCount / this.members.length;
+        const now = this.clock();
+        if (avgPendingPerMember >= this.growQueueRatioThreshold) {
+            if (this.growConsecutiveOverThreshold === 0)
+                this.growStreakStartedAt = now;
+            this.growConsecutiveOverThreshold += 1;
+        }
+        else {
+            this.growConsecutiveOverThreshold = 0;
+            this.growStreakStartedAt = null;
+        }
+        const streakSpanMs = this.growStreakStartedAt === null ? 0 : now - this.growStreakStartedAt;
+        if (this.growConsecutiveOverThreshold >= this.growSustainCount &&
+            streakSpanMs >= this.growSustainWindowMs &&
+            this.members.length < this.maxSize &&
+            now - this.lastGrowAt >= this.growCooldownMs &&
+            !this.growInFlight) {
+            // Fire-and-forget: growth must not add its own (model-load-scale)
+            // latency to THIS request, which should be routed to an existing
+            // member right now, not wait for a brand new one to spin up.
+            void this.grow();
+        }
+        const idx = this.leastLoadedIndex();
+        if (this.inFlight[idx] >= this.admissionLimit) {
+            const retryAfterMs = Math.min(2000, 100 * (this.inFlight[idx] + 1));
+            throw new FastembedBusyError(`fastembed pool saturated: every one of ${this.members.length} member(s) already has ` +
+                `>= ${this.admissionLimit} requests in flight`, retryAfterMs);
+        }
+        this.inFlight[idx] += 1;
+        const target = this.members[idx];
+        try {
+            return await target.request(payload, timeoutMs, signal);
+        }
+        finally {
+            this.inFlight[idx] -= 1;
+        }
+    }
+    async terminate() {
+        this.terminated = true;
+        clearInterval(this.shrinkTimer);
+        await Promise.all(this.members.map((m) => m.terminate()));
+    }
+}
 let _singleton = null;
 /**
  * Process-wide singleton accessor — the ONLY sanctioned place a fastembed
@@ -349,10 +1328,37 @@ let _singleton = null;
  * (BL-238/BL-171). Every `FastembedProvider` obtains its handle through this
  * function instead of constructing its own `worker_threads.Worker` or
  * `child_process`.
+ *
+ * (BL-575) Two shapes, chosen by whether `SOX_EMBED_POOL_SIZE` is set:
+ *
+ *   - PINNED (`resolveFastembedPoolPin()` returns non-null): a FIXED
+ *     `FastembedProcessPool` at exactly that size, no adaptation — an
+ *     operator who set this has already judged their model's footprint and
+ *     the box's headroom; adaptive sizing would second-guess that judgement.
+ *   - DEFAULT (no override): an `AdaptiveFastembedProcessPool` starting at
+ *     `minSize: 1` (the topology real measurement shows wins below ~20
+ *     concurrency) and growing toward `resolveFastembedPoolCeiling()` (the
+ *     SAME ceiling `FastembedProcessPool` used to apply unconditionally)
+ *     only under sustained demand — see that class's doc comment for the
+ *     full measurement and hysteresis policy (BUG-MEMORY-EMBED-HEAD-OF-LINE-
+ *     BLOCKING-001's original qdepth-11+ production regime is exactly the
+ *     sustained-demand case this still grows to meet; the difference is it
+ *     no longer pays that pool's cost on every LOWER-concurrency request
+ *     too).
+ *
+ * Every existing caller keeps working unchanged either way — both classes
+ * implement the same `SharedFastembedClient` shape (`request()`/
+ * `terminate()`/`started`). Setting `SOX_EMBED_POOL_SIZE=1` recovers the
+ * exact pre-BL-575 (and pre-pool) single-child topology.
  */
 export function getSharedFastembedProcess() {
-    if (!_singleton)
-        _singleton = new SharedFastembedProcessClient();
+    if (_singleton)
+        return _singleton;
+    const pin = resolveFastembedPoolPin();
+    _singleton =
+        pin !== null
+            ? new FastembedProcessPool(pin)
+            : new AdaptiveFastembedProcessPool({ minSize: 1, maxSize: resolveFastembedPoolCeiling() });
     return _singleton;
 }
 /**