@adhd/sox-embedding-provider 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fastembed.d.ts +4 -4
- package/dist/fastembed.d.ts.map +1 -1
- package/dist/fastembed.js +49 -56
- package/dist/fastembed.js.map +1 -1
- package/dist/fastembedLock.d.ts +15 -0
- package/dist/fastembedLock.d.ts.map +1 -1
- package/dist/fastembedLock.js.map +1 -1
- package/dist/fastembedModels.d.ts +54 -0
- package/dist/fastembedModels.d.ts.map +1 -0
- package/dist/fastembedModels.js +98 -0
- package/dist/fastembedModels.js.map +1 -0
- package/dist/fastembedProcessHost.d.ts.map +1 -1
- package/dist/fastembedProcessHost.js +96 -39
- package/dist/fastembedProcessHost.js.map +1 -1
- package/dist/index.d.ts +46 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +34 -1
- package/dist/index.js.map +1 -1
- package/dist/package.json +2 -2
- package/dist/sharedFastembedProcess.d.ts +399 -4
- package/dist/sharedFastembedProcess.d.ts.map +1 -1
- package/dist/sharedFastembedProcess.js +945 -13
- package/dist/sharedFastembedProcess.js.map +1 -1
- package/package.json +2 -2
|
@@ -15,12 +15,91 @@
|
|
|
15
15
|
* process alive) but forks a child **process** instead of constructing a
|
|
16
16
|
* `worker_threads.Worker`.
|
|
17
17
|
*/
|
|
18
|
-
|
|
18
|
+
/** TEST-ONLY: clear the TTL cache so a test can force a fresh fs read. */
|
|
19
|
+
export declare function __resetCompetingHostCacheForTests(): void;
|
|
20
|
+
/**
|
|
21
|
+
* Exported for direct unit testing (DEBT-EPIC-HOTPATH-REDUNDANT-IO-001) — see the
|
|
22
|
+
* TTL-cache doc comment above for why this used to be a per-call sync fs read.
|
|
23
|
+
*
|
|
24
|
+
* (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) `ownPoolGroup`, when supplied,
|
|
25
|
+
* suppresses a "competing host" result whose lock entry carries the SAME
|
|
26
|
+
* `poolGroup` — i.e. another member of this client's own `FastembedProcessPool`,
|
|
27
|
+
* not a genuinely unrelated fastembed host. Without this, the BL-432
|
|
28
|
+
* `competing_host_pid` telemetry field would read as permanently "contended"
|
|
29
|
+
* for every pooled request, which is exactly the false-positive noise BL-331's
|
|
30
|
+
* own postmortem warns against trusting.
|
|
31
|
+
*/
|
|
32
|
+
export declare function detectCompetingFastembedHost(ownPid: number | undefined, ownPoolGroup?: string): {
|
|
33
|
+
pid: number;
|
|
34
|
+
startedAt: string;
|
|
35
|
+
} | null;
|
|
36
|
+
/**
|
|
37
|
+
* (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) The structural shape both
|
|
38
|
+
* `SharedFastembedProcessClient` (one child) and `FastembedProcessPool`
|
|
39
|
+
* (N independent children) satisfy. `FastembedProvider` (fastembed.ts)
|
|
40
|
+
* depends on this interface, not the concrete single-child class, so
|
|
41
|
+
* `getSharedFastembedProcess()` can return either without callers caring —
|
|
42
|
+
* every existing `.request()`/`.terminate()`/`.started` call site keeps
|
|
43
|
+
* working unchanged.
|
|
44
|
+
*/
|
|
45
|
+
export interface SharedFastembedClient {
|
|
46
|
+
request<T = Record<string, unknown>>(payload: Record<string, unknown>, timeoutMs?: number, signal?: AbortSignal): Promise<T>;
|
|
47
|
+
terminate(): Promise<void>;
|
|
48
|
+
readonly started: boolean;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Thrown by `FastembedProcessPool
|
|
52
|
+
* .request()` when every pool member already has `admissionLimit` or more
|
|
53
|
+
* requests in flight — the "a queue that never refuses hides its own
|
|
54
|
+
* failure" backstop the bug item asked for. This is NOT the primary fix (the
|
|
55
|
+
* pool itself is — see the class below); it exists purely so a burst that
|
|
56
|
+
* genuinely exceeds total pool capacity gets a fast, typed, retryable
|
|
57
|
+
* rejection instead of silently joining a multi-minute queue. Named
|
|
58
|
+
* `TransientEmbeddingError`-shaped (has `retryAfterMs`) but defined locally
|
|
59
|
+
* rather than importing from `./index.js`, to keep this module free of a
|
|
60
|
+
* dependency on the public interface skeleton — callers can duck-type on
|
|
61
|
+
* `.name === 'FastembedBusyError'` or `instanceof FastembedBusyError`.
|
|
62
|
+
*/
|
|
63
|
+
export declare class FastembedBusyError extends Error {
|
|
64
|
+
readonly retryAfterMs: number;
|
|
65
|
+
constructor(message: string, retryAfterMs: number);
|
|
66
|
+
}
|
|
67
|
+
export declare class SharedFastembedProcessClient implements SharedFastembedClient {
|
|
19
68
|
private child;
|
|
20
69
|
private startingPromise;
|
|
21
70
|
private nextId;
|
|
22
71
|
private pending;
|
|
23
72
|
private readonly hostPath;
|
|
73
|
+
private readonly poolGroup;
|
|
74
|
+
private poolSize;
|
|
75
|
+
private memberIndex;
|
|
76
|
+
/**
|
|
77
|
+
* (BUG-021) The last `{ type: 'init', model, cacheDir }` payload this
|
|
78
|
+
* member has ever been asked to load — kept independently of any pool
|
|
79
|
+
* wrapper above it so a LONE (non-pooled) client is self-healing too.
|
|
80
|
+
* `null` until the first successful `init`.
|
|
81
|
+
*/
|
|
82
|
+
private lastInitPayload;
|
|
83
|
+
/**
|
|
84
|
+
* (BUG-021 root cause) True once the CURRENT `this.child` has actually
|
|
85
|
+
* loaded the model named by `lastInitPayload`. Reset to `false` every time
|
|
86
|
+
* `ensureProcess()` forks a NEW child — including a transparent respawn
|
|
87
|
+
* after the previous child died (crash, OOM-kill, the BL-426/std::bad_alloc
|
|
88
|
+
* native-teardown hazards documented on `fastembedProcessHost.ts`, etc).
|
|
89
|
+
*
|
|
90
|
+
* Before this fix, `ensureProcess()`'s `c.on('exit')`/`c.on('error')`
|
|
91
|
+
* handlers nulled out `this.child`/`this.startingPromise` on an unexpected
|
|
92
|
+
* death, but nothing told the NEXT `request()` call that the freshly
|
|
93
|
+
* (re)forked replacement child has an empty `_embedder` — every real
|
|
94
|
+
* `embed`/`embedBatch` request sent to it failed with "Model not
|
|
95
|
+
* initialized" forever, because the higher-level `FastembedProvider.ready`
|
|
96
|
+
* flag was already latched `true` from the original (pre-crash) init and
|
|
97
|
+
* `ensureReady()` never re-sends `init` once `ready` is true. Live incident
|
|
98
|
+
* BUG-021: the lock file showed a child forked mere SECONDS before every
|
|
99
|
+
* request against it failed — exactly this respawn-without-reinit gap, not
|
|
100
|
+
* a stale/orphaned lock as first suspected.
|
|
101
|
+
*/
|
|
102
|
+
private childInitialized;
|
|
24
103
|
/**
|
|
25
104
|
* @param hostPathOverride Test-only injection point (BL-410): points the
|
|
26
105
|
* fork target at a lightweight fixture host instead of the real
|
|
@@ -28,10 +107,47 @@ export declare class SharedFastembedProcessClient {
|
|
|
28
107
|
* contract without loading fastembed or downloading a model. Production
|
|
29
108
|
* code (`getSharedFastembedProcess()`) never passes this — it always
|
|
30
109
|
* resolves the real host path.
|
|
110
|
+
* @param poolGroup (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Set by
|
|
111
|
+
* `FastembedProcessPool` to a group id shared by every member of that pool.
|
|
112
|
+
* Forwarded to the forked child via `SOX_FASTEMBED_POOL_GROUP` so the
|
|
113
|
+
* BL-331 advisory lock can tell "another member of my own pool" apart from
|
|
114
|
+
* "a genuinely unrelated fastembed host" — see `fastembedLock.ts`'s
|
|
115
|
+
* `poolGroup` doc comment. `undefined` for a lone (non-pooled) client,
|
|
116
|
+
* which preserves today's exact single-host lock behaviour there.
|
|
117
|
+
* @param poolSize / @param memberIndex (BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001,
|
|
118
|
+
* observability addendum) Set by `FastembedProcessPool` so every
|
|
119
|
+
* `fastembed_process.request.*` telemetry record this client emits carries
|
|
120
|
+
* the pool's actual size and this member's index alongside `queue_depth`/
|
|
121
|
+
* `response_ms`. Diagnosing THIS incident required an agent to read source
|
|
122
|
+
* and reconstruct `resolveFastembedPoolSize()`'s arithmetic by hand,
|
|
123
|
+
* because no telemetry field ever recorded what size the pool actually
|
|
124
|
+
* resolved to at runtime — `queue_depth` alone cannot distinguish
|
|
125
|
+
* "contention on an inert 1-member pool" from "genuine over-capacity on a
|
|
126
|
+
* 4-member pool". `undefined` for a lone (non-pooled) client, which omits
|
|
127
|
+
* both fields from telemetry exactly as before this addendum.
|
|
31
128
|
*/
|
|
32
|
-
constructor(hostPathOverride?: string);
|
|
129
|
+
constructor(hostPathOverride?: string, poolGroup?: string, poolSize?: number, memberIndex?: number);
|
|
33
130
|
/** True once the underlying child process has been forked. */
|
|
34
131
|
get started(): boolean;
|
|
132
|
+
/**
|
|
133
|
+
* (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Number of requests already
|
|
134
|
+
* admitted-and-unsettled on this specific child, i.e. how many are ahead
|
|
135
|
+
* of a hypothetical next request. Exposed read-only so `FastembedProcessPool`
|
|
136
|
+
* (below) can route a new request to whichever pool member is least loaded
|
|
137
|
+
* — the exact `this.pending.size` value `request()` already measures
|
|
138
|
+
* internally for the `queue_depth` telemetry field, just made visible to a
|
|
139
|
+
* caller one level up instead of re-derived.
|
|
140
|
+
*/
|
|
141
|
+
get pendingCount(): number;
|
|
142
|
+
/**
|
|
143
|
+
* (BL-575) Update the `pool_size`/`member_index` telemetry fields this
|
|
144
|
+
* client stamps on every `fastembed_process.request.*` record. Package-
|
|
145
|
+
* private (no `readonly`) specifically so `AdaptiveFastembedProcessPool`
|
|
146
|
+
* can keep them truthful as the pool grows/shrinks at runtime — a FIXED
|
|
147
|
+
* `FastembedProcessPool` never calls this (its size is constant for the
|
|
148
|
+
* member's whole lifetime, set once at construction).
|
|
149
|
+
*/
|
|
150
|
+
setPoolMeta(poolSize: number, memberIndex: number): void;
|
|
35
151
|
/** Lazily fork (exactly once) and return the single shared child process. */
|
|
36
152
|
private ensureProcess;
|
|
37
153
|
/**
|
|
@@ -76,8 +192,24 @@ export declare class SharedFastembedProcessClient {
|
|
|
76
192
|
* 3. `competing_host_pid` — present only when a second, still-live
|
|
77
193
|
* `fastembedProcessHost` process is detected (BL-331's advisory lock),
|
|
78
194
|
* since that changes embed latency 25-50x independent of queueing.
|
|
195
|
+
*
|
|
196
|
+
* BL-576: `signal`, when supplied, is an external cancellation source —
|
|
197
|
+
* e.g. `operation-guard.ts`'s `withOperationDeadline` abort controller, so
|
|
198
|
+
* a deadline that fires while an embed is queued behind an unrelated slow
|
|
199
|
+
* request stops WAITING on it immediately rather than riding the deadline
|
|
200
|
+
* out via its own separate mechanism. This does NOT kill the underlying
|
|
201
|
+
* fastembed child mid-inference (the same "no native cancellation
|
|
202
|
+
* primitive" constraint `operation-guard.ts` documents applies here too —
|
|
203
|
+
* the child process is shared across other pending requests, so killing
|
|
204
|
+
* it on one caller's abort would collaterally fail every other in-flight
|
|
205
|
+
* request on that member) — it settles the CALLER's promise immediately
|
|
206
|
+
* with an `AbortError` and removes the pending entry so a late reply from
|
|
207
|
+
* the child (once the real work finishes) is silently dropped instead of
|
|
208
|
+
* resolving/rejecting a promise nobody is awaiting anymore. If `signal`
|
|
209
|
+
* is already aborted when `request()` is called, it rejects immediately
|
|
210
|
+
* without ever sending to the child.
|
|
79
211
|
*/
|
|
80
|
-
request<T = Record<string, unknown>>(payload: Record<string, unknown>, timeoutMs?: number): Promise<T>;
|
|
212
|
+
request<T = Record<string, unknown>>(payload: Record<string, unknown>, timeoutMs?: number, signal?: AbortSignal): Promise<T>;
|
|
81
213
|
/**
|
|
82
214
|
* Terminate the shared fastembed process. Intended ONLY for full process
|
|
83
215
|
* shutdown or test teardown that genuinely owns the whole process's
|
|
@@ -95,14 +227,277 @@ export declare class SharedFastembedProcessClient {
|
|
|
95
227
|
*/
|
|
96
228
|
terminate(): Promise<void>;
|
|
97
229
|
}
|
|
230
|
+
/**
|
|
231
|
+
* (BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001) Estimate REAL available memory
|
|
232
|
+
* (MB), platform-aware — the input `resolveFastembedPoolSize()`'s
|
|
233
|
+
* memory-cap arithmetic below actually needs, as opposed to what
|
|
234
|
+
* `os.freemem()` alone reports.
|
|
235
|
+
*
|
|
236
|
+
* `os.freemem()` is not a usable proxy for "memory this process could
|
|
237
|
+
* actually claim" on macOS: it maps to Mach's raw "free" page count only,
|
|
238
|
+
* which deliberately EXCLUDES "inactive"/"speculative"/"purgeable" pages —
|
|
239
|
+
* pages the kernel is using as disk cache but will hand back instantly
|
|
240
|
+
* (zero swap-in cost) under real pressure. Measured via `vm_stat` on the
|
|
241
|
+
* exact box this defect was diagnosed on (32GB physical, page size 16384):
|
|
242
|
+
* `os.freemem()` reported ~299MB while `Pages inactive` ALONE was 345,579
|
|
243
|
+
* pages (~5.4GB) — the overwhelming majority of genuinely-available memory
|
|
244
|
+
* was invisible to the metric `resolveFastembedPoolSize()` used to
|
|
245
|
+
* compute `memoryCap`. Because `MEMORY_SAFETY_MARGIN_MB` (1024) routinely
|
|
246
|
+
* exceeds `os.freemem()`'s ~300MB-ish reading on macOS regardless of real
|
|
247
|
+
* load, `memoryCap` collapsed to `Math.max(1, negative) === 1` on every
|
|
248
|
+
* macOS box, unconditionally — the pool was permanently INERT (silently
|
|
249
|
+
* behaving exactly like the pre-fix single-child topology) unless an
|
|
250
|
+
* operator manually overrode `SOX_EMBED_POOL_SIZE`. `hol-pool-sizing.spec.ts`
|
|
251
|
+
* even self-documents "this suite was itself first run on a box with only
|
|
252
|
+
* ~128MB free" — the sizing design was validated against the very macOS
|
|
253
|
+
* quirk that made it universally wrong, not a genuinely memory-constrained
|
|
254
|
+
* machine.
|
|
255
|
+
*
|
|
256
|
+
* Fix: compute "available" as free + inactive + speculative + purgeable
|
|
257
|
+
* pages (the standard macOS "reclaimable without swapping" heuristic —
|
|
258
|
+
* matches what `htop`-family tools approximate; NOT Apple's undocumented
|
|
259
|
+
* memory-pressure internals, which have no public API). On Linux,
|
|
260
|
+
* `/proc/meminfo`'s `MemAvailable` is already the kernel's own equivalent
|
|
261
|
+
* estimate and is used directly. Any other platform, or any parse/exec
|
|
262
|
+
* failure, falls back to `os.freemem()` unchanged — this must never throw
|
|
263
|
+
* or block pool sizing on a shell-out failing; it runs once per process
|
|
264
|
+
* (at `getSharedFastembedProcess()` construction), never per-request.
|
|
265
|
+
*/
|
|
266
|
+
export declare function estimateAvailableMemMb(): number;
|
|
267
|
+
/**
|
|
268
|
+
* (BL-575) The explicit `SOX_EMBED_POOL_SIZE` override, if set — a HARD PIN.
|
|
269
|
+
* Split out of `resolveFastembedPoolSize()` so `getSharedFastembedProcess()`
|
|
270
|
+
* can tell "operator pinned an exact size" (disables ALL adaptation —
|
|
271
|
+
* `AdaptiveFastembedProcessPool` is never constructed) apart from "no
|
|
272
|
+
* override, compute a ceiling for adaptive sizing to grow toward" — the two
|
|
273
|
+
* cases used to be indistinguishable from the return value of a single
|
|
274
|
+
* function that already folded the CPU/memory-cap arithmetic into the
|
|
275
|
+
* "no override" branch.
|
|
276
|
+
*/
|
|
277
|
+
export declare function resolveFastembedPoolPin(): number | null;
|
|
278
|
+
/**
|
|
279
|
+
* (BL-575) The maximum pool size the memory/CPU budget allows — i.e. what
|
|
280
|
+
* `resolveFastembedPoolSize()` used to compute unconditionally in its
|
|
281
|
+
* "no override" branch. Now used two ways: (a) unchanged, as
|
|
282
|
+
* `resolveFastembedPoolSize()`'s own fallback when no pin is set, so every
|
|
283
|
+
* existing caller of that function keeps its exact prior behavior; (b) as
|
|
284
|
+
* `AdaptiveFastembedProcessPool`'s `maxSize` — the ceiling it is allowed to
|
|
285
|
+
* grow toward under sustained load, never exceeded regardless of how
|
|
286
|
+
* sustained the demand is.
|
|
287
|
+
*
|
|
288
|
+
* Auto-sized from BOTH real available memory (`estimateAvailableMemMb()` —
|
|
289
|
+
* see its doc comment for why this is NOT simply `os.freemem()`, and
|
|
290
|
+
* BUG-EMBED-POOL-SIZE-DARWIN-FREEMEM-001 for the incident this fixes —
|
|
291
|
+
* against `DEFAULT_PER_MEMBER_MB`, less `MEMORY_SAFETY_MARGIN_MB` headroom —
|
|
292
|
+
* override the per-member budget via `SOX_EMBED_POOL_PER_CHILD_MB` for a
|
|
293
|
+
* non-default model) AND CPU count (half the logical CPUs, cap 4 — fastembed
|
|
294
|
+
* inference is CPU/ANE-bound per request, not embarrassingly parallel across
|
|
295
|
+
* all cores), taking the SMALLER of the two so a memory-constrained box
|
|
296
|
+
* never gets sized past what's actually free. See the module doc comment
|
|
297
|
+
* above for the measurement (footprint/vmmap on 1 vs 4 real children) that
|
|
298
|
+
* justifies this.
|
|
299
|
+
*
|
|
300
|
+
* @param getAvailableMemMb Test-only injection point — production always
|
|
301
|
+
* uses the default `estimateAvailableMemMb`. Lets tests exercise the sizing
|
|
302
|
+
* arithmetic against deterministic MB values instead of the real, inherently
|
|
303
|
+
* machine/moment-dependent OS memory state.
|
|
304
|
+
*/
|
|
305
|
+
export declare function resolveFastembedPoolCeiling(getAvailableMemMb?: () => number): number;
|
|
306
|
+
/**
|
|
307
|
+
* Number of independent fastembed child processes a FIXED (non-adaptive)
|
|
308
|
+
* pool should use. Preserved unchanged for backward compatibility (existing
|
|
309
|
+
* callers/tests) — `SOX_EMBED_POOL_SIZE` honored exactly if set
|
|
310
|
+
* (`resolveFastembedPoolPin()`), else the memory/CPU ceiling
|
|
311
|
+
* (`resolveFastembedPoolCeiling()`). `getSharedFastembedProcess()` itself no
|
|
312
|
+
* longer calls this for its default (adaptive) path as of BL-575 — see
|
|
313
|
+
* `resolveFastembedPoolPin`/`resolveFastembedPoolCeiling`'s doc comments.
|
|
314
|
+
*
|
|
315
|
+
* @param getAvailableMemMb Test-only injection point, forwarded to
|
|
316
|
+
* `resolveFastembedPoolCeiling`.
|
|
317
|
+
*/
|
|
318
|
+
export declare function resolveFastembedPoolSize(getAvailableMemMb?: () => number): number;
|
|
319
|
+
/** Per-member in-flight cap above which `FastembedProcessPool.request()` fast-rejects
|
|
320
|
+
* a non-init request with `FastembedBusyError` instead of enqueuing. Disabled
|
|
321
|
+
* (`Infinity`) by default — see the pool doc comment above for why. Override
|
|
322
|
+
* via `SOX_EMBED_POOL_ADMISSION_LIMIT` (a finite number opts in). */
|
|
323
|
+
export declare function resolveFastembedAdmissionLimit(): number;
|
|
324
|
+
/**
|
|
325
|
+
* A pool of `size` independent `SharedFastembedProcessClient`s — i.e. `size`
|
|
326
|
+
* independent fastembed-hosting OS child processes, each fully isolated from
|
|
327
|
+
* the others (preserving BL-238's process-isolation requirement) — routing
|
|
328
|
+
* each request to whichever member currently has the fewest requests in
|
|
329
|
+
* flight. See the module-level doc comment above for the full rationale and
|
|
330
|
+
* the production measurements this fixes.
|
|
331
|
+
*/
|
|
332
|
+
export declare class FastembedProcessPool implements SharedFastembedClient {
|
|
333
|
+
readonly members: SharedFastembedProcessClient[];
|
|
334
|
+
private readonly admissionLimit;
|
|
335
|
+
/**
|
|
336
|
+
* (BUG-MEMORY-EMBED-HEAD-OF-LINE-BLOCKING-001) Routing/admission MUST NOT
|
|
337
|
+
* read `member.pendingCount` for the decision — that counter is only
|
|
338
|
+
* incremented deep inside `SharedFastembedProcessClient.request()`, AFTER
|
|
339
|
+
* its own `await this.ensureProcess()`, i.e. at least one microtask tick
|
|
340
|
+
* after the call starts. Several callers issued in the same synchronous
|
|
341
|
+
* burst (`Promise.all(callers.map(() => pool.request(...)))`, the exact
|
|
342
|
+
* shape a real caller under load produces) all run their OWN synchronous
|
|
343
|
+
* prefix — including this pool's `leastLoaded()`/admission check — before
|
|
344
|
+
* ANY of them reaches that tick, so every one of them would read
|
|
345
|
+
* `pendingCount === 0` for every member and pile onto `members[0]`,
|
|
346
|
+
* silently defeating both load-balancing and admission control for
|
|
347
|
+
* exactly the bursty-arrival case this fix exists for. `inFlight` is
|
|
348
|
+
* incremented SYNCHRONOUSLY the instant a member is chosen (before any
|
|
349
|
+
* `await`), so the very next synchronous call in the same burst already
|
|
350
|
+
* sees it.
|
|
351
|
+
*/
|
|
352
|
+
private readonly inFlight;
|
|
353
|
+
/** Cached so a member added to routing after the first `init` (there is
|
|
354
|
+
* none today — pool size is fixed at construction — but kept so a future
|
|
355
|
+
* dynamic-resize doesn't silently skip initializing a new member) can be
|
|
356
|
+
* brought up to date. Also lets `request()` short-circuit a redundant
|
|
357
|
+
* broadcast if the model/cacheDir haven't changed. */
|
|
358
|
+
private lastInitPayload;
|
|
359
|
+
constructor(size: number, hostPathOverride?: string, admissionLimit?: number);
|
|
360
|
+
/** True once at least one member has forked its child process. */
|
|
361
|
+
get started(): boolean;
|
|
362
|
+
/** The last `{ type: 'init', model, cacheDir }` payload broadcast to every
|
|
363
|
+
* member, or `null` before the first init. Exposed for introspection/tests
|
|
364
|
+
* only — `request()` itself doesn't need to read this back today (pool
|
|
365
|
+
* size is fixed at construction, so every member is always initialized by
|
|
366
|
+
* the broadcast), it's retained purely so a future dynamic-resize path has
|
|
367
|
+
* the payload on hand to bring a newly-added member up to date. */
|
|
368
|
+
get lastInit(): Record<string, unknown> | null;
|
|
369
|
+
/** Sum of in-flight requests across every member — the pool-wide
|
|
370
|
+
* head-of-line depth an arriving caller would experience. */
|
|
371
|
+
get pendingCount(): number;
|
|
372
|
+
/** Index of the least-loaded member by the SYNCHRONOUS `inFlight` counter
|
|
373
|
+
* — see that field's doc comment for why `member.pendingCount` itself is
|
|
374
|
+
* unsafe to read here. */
|
|
375
|
+
private leastLoadedIndex;
|
|
376
|
+
request<T = Record<string, unknown>>(payload: Record<string, unknown>, timeoutMs?: number): Promise<T>;
|
|
377
|
+
terminate(): Promise<void>;
|
|
378
|
+
}
|
|
379
|
+
export interface AdaptiveFastembedPoolOptions {
|
|
380
|
+
/** Starting (and floor) pool size. Default 1 — the topology real
|
|
381
|
+
* measurement shows wins at concurrency <20. */
|
|
382
|
+
minSize?: number;
|
|
383
|
+
/** Ceiling the pool may grow to — pass `resolveFastembedPoolCeiling()`'s
|
|
384
|
+
* result in production; never exceeded regardless of how sustained
|
|
385
|
+
* demand is. */
|
|
386
|
+
maxSize: number;
|
|
387
|
+
hostPathOverride?: string;
|
|
388
|
+
admissionLimit?: number;
|
|
389
|
+
/** Test-only clock injection — production uses `Date.now`. */
|
|
390
|
+
now?: () => number;
|
|
391
|
+
/** Hysteresis knobs — all optional, defaulting to the values documented
|
|
392
|
+
* above. Exposed so a test can use tiny thresholds/intervals instead of
|
|
393
|
+
* waiting out real minutes-scale windows, and so an operator can tune
|
|
394
|
+
* the policy without a rebuild if the defaults prove wrong for their
|
|
395
|
+
* traffic shape. Production (`getSharedFastembedProcess()`) never
|
|
396
|
+
* overrides any of these — it relies on the documented defaults. */
|
|
397
|
+
growQueueRatioThreshold?: number;
|
|
398
|
+
growSustainCount?: number;
|
|
399
|
+
/** (BL-575 hysteresis fix, post-ship) Minimum WALL-CLOCK span the
|
|
400
|
+
* over-threshold streak must cover before a grow is allowed to fire —
|
|
401
|
+
* see `growSustainWindowMs`'s own doc comment on the field below for why
|
|
402
|
+
* `growSustainCount` alone was not sufficient. */
|
|
403
|
+
growSustainWindowMs?: number;
|
|
404
|
+
growCooldownMs?: number;
|
|
405
|
+
shrinkIdleMs?: number;
|
|
406
|
+
shrinkCheckIntervalMs?: number;
|
|
407
|
+
}
|
|
408
|
+
export declare class AdaptiveFastembedProcessPool implements SharedFastembedClient {
|
|
409
|
+
private members;
|
|
410
|
+
private inFlight;
|
|
411
|
+
private readonly minSize;
|
|
412
|
+
private readonly maxSize;
|
|
413
|
+
private readonly hostPathOverride;
|
|
414
|
+
private readonly admissionLimit;
|
|
415
|
+
private readonly poolGroup;
|
|
416
|
+
private readonly clock;
|
|
417
|
+
private readonly shrinkTimer;
|
|
418
|
+
private readonly growQueueRatioThreshold;
|
|
419
|
+
private readonly growSustainCount;
|
|
420
|
+
private readonly growSustainWindowMs;
|
|
421
|
+
private readonly growCooldownMs;
|
|
422
|
+
private readonly shrinkIdleMs;
|
|
423
|
+
private lastInitPayload;
|
|
424
|
+
private growConsecutiveOverThreshold;
|
|
425
|
+
/** Wall-clock timestamp the CURRENT over-threshold streak began — `null`
|
|
426
|
+
* when not currently in a streak. See `GROW_SUSTAIN_WINDOW_MS`'s doc
|
|
427
|
+
* comment for why a count alone is insufficient. */
|
|
428
|
+
private growStreakStartedAt;
|
|
429
|
+
private lastGrowAt;
|
|
430
|
+
private idleSinceMs;
|
|
431
|
+
/** Serializes concurrent grow attempts — `request()` can observe the
|
|
432
|
+
* over-threshold condition from several concurrent callers in the same
|
|
433
|
+
* burst; only one grow should actually happen. */
|
|
434
|
+
private growInFlight;
|
|
435
|
+
private terminated;
|
|
436
|
+
/** Counts of grow/shrink actions taken — exposed for tests/observability,
|
|
437
|
+
* not consumed by any routing logic. */
|
|
438
|
+
growCount: number;
|
|
439
|
+
shrinkCount: number;
|
|
440
|
+
constructor(opts: AdaptiveFastembedPoolOptions);
|
|
441
|
+
get started(): boolean;
|
|
442
|
+
get lastInit(): Record<string, unknown> | null;
|
|
443
|
+
get pendingCount(): number;
|
|
444
|
+
/** Current pool size — exposed for tests/observability. */
|
|
445
|
+
get size(): number;
|
|
446
|
+
private leastLoadedIndex;
|
|
447
|
+
/** Renumber every member's `pool_size`/`member_index` telemetry fields to
|
|
448
|
+
* match current pool composition — called after every grow/shrink so
|
|
449
|
+
* `fastembed_process.request.*` records stay truthful. */
|
|
450
|
+
private renumberMembers;
|
|
451
|
+
/**
|
|
452
|
+
* Attempt to add one member, up to `maxSize`. Best-effort and internally
|
|
453
|
+
* serialized (`growInFlight`) — safe to call from multiple concurrent
|
|
454
|
+
* `request()` admissions without double-growing. If a `lastInitPayload`
|
|
455
|
+
* exists (the pool has already been initialized with a model), the new
|
|
456
|
+
* member is sent the SAME init payload before being added to routing —
|
|
457
|
+
* an un-initialized member would fail every real embed request it was
|
|
458
|
+
* routed to.
|
|
459
|
+
*/
|
|
460
|
+
private grow;
|
|
461
|
+
/** Periodic shrink check — terminates exactly ONE idle member if the
|
|
462
|
+
* entire pool has been continuously idle for `SHRINK_IDLE_MS`. Never
|
|
463
|
+
* shrinks below `minSize`. Resets the idle clock after shrinking so a
|
|
464
|
+
* demand spike right after only costs one re-grow step, not a full
|
|
465
|
+
* rebuild, and so consecutive shrinks are still spaced `SHRINK_IDLE_MS`
|
|
466
|
+
* apart (the same conservative cadence, not a rapid drain to `minSize`). */
|
|
467
|
+
private maybeShrink;
|
|
468
|
+
request<T = Record<string, unknown>>(payload: Record<string, unknown>, timeoutMs?: number, signal?: AbortSignal): Promise<T>;
|
|
469
|
+
terminate(): Promise<void>;
|
|
470
|
+
}
|
|
98
471
|
/**
|
|
99
472
|
* Process-wide singleton accessor — the ONLY sanctioned place a fastembed
|
|
100
473
|
* child process is forked anywhere in `@adhd/sox-embedding-provider`
|
|
101
474
|
* (BL-238/BL-171). Every `FastembedProvider` obtains its handle through this
|
|
102
475
|
* function instead of constructing its own `worker_threads.Worker` or
|
|
103
476
|
* `child_process`.
|
|
477
|
+
*
|
|
478
|
+
* (BL-575) Two shapes, chosen by whether `SOX_EMBED_POOL_SIZE` is set:
|
|
479
|
+
*
|
|
480
|
+
* - PINNED (`resolveFastembedPoolPin()` returns non-null): a FIXED
|
|
481
|
+
* `FastembedProcessPool` at exactly that size, no adaptation — an
|
|
482
|
+
* operator who set this has already judged their model's footprint and
|
|
483
|
+
* the box's headroom; adaptive sizing would second-guess that judgement.
|
|
484
|
+
* - DEFAULT (no override): an `AdaptiveFastembedProcessPool` starting at
|
|
485
|
+
* `minSize: 1` (the topology real measurement shows wins below ~20
|
|
486
|
+
* concurrency) and growing toward `resolveFastembedPoolCeiling()` (the
|
|
487
|
+
* SAME ceiling `FastembedProcessPool` used to apply unconditionally)
|
|
488
|
+
* only under sustained demand — see that class's doc comment for the
|
|
489
|
+
* full measurement and hysteresis policy (BUG-MEMORY-EMBED-HEAD-OF-LINE-
|
|
490
|
+
* BLOCKING-001's original qdepth-11+ production regime is exactly the
|
|
491
|
+
* sustained-demand case this still grows to meet; the difference is it
|
|
492
|
+
* no longer pays that pool's cost on every LOWER-concurrency request
|
|
493
|
+
* too).
|
|
494
|
+
*
|
|
495
|
+
* Every existing caller keeps working unchanged either way — both classes
|
|
496
|
+
* implement the same `SharedFastembedClient` shape (`request()`/
|
|
497
|
+
* `terminate()`/`started`). Setting `SOX_EMBED_POOL_SIZE=1` recovers the
|
|
498
|
+
* exact pre-BL-575 (and pre-pool) single-child topology.
|
|
104
499
|
*/
|
|
105
|
-
export declare function getSharedFastembedProcess():
|
|
500
|
+
export declare function getSharedFastembedProcess(): SharedFastembedClient;
|
|
106
501
|
/**
|
|
107
502
|
* Test-only: reset the module-level singleton so a test can exercise a
|
|
108
503
|
* fresh shared-process lifecycle (e.g. after deliberately crashing/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sharedFastembedProcess.d.ts","sourceRoot":"","sources":["../src/sharedFastembedProcess.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"sharedFastembedProcess.d.ts","sourceRoot":"","sources":["../src/sharedFastembedProcess.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAwFH,0EAA0E;AAC1E,wBAAgB,iCAAiC,IAAI,IAAI,CAKxD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,MAAM,GAAG,SAAS,EAC1B,YAAY,CAAC,EAAE,MAAM,GACpB;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAoC3C;AASD;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,SAAS,CAAC,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,CAAC,CAAC;IACd,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAClB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;CAKlD;AAED,qBAAa,4BAA6B,YAAW,qBAAqB;IACxE,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,eAAe,CAAsC;IAC7D,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,OAAO,CAAmC;IAClD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,WAAW,CAAqB;IACxC;;;;;OAKG;IACH,OAAO,CAAC,eAAe,CAAwC;IAC/D;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,gBAAgB,CAAS;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;gBACS,gBAAgB,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM;IAOlG,8DAA8D;IAC9D,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;;;;;OAQG;IACH,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAKxD,6EAA6E;IAC7E,OAAO,CAAC,aAAa;IAmFrB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,aAAa;IAKrB,gFAAgF;IAChF,OAAO,CAAC,WAAW;IAOnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACG,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,SAAS,CAAC,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,CAAC;IAuIb;;;;;;;;;;;;;;OAcG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAgCjC;AA+HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CA6B/C;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,IAAI,MAAM,GAAG,IAAI,CAIvD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,2BAA2B,CAAC,iBAAiB,GAAE,MAAM,MAA+B,GAAG,MAAM,CAU5G;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,wBAAwB,CAAC,iBAAiB,GAAE,MAAM,MAA+B,GAAG,MAAM,CAEzG;AAED;;;sEAGsE;AACtE,wBAAgB,8BAA8B,IAAI,MAAM,CAIvD;AAED;;;;;;;GAOG;AACH,qBAAa,oBAAqB,YAAW,qBAAqB;IAChE,QAAQ,CAAC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC;;;;2DAIuD;IACvD,OAAO,CAAC,eAAe,CAAwC;gBAEnD,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,MAAM,EAAE,cAAc,SAAmC;IAiBtG,kEAAkE;IAClE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;;wEAKoE;IACpE,IAAI,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAE7C;IAED;kEAC8D;IAC9D,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;+BAE2B;IAC3B,OAAO,CAAC,gBAAgB;IAQlB,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,CAAC,CAAC;IAqCP,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAGjC;AA4ED,MAAM,WAAW,4BAA4B;IAC3C;qDACiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;qBAEiB;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;;;;;yEAKqE;IACrE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;uDAGmD;IACnD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAoCD,qBAAa,4BAA6B,YAAW,qBAAqB;IACxE,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiC;IAC7D,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,OAAO,CAAC,eAAe,CAAwC;IAC/D,OAAO,CAAC,4BAA4B,CAAK;IACzC;;yDAEqD;IACrD,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAgB;IACnC;;uDAEmD;IACnD,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,UAAU,CAAS;IAE3B;6CACyC;IACzC,SAAS,SAAK;IACd,WAAW,SAAK;gBAEJ,IAAI,EAAE,4BAA4B;IA4B9C,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAE7C;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,2DAA2D;IAC3D,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,OAAO,CAAC,gBAAgB;IAQxB;;+DAE2D;IAC3D,OAAO,CAAC,eAAe;IAMvB;;;;;;;;OAQG;YACW,IAAI;IA+ClB;;;;;iFAK6E;IAC7E,OAAO,CAAC,WAAW;IA+Bb,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACvC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,SAAS,CAAC,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,CAAC;IAkEP,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAKjC;AAID;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,yBAAyB,IAAI,qBAAqB,CAQjE;AAED;;;;;;GAMG;AACH,wBAAgB,qCAAqC,IAAI,IAAI,CAE5D"}
|