@vellumai/assistant 0.12.2-dev.202609212016.fb4c53e → 0.12.2-dev.202609212215.5512e77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/package.json +1 -1
- package/src/cli/commands/channel-verification-sessions.help.ts +12 -2
- package/src/cli/lib/bundled-marketplace.json +4 -4
- package/src/config/feature-flag-registry.json +0 -8
- package/src/daemon/handlers/config-channels.ts +3 -1
- package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +157 -32
- package/src/persistence/embeddings/embedding-backend-shutdown.test.ts +47 -0
- package/src/persistence/embeddings/embedding-backend.ts +23 -41
- package/src/persistence/embeddings/embedding-local.ts +72 -17
- package/src/persistence/embeddings/embedding-types.ts +0 -19
- package/src/runtime/verification-outbound-actions.ts +2 -2
- package/src/util/worker-pipe.test.ts +100 -0
- package/src/util/worker-pipe.ts +44 -0
package/README.md
CHANGED
|
@@ -70,16 +70,16 @@ bun run src/index.ts # interactive CLI session
|
|
|
70
70
|
|
|
71
71
|
### CLI commands
|
|
72
72
|
|
|
73
|
-
| Command
|
|
74
|
-
|
|
|
75
|
-
| `vellum wake`
|
|
76
|
-
| `vellum sleep`
|
|
77
|
-
| `vellum ps`
|
|
78
|
-
| `assistant`
|
|
73
|
+
| Command | Description |
|
|
74
|
+
| ---------------------------------------------------------- | ------------------------------------------------ |
|
|
75
|
+
| `vellum wake` | Start assistant + gateway from current checkout |
|
|
76
|
+
| `vellum sleep` | Stop assistant + gateway processes |
|
|
77
|
+
| `vellum ps` | List assistants and per-assistant process status |
|
|
78
|
+
| `assistant` | Launch interactive CLI session |
|
|
79
79
|
| `assistant conversations list\|search\|new\|export\|clear` | Manage conversations |
|
|
80
|
-
| `assistant config set\|get\|list`
|
|
81
|
-
| `assistant keys set\|list\|delete`
|
|
82
|
-
| `assistant trust list\|add\|update\|remove`
|
|
80
|
+
| `assistant config set\|get\|list` | Manage configuration |
|
|
81
|
+
| `assistant keys set\|list\|delete` | Manage API keys in secure storage |
|
|
82
|
+
| `assistant trust list\|add\|update\|remove` | Manage trust rules |
|
|
83
83
|
|
|
84
84
|
## Project Structure
|
|
85
85
|
|
|
@@ -244,8 +244,8 @@ The channel guardian service generates verification challenge instructions with
|
|
|
244
244
|
### Operator Notes
|
|
245
245
|
|
|
246
246
|
- **Verification input format:** Channel verification accepts a bare code reply only (6-digit numeric for identity-bound sessions; 64-char hex for unbound inbound/bootstrap compatibility).
|
|
247
|
-
- **Rebind requirement:** Creating a new guardian challenge when a binding already exists requires `rebind: true` in the HTTP request. Without it, the assistant returns `already_bound`.
|
|
248
|
-
- **Takeover prevention:**
|
|
247
|
+
- **Rebind requirement:** Creating a new guardian challenge when a binding already exists requires `rebind: true` in the HTTP request. Without it, the assistant returns `already_bound`. On text channels the resulting code can only be redeemed by the account already bound, which is how the same guardian verifies again; to move the guardian to a different account, revoke the binding first and verify again. An outbound phone verification is the exception: its code replaces the bound number.
|
|
248
|
+
- **Takeover prevention:** On text channels, verification is rejected when an active binding exists for a different external user: the code is spent, the reply is the generic "invalid or expired", and the sender is made neither guardian nor contact. Same-user re-verification is allowed, including for a guardian who revoked their binding and is verifying the same account again. Phone differs: an outbound verification with `rebind: true` replaces the bound number once its code is entered, and an inbound call from a different number is not bound.
|
|
249
249
|
|
|
250
250
|
### Vellum Guardian Identity (Actor Tokens)
|
|
251
251
|
|
package/package.json
CHANGED
|
@@ -44,7 +44,11 @@ Examples:
|
|
|
44
44
|
description:
|
|
45
45
|
"Destination address for outbound verification (handle, phone number, user ID, or email address)",
|
|
46
46
|
},
|
|
47
|
-
{
|
|
47
|
+
{
|
|
48
|
+
flags: "--rebind",
|
|
49
|
+
description:
|
|
50
|
+
"Create a code although the channel already has a guardian (see below)",
|
|
51
|
+
},
|
|
48
52
|
{
|
|
49
53
|
flags: "--conversation-id <conversationId>",
|
|
50
54
|
description: "Conversation ID for inbound challenges",
|
|
@@ -74,7 +78,13 @@ Routes between three creation modes based on the provided options:
|
|
|
74
78
|
2. Outbound: --channel <ch> --destination <dest>
|
|
75
79
|
Sends a verification code to the given destination. Supports telegram
|
|
76
80
|
(handle or chat ID), phone (E.164 number), slack and discord (user ID),
|
|
77
|
-
and email.
|
|
81
|
+
and email.
|
|
82
|
+
|
|
83
|
+
--rebind creates a code although the channel already has a guardian. On
|
|
84
|
+
telegram, slack, discord and email, only the account already bound can redeem
|
|
85
|
+
that code, which is how the same guardian verifies again. A code never moves
|
|
86
|
+
the guardian to a different account there: run "revoke" first, then create
|
|
87
|
+
without --rebind. On phone, an outbound code replaces the bound number.
|
|
78
88
|
|
|
79
89
|
3. Inbound: --channel <ch> (no --destination)
|
|
80
90
|
Generates a challenge secret for the guardian to send back on the
|
|
@@ -176,16 +176,16 @@
|
|
|
176
176
|
"icon": "\ud83d\udcbb"
|
|
177
177
|
},
|
|
178
178
|
{
|
|
179
|
-
"name": "ecommerce-
|
|
179
|
+
"name": "ecommerce-order-supporter",
|
|
180
180
|
"description": "Resolves ecommerce Where-Is-My-Order (WISMO) support inquiries, cross-checks real-time carrier tracking milestones, flags delivery exceptions, and stages customer update drafts without helpdesk seat taxes.",
|
|
181
181
|
"category": "commerce",
|
|
182
182
|
"icon": "📦",
|
|
183
183
|
"license": "MIT",
|
|
184
|
-
"homepage": "https://github.com/ZeebBoyBlue/ecommerce-
|
|
184
|
+
"homepage": "https://github.com/ZeebBoyBlue/ecommerce-order-supporter",
|
|
185
185
|
"source": {
|
|
186
186
|
"source": "github",
|
|
187
|
-
"repo": "ZeebBoyBlue/ecommerce-
|
|
188
|
-
"ref": "
|
|
187
|
+
"repo": "ZeebBoyBlue/ecommerce-order-supporter",
|
|
188
|
+
"ref": "35e2165a14447a897aee750a198c3f166458226b"
|
|
189
189
|
}
|
|
190
190
|
},
|
|
191
191
|
{
|
|
@@ -170,14 +170,6 @@
|
|
|
170
170
|
"description": "Enable Anthropic fast mode for Opus models (4.6, 4.7, 4.8), delivering up to 2.5x higher output tokens per second at premium pricing",
|
|
171
171
|
"defaultEnabled": false
|
|
172
172
|
},
|
|
173
|
-
{
|
|
174
|
-
"id": "teleport",
|
|
175
|
-
"scope": "client",
|
|
176
|
-
"key": "teleport",
|
|
177
|
-
"label": "Teleport",
|
|
178
|
-
"description": "Enable the local-to-platform teleport UI in General settings. Platform-to-local teleport is GA and not gated by this flag",
|
|
179
|
-
"defaultEnabled": false
|
|
180
|
-
},
|
|
181
173
|
{
|
|
182
174
|
"id": "apple-container",
|
|
183
175
|
"scope": "client",
|
|
@@ -156,7 +156,9 @@ export async function createInboundChallenge(
|
|
|
156
156
|
success: false,
|
|
157
157
|
error: "already_bound",
|
|
158
158
|
message:
|
|
159
|
-
|
|
159
|
+
resolvedChannel === "phone"
|
|
160
|
+
? "A guardian is already bound for this channel. On phone, verify through an outbound call with rebind: true. To a new number, it replaces the bound number once the code is entered. To the bound number, it confirms that number again and leaves the binding as it is. An inbound challenge cannot be completed by phone: a call collects a six-digit code, which an inbound challenge does not issue."
|
|
161
|
+
: "A guardian is already bound for this channel. To verify a different account, revoke the existing binding first. Set rebind: true only to verify the same account again.",
|
|
160
162
|
channel: resolvedChannel,
|
|
161
163
|
};
|
|
162
164
|
}
|
|
@@ -27,9 +27,14 @@ import { LocalEmbeddingBackend } from "../embedding-local.js";
|
|
|
27
27
|
type Internals = any;
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
|
-
* A
|
|
31
|
-
*
|
|
30
|
+
* A backend of its own for one test. Production gets the per-model instance
|
|
31
|
+
* from `forModel`; constructing directly keeps tests from sharing state.
|
|
32
32
|
*/
|
|
33
|
+
function newBackend(): Internals {
|
|
34
|
+
return new (LocalEmbeddingBackend as Internals)("test-model");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A stand-in for a Bun subprocess whose stdin `write` fails synchronously. */
|
|
33
38
|
function brokenPipeProc(pid = 4242) {
|
|
34
39
|
return {
|
|
35
40
|
pid,
|
|
@@ -196,7 +201,7 @@ describe("PID file ownership", () => {
|
|
|
196
201
|
*/
|
|
197
202
|
test("releasePidFile leaves an entry that names someone else's worker", () => {
|
|
198
203
|
writeFileSync(pidPath, "777");
|
|
199
|
-
const backend =
|
|
204
|
+
const backend = newBackend();
|
|
200
205
|
|
|
201
206
|
backend.releasePidFile(888);
|
|
202
207
|
|
|
@@ -206,7 +211,7 @@ describe("PID file ownership", () => {
|
|
|
206
211
|
|
|
207
212
|
test("releasePidFile drops the entry when it names our worker", () => {
|
|
208
213
|
writeFileSync(pidPath, "777");
|
|
209
|
-
const backend =
|
|
214
|
+
const backend = newBackend();
|
|
210
215
|
|
|
211
216
|
backend.releasePidFile(777);
|
|
212
217
|
|
|
@@ -219,7 +224,7 @@ describe("PID file ownership", () => {
|
|
|
219
224
|
*/
|
|
220
225
|
test("disposing an instance with no worker keeps another's PID entry", () => {
|
|
221
226
|
writeFileSync(pidPath, "777");
|
|
222
|
-
const backend =
|
|
227
|
+
const backend = newBackend();
|
|
223
228
|
backend.disposeRequested = true;
|
|
224
229
|
backend.workerProc = null;
|
|
225
230
|
|
|
@@ -232,12 +237,12 @@ describe("PID file ownership", () => {
|
|
|
232
237
|
|
|
233
238
|
describe("broken worker pipe", () => {
|
|
234
239
|
/**
|
|
235
|
-
* A
|
|
236
|
-
*
|
|
237
|
-
*
|
|
240
|
+
* A broken pipe comes back as an ordinary failed embed the backend chain can
|
|
241
|
+
* fall back from. An escaping rejection reaches the daemon's
|
|
242
|
+
* `unhandledRejection` handler, which shuts the process down.
|
|
238
243
|
*/
|
|
239
244
|
test("EPIPE on write resolves the request as an error", async () => {
|
|
240
|
-
const backend =
|
|
245
|
+
const backend = newBackend();
|
|
241
246
|
backend.workerProc = brokenPipeProc();
|
|
242
247
|
|
|
243
248
|
const response = await backend.sendRequest(["hello"]);
|
|
@@ -247,7 +252,7 @@ describe("broken worker pipe", () => {
|
|
|
247
252
|
});
|
|
248
253
|
|
|
249
254
|
test("EPIPE surfaces to embed() as a rejection, not a process crash", async () => {
|
|
250
|
-
const backend =
|
|
255
|
+
const backend = newBackend();
|
|
251
256
|
backend.workerProc = brokenPipeProc();
|
|
252
257
|
// Skip initialization: the worker is already (notionally) running.
|
|
253
258
|
backend.ensureInitialized = async () => {};
|
|
@@ -257,8 +262,34 @@ describe("broken worker pipe", () => {
|
|
|
257
262
|
);
|
|
258
263
|
});
|
|
259
264
|
|
|
265
|
+
/**
|
|
266
|
+
* A real pipe, because the failure lives in Bun's behaviour rather than in
|
|
267
|
+
* ours: a batch larger than the pipe buffer, sent to a worker too busy to
|
|
268
|
+
* drain stdin, leaves the write pending, and the pending write rejects when
|
|
269
|
+
* the worker dies. A synchronous guard never sees that rejection.
|
|
270
|
+
*/
|
|
271
|
+
test("a write still pending when the worker dies resolves the request as an error", async () => {
|
|
272
|
+
const backend = newBackend();
|
|
273
|
+
const proc = Bun.spawn({
|
|
274
|
+
cmd: [process.execPath, "-e", "setTimeout(() => {}, 60_000)"],
|
|
275
|
+
windowsHide: true,
|
|
276
|
+
stdin: "pipe",
|
|
277
|
+
stdout: "ignore",
|
|
278
|
+
stderr: "ignore",
|
|
279
|
+
});
|
|
280
|
+
spawned.push(proc);
|
|
281
|
+
backend.workerProc = proc;
|
|
282
|
+
|
|
283
|
+
const request = backend.sendRequest(["x".repeat(4 * 1024 * 1024)]);
|
|
284
|
+
proc.kill("SIGKILL");
|
|
285
|
+
const response = await request;
|
|
286
|
+
|
|
287
|
+
expect(response.error).toContain("worker pipe write failed");
|
|
288
|
+
expect(backend.pendingRequests.size).toBe(0);
|
|
289
|
+
});
|
|
290
|
+
|
|
260
291
|
test("a failed write does not leak the pending request", async () => {
|
|
261
|
-
const backend =
|
|
292
|
+
const backend = newBackend();
|
|
262
293
|
backend.workerProc = brokenPipeProc();
|
|
263
294
|
|
|
264
295
|
await backend.sendRequest(["hello"]);
|
|
@@ -274,7 +305,7 @@ describe("releasing ownership of a worker", () => {
|
|
|
274
305
|
* strands it: no handle, no PID entry, invisible to every later reclaim.
|
|
275
306
|
*/
|
|
276
307
|
test("stdout ending kills the child before clearing workerProc", async () => {
|
|
277
|
-
const backend =
|
|
308
|
+
const backend = newBackend();
|
|
278
309
|
const proc = liveProc();
|
|
279
310
|
backend.workerProc = proc;
|
|
280
311
|
|
|
@@ -285,7 +316,7 @@ describe("releasing ownership of a worker", () => {
|
|
|
285
316
|
});
|
|
286
317
|
|
|
287
318
|
test("shutdown reaps the worker and resolves only once it exits", async () => {
|
|
288
|
-
const backend =
|
|
319
|
+
const backend = newBackend();
|
|
289
320
|
const proc = liveProc(555);
|
|
290
321
|
backend.workerProc = proc;
|
|
291
322
|
writeFileSync(getEmbedWorkerPidPath(), "555");
|
|
@@ -298,7 +329,7 @@ describe("releasing ownership of a worker", () => {
|
|
|
298
329
|
});
|
|
299
330
|
|
|
300
331
|
test("shutdown settles in-flight requests instead of hanging them", async () => {
|
|
301
|
-
const backend =
|
|
332
|
+
const backend = newBackend();
|
|
302
333
|
backend.workerProc = liveProc(556);
|
|
303
334
|
|
|
304
335
|
const inFlight = backend.sendRequest(["hello"]);
|
|
@@ -315,7 +346,7 @@ describe("releasing ownership of a worker", () => {
|
|
|
315
346
|
* pending requests are only settled after the wait.
|
|
316
347
|
*/
|
|
317
348
|
test("a worker that ignores SIGTERM is escalated to SIGKILL, not waited on forever", async () => {
|
|
318
|
-
const backend =
|
|
349
|
+
const backend = newBackend();
|
|
319
350
|
backend.terminateGraceMs = 50;
|
|
320
351
|
const signals: (string | undefined)[] = [];
|
|
321
352
|
const wedged = {
|
|
@@ -354,7 +385,7 @@ describe("termination that cannot be confirmed", () => {
|
|
|
354
385
|
}
|
|
355
386
|
|
|
356
387
|
test("keeps the PID publication instead of unpublishing a possibly-live child", async () => {
|
|
357
|
-
const backend =
|
|
388
|
+
const backend = newBackend();
|
|
358
389
|
backend.terminateGraceMs = 20;
|
|
359
390
|
const proc = wedgedProc();
|
|
360
391
|
backend.workerProc = proc;
|
|
@@ -370,7 +401,7 @@ describe("termination that cannot be confirmed", () => {
|
|
|
370
401
|
});
|
|
371
402
|
|
|
372
403
|
test("refuses to start a replacement while the child may still be alive", async () => {
|
|
373
|
-
const backend =
|
|
404
|
+
const backend = newBackend();
|
|
374
405
|
// No workerPath recorded, so liveness falls back to a PID probe. Our own
|
|
375
406
|
// PID is certainly alive, standing in for a child that will not die.
|
|
376
407
|
backend.unconfirmedWorker = process.pid;
|
|
@@ -381,7 +412,7 @@ describe("termination that cannot be confirmed", () => {
|
|
|
381
412
|
});
|
|
382
413
|
|
|
383
414
|
test("surfaces the incomplete teardown through embed() rather than failing silently", async () => {
|
|
384
|
-
const backend =
|
|
415
|
+
const backend = newBackend();
|
|
385
416
|
backend.unconfirmedWorker = process.pid;
|
|
386
417
|
|
|
387
418
|
await expect(backend.embed(["hello"])).rejects.toThrow(
|
|
@@ -401,7 +432,7 @@ describe("termination that cannot be confirmed", () => {
|
|
|
401
432
|
});
|
|
402
433
|
await Bun.sleep(400);
|
|
403
434
|
|
|
404
|
-
const backend =
|
|
435
|
+
const backend = newBackend();
|
|
405
436
|
backend.workerPath = scriptPath;
|
|
406
437
|
backend.unconfirmedWorker = proc.pid;
|
|
407
438
|
writeFileSync(getEmbedWorkerPidPath(), String(proc.pid));
|
|
@@ -440,7 +471,7 @@ describe("reclaiming before spawning a replacement", () => {
|
|
|
440
471
|
await Bun.sleep(400);
|
|
441
472
|
writeFileSync(getEmbedWorkerPidPath(), String(proc.pid));
|
|
442
473
|
|
|
443
|
-
const backend =
|
|
474
|
+
const backend = newBackend();
|
|
444
475
|
await backend.reclaimOwnedWorkers(scriptPath);
|
|
445
476
|
|
|
446
477
|
// Confirmed gone by the time reclaim returns, not merely signalled.
|
|
@@ -464,7 +495,7 @@ describe("reclaiming before spawning a replacement", () => {
|
|
|
464
495
|
spawned.push(proc);
|
|
465
496
|
await Bun.sleep(400);
|
|
466
497
|
|
|
467
|
-
const backend =
|
|
498
|
+
const backend = newBackend();
|
|
468
499
|
backend.terminateGraceMs = 300;
|
|
469
500
|
await backend.reclaimOwnedWorkers(scriptPath);
|
|
470
501
|
|
|
@@ -497,7 +528,7 @@ setTimeout(() => {}, 60_000);\n`,
|
|
|
497
528
|
expect(workers.length).toBe(1);
|
|
498
529
|
expect(workers[0].ppid).toBe(parent.pid);
|
|
499
530
|
|
|
500
|
-
const backend =
|
|
531
|
+
const backend = newBackend();
|
|
501
532
|
backend.terminateGraceMs = 300;
|
|
502
533
|
await backend.reclaimOwnedWorkers(scriptPath);
|
|
503
534
|
|
|
@@ -521,7 +552,7 @@ describe("shutdown versus in-flight initialization", () => {
|
|
|
521
552
|
* lets the initializer spawn a worker afterwards, with nobody left to reap it.
|
|
522
553
|
*/
|
|
523
554
|
test("shutdown waits for an in-flight init and reaps the worker it creates", async () => {
|
|
524
|
-
const backend =
|
|
555
|
+
const backend = newBackend();
|
|
525
556
|
const proc = liveProc(600);
|
|
526
557
|
|
|
527
558
|
let finishInit: () => void = () => {};
|
|
@@ -564,7 +595,7 @@ describe("parent exit orderings", () => {
|
|
|
564
595
|
* process's and left it alone. So it kills its own child on the way out.
|
|
565
596
|
*/
|
|
566
597
|
test("terminateNow kills the worker and releases the entry without waiting", () => {
|
|
567
|
-
const backend =
|
|
598
|
+
const backend = newBackend();
|
|
568
599
|
const signals: string[] = [];
|
|
569
600
|
const proc = {
|
|
570
601
|
pid: 909,
|
|
@@ -590,7 +621,7 @@ describe("parent exit orderings", () => {
|
|
|
590
621
|
* process while the child was still being adopted.
|
|
591
622
|
*/
|
|
592
623
|
test("shutdown does not inherit the model-load wait when init never settles", async () => {
|
|
593
|
-
const backend =
|
|
624
|
+
const backend = newBackend();
|
|
594
625
|
backend.terminateGraceMs = 50;
|
|
595
626
|
// An initialization that never resolves, as during a cold model load.
|
|
596
627
|
backend.initInFlight = new Promise<void>(() => {});
|
|
@@ -602,7 +633,7 @@ describe("parent exit orderings", () => {
|
|
|
602
633
|
});
|
|
603
634
|
|
|
604
635
|
test("a backend already shutting down refuses to spawn a replacement", async () => {
|
|
605
|
-
const backend =
|
|
636
|
+
const backend = newBackend();
|
|
606
637
|
backend.disposeRequested = true;
|
|
607
638
|
|
|
608
639
|
await expect(backend.embed(["hello"])).rejects.toThrow(/shutting down/);
|
|
@@ -621,7 +652,7 @@ describe("transient failures stay transient", () => {
|
|
|
621
652
|
const PERMANENT_FAILURE_SENTINEL = "Local embedding backend unavailable";
|
|
622
653
|
|
|
623
654
|
test("an unconfirmed termination does not claim permanent unavailability", async () => {
|
|
624
|
-
const backend =
|
|
655
|
+
const backend = newBackend();
|
|
625
656
|
backend.unconfirmedWorker = process.pid;
|
|
626
657
|
|
|
627
658
|
const err = await backend.ensureInitialized().catch((e: Error) => e);
|
|
@@ -631,7 +662,7 @@ describe("transient failures stay transient", () => {
|
|
|
631
662
|
});
|
|
632
663
|
|
|
633
664
|
test("shutting down does not claim permanent unavailability", async () => {
|
|
634
|
-
const backend =
|
|
665
|
+
const backend = newBackend();
|
|
635
666
|
backend.disposeRequested = true;
|
|
636
667
|
|
|
637
668
|
const err = await backend.embed(["hello"]).catch((e: Error) => e);
|
|
@@ -652,7 +683,7 @@ describe("audit regressions", () => {
|
|
|
652
683
|
* embed queued behind it.
|
|
653
684
|
*/
|
|
654
685
|
test("a startup failure against a wedged worker does not hang", async () => {
|
|
655
|
-
const backend =
|
|
686
|
+
const backend = newBackend();
|
|
656
687
|
backend.terminateGraceMs = 30;
|
|
657
688
|
const signals: (string | undefined)[] = [];
|
|
658
689
|
backend.workerProc = {
|
|
@@ -680,7 +711,7 @@ describe("audit regressions", () => {
|
|
|
680
711
|
* startup for the full two-minute timeout.
|
|
681
712
|
*/
|
|
682
713
|
test("terminateNow clears the stdout buffer so a retry can parse ready", () => {
|
|
683
|
-
const backend =
|
|
714
|
+
const backend = newBackend();
|
|
684
715
|
backend.stdoutBuffer = '{"id":5,"vect';
|
|
685
716
|
backend.workerProc = null;
|
|
686
717
|
|
|
@@ -707,7 +738,7 @@ describe("audit regressions", () => {
|
|
|
707
738
|
spawned.push(proc);
|
|
708
739
|
await Bun.sleep(400);
|
|
709
740
|
|
|
710
|
-
const backend =
|
|
741
|
+
const backend = newBackend();
|
|
711
742
|
backend.workerPath = scriptPath;
|
|
712
743
|
backend.workerProc = null; // spawned, never adopted
|
|
713
744
|
|
|
@@ -722,7 +753,7 @@ describe("audit regressions", () => {
|
|
|
722
753
|
* contract, so it must leave the instance able to start a replacement.
|
|
723
754
|
*/
|
|
724
755
|
test("terminateNow leaves the instance reusable", () => {
|
|
725
|
-
const backend =
|
|
756
|
+
const backend = newBackend();
|
|
726
757
|
backend.workerProc = null;
|
|
727
758
|
backend.stdoutReaderActive = true;
|
|
728
759
|
|
|
@@ -766,3 +797,97 @@ test("orphan reclaim outlasts the embedding reap it must not truncate", () => {
|
|
|
766
797
|
EMBEDDING_SHUTDOWN_BUDGET_MS + 1_000,
|
|
767
798
|
);
|
|
768
799
|
});
|
|
800
|
+
|
|
801
|
+
describe("one backend per model", () => {
|
|
802
|
+
test("forModel hands out one instance per model", () => {
|
|
803
|
+
const first = LocalEmbeddingBackend.forModel("per-model-a");
|
|
804
|
+
|
|
805
|
+
expect(LocalEmbeddingBackend.forModel("per-model-a")).toBe(first);
|
|
806
|
+
expect(LocalEmbeddingBackend.forModel("per-model-b")).not.toBe(first);
|
|
807
|
+
});
|
|
808
|
+
|
|
809
|
+
/**
|
|
810
|
+
* A backend-cache reset disposes the backend it evicts, and that disposal
|
|
811
|
+
* waits for in-flight embeds. The next embed needs the model again inside
|
|
812
|
+
* that window, and the embeds in flight keep their worker.
|
|
813
|
+
*/
|
|
814
|
+
test("needing the model again while a disposal is pending keeps the busy worker", () => {
|
|
815
|
+
const backend = LocalEmbeddingBackend.forModel(
|
|
816
|
+
"per-model-reopen",
|
|
817
|
+
) as Internals;
|
|
818
|
+
const proc = liveProc(7101);
|
|
819
|
+
backend.workerProc = proc;
|
|
820
|
+
backend.activeEmbeds = 1;
|
|
821
|
+
backend.dispose();
|
|
822
|
+
|
|
823
|
+
const again = LocalEmbeddingBackend.forModel(
|
|
824
|
+
"per-model-reopen",
|
|
825
|
+
) as Internals;
|
|
826
|
+
backend.activeEmbeds = 0;
|
|
827
|
+
backend.disposeIfIdle();
|
|
828
|
+
|
|
829
|
+
expect(again).toBe(backend);
|
|
830
|
+
expect(again.workerProc).toBe(proc);
|
|
831
|
+
expect(proc.killed).toBe(false);
|
|
832
|
+
});
|
|
833
|
+
|
|
834
|
+
test("a pending disposal nobody withdraws still releases the worker once idle", () => {
|
|
835
|
+
const backend = LocalEmbeddingBackend.forModel(
|
|
836
|
+
"per-model-dispose",
|
|
837
|
+
) as Internals;
|
|
838
|
+
const proc = liveProc(7102);
|
|
839
|
+
backend.workerProc = proc;
|
|
840
|
+
backend.activeEmbeds = 1;
|
|
841
|
+
backend.dispose();
|
|
842
|
+
|
|
843
|
+
backend.activeEmbeds = 0;
|
|
844
|
+
backend.disposeIfIdle();
|
|
845
|
+
|
|
846
|
+
expect(backend.workerProc).toBeNull();
|
|
847
|
+
expect(proc.killed).toBe(true);
|
|
848
|
+
});
|
|
849
|
+
|
|
850
|
+
/** The process is exiting, so nothing may spawn a worker behind the reap. */
|
|
851
|
+
test("a shut-down backend stays shut down", async () => {
|
|
852
|
+
const backend = LocalEmbeddingBackend.forModel("per-model-shutdown");
|
|
853
|
+
await backend.shutdown();
|
|
854
|
+
|
|
855
|
+
const again = LocalEmbeddingBackend.forModel("per-model-shutdown");
|
|
856
|
+
|
|
857
|
+
expect(again).toBe(backend);
|
|
858
|
+
await expect(again.embed(["hello"])).rejects.toThrow(/shutting down/);
|
|
859
|
+
});
|
|
860
|
+
|
|
861
|
+
/**
|
|
862
|
+
* The backend cache forgets a backend whose disposal is pending, so process
|
|
863
|
+
* teardown reaches it through the class's own list.
|
|
864
|
+
*/
|
|
865
|
+
test("shutdownAll reaps a busy backend no cache still holds", async () => {
|
|
866
|
+
const backend = LocalEmbeddingBackend.forModel(
|
|
867
|
+
"per-model-evicted",
|
|
868
|
+
) as Internals;
|
|
869
|
+
const proc = liveProc(7103);
|
|
870
|
+
backend.workerProc = proc;
|
|
871
|
+
backend.activeEmbeds = 1;
|
|
872
|
+
backend.dispose();
|
|
873
|
+
|
|
874
|
+
await LocalEmbeddingBackend.shutdownAll();
|
|
875
|
+
|
|
876
|
+
expect(proc.killed).toBe(true);
|
|
877
|
+
expect(backend.workerProc).toBeNull();
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
test("terminateAllNow kills a busy backend no cache still holds", () => {
|
|
881
|
+
const backend = LocalEmbeddingBackend.forModel(
|
|
882
|
+
"per-model-evicted-now",
|
|
883
|
+
) as Internals;
|
|
884
|
+
const proc = liveProc(7104);
|
|
885
|
+
backend.workerProc = proc;
|
|
886
|
+
backend.activeEmbeds = 1;
|
|
887
|
+
backend.dispose();
|
|
888
|
+
|
|
889
|
+
LocalEmbeddingBackend.terminateAllNow();
|
|
890
|
+
|
|
891
|
+
expect(proc.killed).toBe(true);
|
|
892
|
+
});
|
|
893
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process teardown reaches every local embedding worker the process owns.
|
|
3
|
+
*
|
|
4
|
+
* Kept in its own file: `shutdownEmbeddingBackends` latches the module into its
|
|
5
|
+
* shut-down state for the rest of the process.
|
|
6
|
+
*/
|
|
7
|
+
import { expect, test } from "bun:test";
|
|
8
|
+
|
|
9
|
+
import { AssistantConfigSchema } from "../../config/schema.js";
|
|
10
|
+
import {
|
|
11
|
+
clearEmbeddingBackendCache,
|
|
12
|
+
selectEmbeddingBackend,
|
|
13
|
+
shutdownEmbeddingBackends,
|
|
14
|
+
} from "./embedding-backend.js";
|
|
15
|
+
|
|
16
|
+
/** Reach past `private`, which is compile-time only, so tests drive real state. */
|
|
17
|
+
type Internals = any;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A backend-cache reset forgets a backend whose disposal is waiting on
|
|
21
|
+
* in-flight embeds, while that backend still holds a live worker.
|
|
22
|
+
*/
|
|
23
|
+
test("shutdown reaps a busy local worker the cache has already forgotten", async () => {
|
|
24
|
+
const config = AssistantConfigSchema.parse({
|
|
25
|
+
memory: { embeddings: { provider: "local" } },
|
|
26
|
+
});
|
|
27
|
+
const { backend } = await selectEmbeddingBackend(config);
|
|
28
|
+
const delegate = await (backend as Internals).getDelegate();
|
|
29
|
+
let killed = false;
|
|
30
|
+
delegate.workerProc = {
|
|
31
|
+
pid: 7201,
|
|
32
|
+
exited: new Promise<number>(() => {}),
|
|
33
|
+
kill() {
|
|
34
|
+
killed = true;
|
|
35
|
+
},
|
|
36
|
+
stdin: { write: () => 0, flush: () => 0 },
|
|
37
|
+
};
|
|
38
|
+
delegate.terminateGraceMs = 10;
|
|
39
|
+
delegate.activeEmbeds = 1;
|
|
40
|
+
|
|
41
|
+
clearEmbeddingBackendCache();
|
|
42
|
+
expect(killed).toBe(false);
|
|
43
|
+
|
|
44
|
+
await shutdownEmbeddingBackends();
|
|
45
|
+
|
|
46
|
+
expect(killed).toBe(true);
|
|
47
|
+
});
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
recordBillingSuccess,
|
|
17
17
|
} from "./embedding-billing-breaker.js";
|
|
18
18
|
import { GeminiEmbeddingBackend } from "./embedding-gemini.js";
|
|
19
|
+
import type { LocalEmbeddingBackend } from "./embedding-local.js";
|
|
19
20
|
import { OllamaEmbeddingBackend } from "./embedding-ollama.js";
|
|
20
21
|
import {
|
|
21
22
|
OpenAIEmbeddingBackend,
|
|
@@ -78,6 +79,13 @@ export const EMBEDDING_SHUTDOWN_BUDGET_MS = 8_000;
|
|
|
78
79
|
* the failure is contained and other embedding backends can be used instead.
|
|
79
80
|
*/
|
|
80
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The local backend class, once {@link LazyLocalEmbeddingBackend} has loaded it.
|
|
84
|
+
* It lists every local backend this process owns, which process teardown needs
|
|
85
|
+
* and {@link backendCache} cannot supply.
|
|
86
|
+
*/
|
|
87
|
+
let localEmbeddingBackends: typeof LocalEmbeddingBackend | null = null;
|
|
88
|
+
|
|
81
89
|
class LazyLocalEmbeddingBackend implements EmbeddingBackend {
|
|
82
90
|
readonly provider = "local" as const;
|
|
83
91
|
readonly model: string;
|
|
@@ -114,21 +122,9 @@ class LazyLocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
114
122
|
this.delegate?.dispose?.();
|
|
115
123
|
}
|
|
116
124
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
async sweepOwnedWorkers(): Promise<void> {
|
|
122
|
-
await this.delegate?.sweepOwnedWorkers?.();
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
async shutdown(): Promise<void> {
|
|
126
|
-
// A delegate under construction still ends up owning a worker, so settle
|
|
127
|
-
// the in-flight import before tearing down rather than skipping it.
|
|
128
|
-
if (!this.delegate && this.initPromise) {
|
|
129
|
-
await this.initPromise.catch(() => undefined);
|
|
130
|
-
}
|
|
131
|
-
await this.delegate?.shutdown?.();
|
|
125
|
+
/** Settles once a delegate under construction has been handed out or failed. */
|
|
126
|
+
async loaded(): Promise<void> {
|
|
127
|
+
await this.initPromise?.catch(() => undefined);
|
|
132
128
|
}
|
|
133
129
|
|
|
134
130
|
resetForRetry(): void {
|
|
@@ -146,7 +142,8 @@ class LazyLocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
146
142
|
try {
|
|
147
143
|
const { LocalEmbeddingBackend } =
|
|
148
144
|
await import("./embedding-local.js");
|
|
149
|
-
|
|
145
|
+
localEmbeddingBackends = LocalEmbeddingBackend;
|
|
146
|
+
this.delegate = LocalEmbeddingBackend.forModel(this.model);
|
|
150
147
|
return this.delegate;
|
|
151
148
|
} catch (err) {
|
|
152
149
|
localBackendBroken = true;
|
|
@@ -270,7 +267,7 @@ export function clearEmbeddingBackendCache(): void {
|
|
|
270
267
|
}
|
|
271
268
|
|
|
272
269
|
/**
|
|
273
|
-
* Tear down every
|
|
270
|
+
* Tear down every local embedding worker this process owns and empty the caches.
|
|
274
271
|
*
|
|
275
272
|
* Called on daemon shutdown so process-owned embedding workers exit with their
|
|
276
273
|
* owner instead of being orphaned. `clearEmbeddingBackendCache()` is the
|
|
@@ -279,24 +276,18 @@ export function clearEmbeddingBackendCache(): void {
|
|
|
279
276
|
*/
|
|
280
277
|
export async function shutdownEmbeddingBackends(): Promise<void> {
|
|
281
278
|
embeddingBackendsShutDown = true;
|
|
282
|
-
const
|
|
279
|
+
const loading = [...backendCache.values()].filter(
|
|
280
|
+
(backend) => backend instanceof LazyLocalEmbeddingBackend,
|
|
281
|
+
);
|
|
283
282
|
backendCache.clear();
|
|
284
283
|
vectorCache.clear();
|
|
285
284
|
vectorCacheBytes = 0;
|
|
286
285
|
backendDimCache.clear();
|
|
287
286
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
await backend.sweepOwnedWorkers?.();
|
|
293
|
-
} catch (err) {
|
|
294
|
-
log.warn(
|
|
295
|
-
{ err, provider: backend.provider, model: backend.model },
|
|
296
|
-
"Failed to shut down embedding backend",
|
|
297
|
-
);
|
|
298
|
-
}
|
|
299
|
-
}),
|
|
287
|
+
// A delegate under construction still ends up owning a worker, so let it
|
|
288
|
+
// join the class's list before tearing that list down.
|
|
289
|
+
const teardown = Promise.all(loading.map((backend) => backend.loaded())).then(
|
|
290
|
+
() => localEmbeddingBackends?.shutdownAll(),
|
|
300
291
|
);
|
|
301
292
|
|
|
302
293
|
const timedOut = Symbol("timeout");
|
|
@@ -313,7 +304,7 @@ export async function shutdownEmbeddingBackends(): Promise<void> {
|
|
|
313
304
|
}
|
|
314
305
|
|
|
315
306
|
/**
|
|
316
|
-
* SIGKILL every
|
|
307
|
+
* SIGKILL every local embedding worker this process owns, synchronously.
|
|
317
308
|
*
|
|
318
309
|
* For a process that must exit immediately and cannot run the graceful
|
|
319
310
|
* teardown: the memory worker on PID-file eviction, where staying alive to reap
|
|
@@ -322,16 +313,7 @@ export async function shutdownEmbeddingBackends(): Promise<void> {
|
|
|
322
313
|
* sweep has passed, leaving two workers alive (JARVIS-1125).
|
|
323
314
|
*/
|
|
324
315
|
export function terminateEmbeddingWorkersNow(): void {
|
|
325
|
-
|
|
326
|
-
try {
|
|
327
|
-
backend.terminateNow?.();
|
|
328
|
-
} catch (err) {
|
|
329
|
-
log.warn(
|
|
330
|
-
{ err, provider: backend.provider, model: backend.model },
|
|
331
|
-
"Failed to terminate embedding worker",
|
|
332
|
-
);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
316
|
+
localEmbeddingBackends?.terminateAllNow();
|
|
335
317
|
}
|
|
336
318
|
|
|
337
319
|
/** Reset the sticky local-backend failure flag without evicting live backends. */
|
|
@@ -7,6 +7,8 @@ import {
|
|
|
7
7
|
} from "node:fs";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
|
|
10
|
+
import type { Subprocess } from "bun";
|
|
11
|
+
|
|
10
12
|
import { getIsContainerized } from "../../config/env-registry.js";
|
|
11
13
|
import { getLogger } from "../../util/logger.js";
|
|
12
14
|
import {
|
|
@@ -22,6 +24,7 @@ import {
|
|
|
22
24
|
listWorkerProcesses,
|
|
23
25
|
pid1OwnsWorkers,
|
|
24
26
|
} from "../../util/worker-ownership.js";
|
|
27
|
+
import { writeWorkerLine } from "../../util/worker-pipe.js";
|
|
25
28
|
import { EmbeddingRuntimeManager } from "./embedding-runtime-manager.js";
|
|
26
29
|
import {
|
|
27
30
|
type EmbeddingBackend,
|
|
@@ -106,9 +109,7 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
106
109
|
readonly provider = "local" as const;
|
|
107
110
|
readonly model: string;
|
|
108
111
|
|
|
109
|
-
|
|
110
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
111
|
-
private workerProc: any = null;
|
|
112
|
+
private workerProc: Subprocess<"pipe", "pipe", "pipe"> | null = null;
|
|
112
113
|
private stdoutBuffer = "";
|
|
113
114
|
private requestCounter = 0;
|
|
114
115
|
private pendingRequests = new Map<
|
|
@@ -119,7 +120,10 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
119
120
|
>();
|
|
120
121
|
private stdoutReaderActive = false;
|
|
121
122
|
private activeEmbeds = 0;
|
|
123
|
+
/** Release the worker once idle. {@link forModel} withdraws the request. */
|
|
122
124
|
private disposeRequested = false;
|
|
125
|
+
/** Set by {@link shutdown}. The process is exiting, so nothing reopens this. */
|
|
126
|
+
private processExiting = false;
|
|
123
127
|
|
|
124
128
|
private readonly initGuard = new PromiseGuard<void>();
|
|
125
129
|
private initInFlight: Promise<void> | null = null;
|
|
@@ -130,7 +134,67 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
130
134
|
/** Overridable so tests can exercise the escalation path without the wait. */
|
|
131
135
|
private terminateGraceMs = WORKER_TERMINATE_GRACE_MS;
|
|
132
136
|
|
|
133
|
-
|
|
137
|
+
/**
|
|
138
|
+
* Every backend this process has handed out, one per model.
|
|
139
|
+
*
|
|
140
|
+
* {@link reclaimOwnedWorkers} terminates a same-model worker parented to this
|
|
141
|
+
* process that the instance holds no handle for. That is sound only while no
|
|
142
|
+
* other instance in this process can hold that worker, so instances come from
|
|
143
|
+
* {@link forModel} and are never constructed by callers.
|
|
144
|
+
*
|
|
145
|
+
* This is also the complete list of workers the process owns. The backend
|
|
146
|
+
* cache is not: it forgets a backend whose disposal is still pending.
|
|
147
|
+
*/
|
|
148
|
+
private static readonly byModel = new Map<string, LocalEmbeddingBackend>();
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The backend for `model`. A pending {@link dispose} is withdrawn, so embeds
|
|
152
|
+
* still running on the instance keep their worker.
|
|
153
|
+
*/
|
|
154
|
+
static forModel(model: string): LocalEmbeddingBackend {
|
|
155
|
+
let backend = LocalEmbeddingBackend.byModel.get(model);
|
|
156
|
+
if (!backend) {
|
|
157
|
+
backend = new LocalEmbeddingBackend(model);
|
|
158
|
+
LocalEmbeddingBackend.byModel.set(model, backend);
|
|
159
|
+
}
|
|
160
|
+
if (!backend.processExiting) {
|
|
161
|
+
backend.disposeRequested = false;
|
|
162
|
+
}
|
|
163
|
+
return backend;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** {@link shutdown} every backend, then reap any worker still parented here. */
|
|
167
|
+
static async shutdownAll(): Promise<void> {
|
|
168
|
+
await Promise.all(
|
|
169
|
+
[...LocalEmbeddingBackend.byModel.values()].map(async (backend) => {
|
|
170
|
+
try {
|
|
171
|
+
await backend.shutdown();
|
|
172
|
+
await backend.sweepOwnedWorkers();
|
|
173
|
+
} catch (err) {
|
|
174
|
+
log.warn(
|
|
175
|
+
{ err, model: backend.model },
|
|
176
|
+
"Failed to shut down local embedding backend",
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
}),
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** {@link terminateNow} every backend. */
|
|
184
|
+
static terminateAllNow(): void {
|
|
185
|
+
for (const backend of LocalEmbeddingBackend.byModel.values()) {
|
|
186
|
+
try {
|
|
187
|
+
backend.terminateNow();
|
|
188
|
+
} catch (err) {
|
|
189
|
+
log.warn(
|
|
190
|
+
{ err, model: backend.model },
|
|
191
|
+
"Failed to terminate local embedding worker",
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
private constructor(model: string) {
|
|
134
198
|
this.model = model;
|
|
135
199
|
}
|
|
136
200
|
|
|
@@ -193,19 +257,9 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
193
257
|
}
|
|
194
258
|
this.pendingRequests.set(id, { resolve });
|
|
195
259
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
// which tears down the whole process (JARVIS-1125).
|
|
200
|
-
//
|
|
201
|
-
// The guard covers `write` as well as `flush`: both are synchronous on
|
|
202
|
-
// Bun's FileSink, and `write` is the call that raises the broken pipe.
|
|
203
|
-
try {
|
|
204
|
-
proc.stdin.write(JSON.stringify({ id, texts }) + "\n");
|
|
205
|
-
proc.stdin.flush();
|
|
206
|
-
} catch (err) {
|
|
207
|
-
this.failPendingRequest(id, err);
|
|
208
|
-
}
|
|
260
|
+
writeWorkerLine(proc.stdin, JSON.stringify({ id, texts }), (err) =>
|
|
261
|
+
this.failPendingRequest(id, err),
|
|
262
|
+
);
|
|
209
263
|
});
|
|
210
264
|
}
|
|
211
265
|
|
|
@@ -929,6 +983,7 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
929
983
|
* the OS confirms the worker is gone.
|
|
930
984
|
*/
|
|
931
985
|
async shutdown(): Promise<void> {
|
|
986
|
+
this.processExiting = true;
|
|
932
987
|
this.disposeRequested = true;
|
|
933
988
|
|
|
934
989
|
// An initialization already in flight has not necessarily assigned
|
|
@@ -113,25 +113,6 @@ export interface EmbeddingBackend {
|
|
|
113
113
|
options?: EmbeddingRequestOptions,
|
|
114
114
|
): Promise<number[][]>;
|
|
115
115
|
dispose?(): void;
|
|
116
|
-
/**
|
|
117
|
-
* Deterministic teardown for daemon shutdown: release any OS resources the
|
|
118
|
-
* backend owns (e.g. a worker subprocess) and resolve only once they are
|
|
119
|
-
* confirmed gone. Unlike {@link dispose}, which may defer while embeds are
|
|
120
|
-
* in flight, this must not leave a child process behind.
|
|
121
|
-
*/
|
|
122
|
-
shutdown?(): Promise<void>;
|
|
123
|
-
/**
|
|
124
|
-
* Release owned OS resources synchronously, for a process that must exit this
|
|
125
|
-
* tick. Unlike {@link shutdown} it cannot wait for confirmation, so it uses
|
|
126
|
-
* the uncatchable signal rather than the graceful one.
|
|
127
|
-
*/
|
|
128
|
-
terminateNow?(): void;
|
|
129
|
-
/**
|
|
130
|
-
* Reap any owned child still attributable to this process, without needing a
|
|
131
|
-
* handle for it. Backstop for a teardown that could not resolve its handle in
|
|
132
|
-
* time.
|
|
133
|
-
*/
|
|
134
|
-
sweepOwnedWorkers?(): Promise<void>;
|
|
135
116
|
}
|
|
136
117
|
|
|
137
118
|
/**
|
|
@@ -278,7 +278,7 @@ async function startOutboundTelegram(
|
|
|
278
278
|
success: false,
|
|
279
279
|
error: "already_bound",
|
|
280
280
|
message:
|
|
281
|
-
"A guardian is already bound for this channel. Set rebind: true to
|
|
281
|
+
"A guardian is already bound for this channel. To verify a different account, revoke the existing binding first. Set rebind: true only to verify the same account again.",
|
|
282
282
|
channel,
|
|
283
283
|
};
|
|
284
284
|
}
|
|
@@ -820,7 +820,7 @@ async function startOutboundTextChannel(
|
|
|
820
820
|
success: false,
|
|
821
821
|
error: "already_bound",
|
|
822
822
|
message:
|
|
823
|
-
"A guardian is already bound for this channel. Set rebind: true to
|
|
823
|
+
"A guardian is already bound for this channel. To verify a different account, revoke the existing binding first. Set rebind: true only to verify the same account again.",
|
|
824
824
|
channel,
|
|
825
825
|
};
|
|
826
826
|
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { writeWorkerLine } from "./worker-pipe.js";
|
|
4
|
+
|
|
5
|
+
/** Comfortably larger than any OS pipe buffer, so the write cannot complete. */
|
|
6
|
+
const OVERSIZED_LINE = "x".repeat(4 * 1024 * 1024);
|
|
7
|
+
|
|
8
|
+
const spawned: { kill(signal?: NodeJS.Signals): void }[] = [];
|
|
9
|
+
|
|
10
|
+
afterEach(() => {
|
|
11
|
+
for (const proc of spawned.splice(0)) {
|
|
12
|
+
try {
|
|
13
|
+
proc.kill("SIGKILL");
|
|
14
|
+
} catch {
|
|
15
|
+
/* already gone */
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
/** A live child that never reads stdin, the way a worker busy in ONNX does not. */
|
|
21
|
+
function spawnBusyWorker() {
|
|
22
|
+
const proc = Bun.spawn({
|
|
23
|
+
cmd: [process.execPath, "-e", "setTimeout(() => {}, 60_000)"],
|
|
24
|
+
windowsHide: true,
|
|
25
|
+
stdin: "pipe",
|
|
26
|
+
stdout: "ignore",
|
|
27
|
+
stderr: "ignore",
|
|
28
|
+
});
|
|
29
|
+
spawned.push(proc);
|
|
30
|
+
return proc;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe("writeWorkerLine", () => {
|
|
34
|
+
/**
|
|
35
|
+
* The production failure: the payload outruns the pipe buffer, Bun returns a
|
|
36
|
+
* pending promise, and the worker dies before draining it. Unobserved, that
|
|
37
|
+
* rejection is an `unhandledRejection`, which `bun test` reports as a failure
|
|
38
|
+
* of this test.
|
|
39
|
+
*/
|
|
40
|
+
test("reports a write still pending when the worker dies", async () => {
|
|
41
|
+
const proc = spawnBusyWorker();
|
|
42
|
+
const failure = Promise.withResolvers<unknown>();
|
|
43
|
+
|
|
44
|
+
writeWorkerLine(proc.stdin, OVERSIZED_LINE, failure.resolve);
|
|
45
|
+
proc.kill("SIGKILL");
|
|
46
|
+
|
|
47
|
+
const err = await failure.promise;
|
|
48
|
+
expect(err).toBeInstanceOf(Error);
|
|
49
|
+
expect((err as NodeJS.ErrnoException).code).toBe("EPIPE");
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("reports a synchronous throw", () => {
|
|
53
|
+
const failures: unknown[] = [];
|
|
54
|
+
const epipe = Object.assign(new Error("EPIPE: broken pipe, write"), {
|
|
55
|
+
code: "EPIPE",
|
|
56
|
+
});
|
|
57
|
+
const stdin = {
|
|
58
|
+
write() {
|
|
59
|
+
throw epipe;
|
|
60
|
+
},
|
|
61
|
+
flush: () => 0,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
writeWorkerLine(stdin, "line", (err) => failures.push(err));
|
|
65
|
+
|
|
66
|
+
expect(failures).toEqual([epipe]);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
/** `write` and `flush` both report the same broken pipe. */
|
|
70
|
+
test("reports one failure when write and flush both reject", async () => {
|
|
71
|
+
const failures: unknown[] = [];
|
|
72
|
+
const epipe = new Error("EPIPE: broken pipe, send");
|
|
73
|
+
const stdin = {
|
|
74
|
+
write: () => Promise.reject(epipe),
|
|
75
|
+
flush: () => Promise.reject(epipe),
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
writeWorkerLine(stdin, "line", (err) => failures.push(err));
|
|
79
|
+
await Bun.sleep(0);
|
|
80
|
+
|
|
81
|
+
expect(failures).toEqual([epipe]);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("stays silent when the line is delivered", async () => {
|
|
85
|
+
const failures: unknown[] = [];
|
|
86
|
+
const proc = Bun.spawn({
|
|
87
|
+
cmd: [process.execPath, "-e", "process.stdin.on('data', () => {})"],
|
|
88
|
+
windowsHide: true,
|
|
89
|
+
stdin: "pipe",
|
|
90
|
+
stdout: "ignore",
|
|
91
|
+
stderr: "ignore",
|
|
92
|
+
});
|
|
93
|
+
spawned.push(proc);
|
|
94
|
+
|
|
95
|
+
writeWorkerLine(proc.stdin, "hello", (err) => failures.push(err));
|
|
96
|
+
await Bun.sleep(50);
|
|
97
|
+
|
|
98
|
+
expect(failures).toEqual([]);
|
|
99
|
+
});
|
|
100
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { FileSink } from "bun";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Write one line to a worker subprocess's stdin, reporting a broken pipe to
|
|
5
|
+
* `onFailure` however Bun surfaces it.
|
|
6
|
+
*
|
|
7
|
+
* `FileSink.write` and `FileSink.flush` return `number | Promise<number>`.
|
|
8
|
+
* While the pipe has room they complete synchronously. Once a payload outruns
|
|
9
|
+
* the pipe buffer, which is what happens when the worker is busy and not
|
|
10
|
+
* draining stdin, they return a pending promise instead, and that promise
|
|
11
|
+
* rejects with EPIPE if the worker dies before the write completes. A
|
|
12
|
+
* `try/catch` sees only the synchronous half: the rejection escapes it, reaches
|
|
13
|
+
* the process's `unhandledRejection` handler, and in the daemon that handler
|
|
14
|
+
* shuts the whole process down.
|
|
15
|
+
*
|
|
16
|
+
* `onFailure` runs at most once per call, even though `write` and `flush` can
|
|
17
|
+
* each report the same broken pipe.
|
|
18
|
+
*/
|
|
19
|
+
export function writeWorkerLine(
|
|
20
|
+
stdin: Pick<FileSink, "write" | "flush">,
|
|
21
|
+
line: string,
|
|
22
|
+
onFailure: (err: unknown) => void,
|
|
23
|
+
): void {
|
|
24
|
+
let failed = false;
|
|
25
|
+
const fail = (err: unknown): void => {
|
|
26
|
+
if (failed) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
failed = true;
|
|
30
|
+
onFailure(err);
|
|
31
|
+
};
|
|
32
|
+
const observe = (result: number | Promise<number>): void => {
|
|
33
|
+
if (result instanceof Promise) {
|
|
34
|
+
result.catch(fail);
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
try {
|
|
39
|
+
observe(stdin.write(line + "\n"));
|
|
40
|
+
observe(stdin.flush());
|
|
41
|
+
} catch (err) {
|
|
42
|
+
fail(err);
|
|
43
|
+
}
|
|
44
|
+
}
|