@vellumai/assistant 0.11.2-dev.202608050507.7e832db → 0.11.2-dev.202608050900.b357993
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openapi.yaml +0 -2
- package/package.json +1 -1
- package/src/cli/commands/notifications.ts +3 -6
- package/src/heartbeat/heartbeat-service.ts +0 -1
- package/src/notifications/emit-signal.ts +22 -10
- package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
- package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
- package/src/runtime/background-job-runner.ts +5 -27
- package/src/runtime/routes/notification-routes.ts +0 -14
- package/src/util/__tests__/sqlite-retry.test.ts +112 -0
- package/src/notifications/deferred-emit.ts +0 -147
package/openapi.yaml
CHANGED
|
@@ -21920,8 +21920,6 @@ paths:
|
|
|
21920
21920
|
type: string
|
|
21921
21921
|
throwOnError:
|
|
21922
21922
|
type: boolean
|
|
21923
|
-
originatingConversationId:
|
|
21924
|
-
type: string
|
|
21925
21923
|
required:
|
|
21926
21924
|
- sourceEventName
|
|
21927
21925
|
- sourceChannel
|
package/package.json
CHANGED
|
@@ -258,9 +258,9 @@ export function registerNotificationsCommand(program: Command): void {
|
|
|
258
258
|
return;
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
-
// Picks up __CONVERSATION_ID / __SKILL_CONTEXT_JSON env vars
|
|
262
|
-
//
|
|
263
|
-
//
|
|
261
|
+
// Picks up __CONVERSATION_ID / __SKILL_CONTEXT_JSON env vars so
|
|
262
|
+
// a send invoked from inside an agent turn resolves the
|
|
263
|
+
// conversation it originated from.
|
|
264
264
|
const originatingConversationId = tryResolveConversationId();
|
|
265
265
|
|
|
266
266
|
// The signal's `sourceContextId` doubles as the home-feed's
|
|
@@ -306,9 +306,6 @@ export function registerNotificationsCommand(program: Command): void {
|
|
|
306
306
|
...(conversationId
|
|
307
307
|
? { conversationAffinityHint: { vellum: conversationId } }
|
|
308
308
|
: {}),
|
|
309
|
-
...(originatingConversationId
|
|
310
|
-
? { originatingConversationId }
|
|
311
|
-
: {}),
|
|
312
309
|
throwOnError: true,
|
|
313
310
|
},
|
|
314
311
|
});
|
|
@@ -848,7 +848,6 @@ export class HeartbeatService {
|
|
|
848
848
|
callSite: "heartbeatAgent",
|
|
849
849
|
timeoutMs: getConfig().timeouts.backgroundTurnTimeoutSec * 1000,
|
|
850
850
|
origin: "heartbeat",
|
|
851
|
-
deferNotifications: true,
|
|
852
851
|
onConversationCreated: (newConversationId) => {
|
|
853
852
|
conversationId = newConversationId;
|
|
854
853
|
broadcastMessage({
|
|
@@ -21,6 +21,7 @@ import type { ConversationCreateType } from "../persistence/conversation-types.j
|
|
|
21
21
|
import { isPlatformClientConfigured } from "../platform/client.js";
|
|
22
22
|
import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
23
23
|
import { getLogger } from "../util/logger.js";
|
|
24
|
+
import { withSqliteRetry } from "../util/sqlite-retry.js";
|
|
24
25
|
import { VellumAdapter } from "./adapters/macos.js";
|
|
25
26
|
import { PlatformPushAdapter } from "./adapters/platform.js";
|
|
26
27
|
import { SlackAdapter } from "./adapters/slack.js";
|
|
@@ -263,16 +264,27 @@ export async function emitNotificationSignal<TEventName extends string>(
|
|
|
263
264
|
};
|
|
264
265
|
|
|
265
266
|
try {
|
|
266
|
-
// Step 1: Persist the event
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
267
|
+
// Step 1: Persist the event. The insert contends with other writers on
|
|
268
|
+
// the shared database (notably the memory worker's bulk writes), and a
|
|
269
|
+
// lost signal is unrecoverable (the producer has already returned by
|
|
270
|
+
// the time contention surfaces), so transient `SQLITE_BUSY`/`SQLITE_IOERR`
|
|
271
|
+
// rides the shared retry helper instead of failing the pipeline.
|
|
272
|
+
const eventRow = await withSqliteRetry(
|
|
273
|
+
() =>
|
|
274
|
+
createEvent({
|
|
275
|
+
id: signalId,
|
|
276
|
+
sourceEventName: params.sourceEventName,
|
|
277
|
+
sourceChannel: params.sourceChannel,
|
|
278
|
+
sourceContextId: params.sourceContextId,
|
|
279
|
+
attentionHints: params.attentionHints,
|
|
280
|
+
payload: (params.contextPayload ?? {}) as Record<string, unknown>,
|
|
281
|
+
dedupeKey: params.dedupeKey,
|
|
282
|
+
}),
|
|
283
|
+
{
|
|
284
|
+
op: "notification-create-event",
|
|
285
|
+
context: { signalId, sourceEventName: params.sourceEventName },
|
|
286
|
+
},
|
|
287
|
+
);
|
|
276
288
|
|
|
277
289
|
if (!eventRow) {
|
|
278
290
|
log.info(
|
|
@@ -31,11 +31,16 @@ const log = getLogger("memory-v2-consolidate");
|
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
33
|
* Hard timeout for the consolidation run. Consolidation reads the buffer,
|
|
34
|
-
* rewrites several files, and re-encodes essentials/threads:
|
|
35
|
-
*
|
|
36
|
-
*
|
|
34
|
+
* rewrites several files, and re-encodes essentials/threads: on a mature
|
|
35
|
+
* corpus a full pass legitimately runs ~20 minutes under cross-process write
|
|
36
|
+
* contention, so the bound leaves headroom above that while still keeping a
|
|
37
|
+
* stuck run from pinning the worker indefinitely. Matches the default
|
|
38
|
+
* `timeouts.backgroundTurnTimeoutSec` budget for heartbeat/background turns.
|
|
39
|
+
* Overrunning it marks the run failed and skips its follow-up jobs (reembeds,
|
|
40
|
+
* v3 maintenance) even though the agent's file edits land, so the bound must
|
|
41
|
+
* comfortably exceed a healthy run's duration.
|
|
37
42
|
*/
|
|
38
|
-
export const CONSOLIDATION_TIMEOUT_MS =
|
|
43
|
+
export const CONSOLIDATION_TIMEOUT_MS = 30 * 60 * 1000;
|
|
39
44
|
|
|
40
45
|
/**
|
|
41
46
|
* Age past which a lock held by an apparently-live PID is taken over anyway.
|
|
@@ -96,8 +96,6 @@ mock.module("../pre-first-message-gate.js", () => ({
|
|
|
96
96
|
|
|
97
97
|
// Import after mocks are in place.
|
|
98
98
|
const { runBackgroundJob } = await import("../background-job-runner.js");
|
|
99
|
-
const { bufferIfDeferred, resetDeferredForTest } =
|
|
100
|
-
await import("../../notifications/deferred-emit.js");
|
|
101
99
|
|
|
102
100
|
// ── Shared fixtures ──────────────────────────────────────────────────
|
|
103
101
|
|
|
@@ -125,7 +123,6 @@ beforeEach(() => {
|
|
|
125
123
|
processMessageCalls.length = 0;
|
|
126
124
|
emitCalls.length = 0;
|
|
127
125
|
addMessageCalls.length = 0;
|
|
128
|
-
resetDeferredForTest();
|
|
129
126
|
preFirstMessageGateOpen = true;
|
|
130
127
|
processMessageImpl = async () => ({ messageId: "msg-1" });
|
|
131
128
|
emitImpl = async () => ({
|
|
@@ -290,22 +287,6 @@ describe("runBackgroundJob", () => {
|
|
|
290
287
|
).toBe("model_provider");
|
|
291
288
|
});
|
|
292
289
|
|
|
293
|
-
test("non-throwing turn failure with deferNotifications drops buffered notifications", async () => {
|
|
294
|
-
processMessageImpl = async () => ({
|
|
295
|
-
messageId: "msg-failed-turn",
|
|
296
|
-
turnFailure: { failureCode: "provider_error" },
|
|
297
|
-
});
|
|
298
|
-
|
|
299
|
-
const result = await runBackgroundJob(
|
|
300
|
-
baseOpts({ deferNotifications: true }),
|
|
301
|
-
);
|
|
302
|
-
|
|
303
|
-
expect(result.ok).toBe(false);
|
|
304
|
-
// Only the runner's failure signal emits — no committed "success" signal.
|
|
305
|
-
expect(emitCalls).toHaveLength(1);
|
|
306
|
-
expect(emitCalls[0].sourceEventName).toBe("activity.failed");
|
|
307
|
-
});
|
|
308
|
-
|
|
309
290
|
test("timeout: returns ok=false with errorKind=timeout and emits activity.failed", async () => {
|
|
310
291
|
// Never resolve — force timeout to win the race.
|
|
311
292
|
processMessageImpl = () => new Promise(() => {});
|
|
@@ -464,129 +445,4 @@ describe("runBackgroundJob", () => {
|
|
|
464
445
|
expect(processMessageCalls).toHaveLength(1);
|
|
465
446
|
});
|
|
466
447
|
});
|
|
467
|
-
|
|
468
|
-
describe("deferNotifications", () => {
|
|
469
|
-
function buildSkillNotificationParams(message: string) {
|
|
470
|
-
return {
|
|
471
|
-
sourceEventName: "assistant.share",
|
|
472
|
-
sourceChannel: "assistant_tool" as const,
|
|
473
|
-
sourceContextId: "skill-ctx",
|
|
474
|
-
contextPayload: { requestedMessage: message },
|
|
475
|
-
attentionHints: {
|
|
476
|
-
requiresAction: false,
|
|
477
|
-
urgency: "low" as const,
|
|
478
|
-
isAsyncBackground: false,
|
|
479
|
-
visibleInSourceNow: false,
|
|
480
|
-
},
|
|
481
|
-
};
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
test("success path commits buffered in-band notifications", async () => {
|
|
485
|
-
processMessageImpl = async () => {
|
|
486
|
-
// Stand in for the IPC route's bufferIfDeferred call when the model
|
|
487
|
-
// invokes `notifications send` mid-turn.
|
|
488
|
-
const buffered = bufferIfDeferred(
|
|
489
|
-
STUB_CONVERSATION_ID,
|
|
490
|
-
buildSkillNotificationParams("all green"),
|
|
491
|
-
);
|
|
492
|
-
expect(buffered).not.toBeNull();
|
|
493
|
-
expect(buffered!.dispatched).toBe(false);
|
|
494
|
-
return { messageId: "msg-success" };
|
|
495
|
-
};
|
|
496
|
-
|
|
497
|
-
const result = await runBackgroundJob(
|
|
498
|
-
baseOpts({ deferNotifications: true }),
|
|
499
|
-
);
|
|
500
|
-
|
|
501
|
-
expect(result.ok).toBe(true);
|
|
502
|
-
// Commit flushed the buffered notification through emitNotificationSignal.
|
|
503
|
-
const successEmits = emitCalls.filter(
|
|
504
|
-
(e) => e.sourceEventName !== "activity.failed",
|
|
505
|
-
);
|
|
506
|
-
expect(successEmits).toHaveLength(1);
|
|
507
|
-
expect(successEmits[0].sourceEventName).toBe("assistant.share");
|
|
508
|
-
});
|
|
509
|
-
|
|
510
|
-
// Regression for the PR #31216 Codex P1 finding: a heartbeat that calls
|
|
511
|
-
// the notifications skill and then times out must not leave a "success"
|
|
512
|
-
// notification standing alongside the runner's `activity.failed` emit.
|
|
513
|
-
test("timeout drops buffered in-band notifications; only activity.failed emits", async () => {
|
|
514
|
-
processMessageImpl = () => {
|
|
515
|
-
bufferIfDeferred(
|
|
516
|
-
STUB_CONVERSATION_ID,
|
|
517
|
-
buildSkillNotificationParams("premature success"),
|
|
518
|
-
);
|
|
519
|
-
return new Promise(() => {});
|
|
520
|
-
};
|
|
521
|
-
|
|
522
|
-
const result = await runBackgroundJob(
|
|
523
|
-
baseOpts({ deferNotifications: true, timeoutMs: 30 }),
|
|
524
|
-
);
|
|
525
|
-
|
|
526
|
-
expect(result.ok).toBe(false);
|
|
527
|
-
expect(result.errorKind).toBe("timeout");
|
|
528
|
-
// Only the runner's failure signal makes it out — the buffered
|
|
529
|
-
// "success" notification is discarded.
|
|
530
|
-
expect(emitCalls).toHaveLength(1);
|
|
531
|
-
expect(emitCalls[0].sourceEventName).toBe("activity.failed");
|
|
532
|
-
});
|
|
533
|
-
|
|
534
|
-
test("thrown exception also drops buffered notifications", async () => {
|
|
535
|
-
processMessageImpl = async () => {
|
|
536
|
-
bufferIfDeferred(
|
|
537
|
-
STUB_CONVERSATION_ID,
|
|
538
|
-
buildSkillNotificationParams("doomed"),
|
|
539
|
-
);
|
|
540
|
-
throw new Error("kaboom");
|
|
541
|
-
};
|
|
542
|
-
|
|
543
|
-
const result = await runBackgroundJob(
|
|
544
|
-
baseOpts({ deferNotifications: true }),
|
|
545
|
-
);
|
|
546
|
-
|
|
547
|
-
expect(result.ok).toBe(false);
|
|
548
|
-
expect(emitCalls).toHaveLength(1);
|
|
549
|
-
expect(emitCalls[0].sourceEventName).toBe("activity.failed");
|
|
550
|
-
});
|
|
551
|
-
|
|
552
|
-
// Regression: after timeout, `processMessage` keeps running and may
|
|
553
|
-
// emit a late skill call. The tombstone must swallow it instead of
|
|
554
|
-
// letting it bypass the buffer and reach the dispatch pipeline.
|
|
555
|
-
test("late skill call after timeout is swallowed by the tombstone", async () => {
|
|
556
|
-
processMessageImpl = () => new Promise(() => {});
|
|
557
|
-
|
|
558
|
-
const result = await runBackgroundJob(
|
|
559
|
-
baseOpts({ deferNotifications: true, timeoutMs: 20 }),
|
|
560
|
-
);
|
|
561
|
-
expect(result.ok).toBe(false);
|
|
562
|
-
|
|
563
|
-
const late = bufferIfDeferred(
|
|
564
|
-
STUB_CONVERSATION_ID,
|
|
565
|
-
buildSkillNotificationParams("post-timeout"),
|
|
566
|
-
);
|
|
567
|
-
expect(late).not.toBeNull();
|
|
568
|
-
expect(late!.dispatched).toBe(false);
|
|
569
|
-
expect(late!.reason).toMatch(/did not complete/);
|
|
570
|
-
|
|
571
|
-
// Only the runner's failure signal made it out.
|
|
572
|
-
expect(emitCalls).toHaveLength(1);
|
|
573
|
-
expect(emitCalls[0].sourceEventName).toBe("activity.failed");
|
|
574
|
-
});
|
|
575
|
-
|
|
576
|
-
test("without deferNotifications, bufferIfDeferred is a no-op", async () => {
|
|
577
|
-
processMessageImpl = async () => {
|
|
578
|
-
const buffered = bufferIfDeferred(
|
|
579
|
-
STUB_CONVERSATION_ID,
|
|
580
|
-
buildSkillNotificationParams("immediate"),
|
|
581
|
-
);
|
|
582
|
-
// Buffer was never armed, so the call returns null and the IPC
|
|
583
|
-
// handler would emit directly.
|
|
584
|
-
expect(buffered).toBeNull();
|
|
585
|
-
return { messageId: "msg-success" };
|
|
586
|
-
};
|
|
587
|
-
|
|
588
|
-
const result = await runBackgroundJob(baseOpts());
|
|
589
|
-
expect(result.ok).toBe(true);
|
|
590
|
-
});
|
|
591
|
-
});
|
|
592
448
|
});
|
|
@@ -22,11 +22,6 @@ import type { LLMCallSite } from "../config/schemas/llm.js";
|
|
|
22
22
|
import { processMessage } from "../daemon/process-message.js";
|
|
23
23
|
import type { SubagentToolGateMode } from "../daemon/tool-setup-types.js";
|
|
24
24
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
25
|
-
import {
|
|
26
|
-
commitDeferredConversation,
|
|
27
|
-
discardDeferredConversation,
|
|
28
|
-
registerDeferredConversation,
|
|
29
|
-
} from "../notifications/deferred-emit.js";
|
|
30
25
|
import { emitNotificationSignal } from "../notifications/emit-signal.js";
|
|
31
26
|
import type { AttentionHints } from "../notifications/signal.js";
|
|
32
27
|
import { bootstrapConversation } from "../persistence/conversation-bootstrap.js";
|
|
@@ -188,11 +183,6 @@ export interface RunBackgroundJobOptions {
|
|
|
188
183
|
* the `assistant` role and cannot override the action prompt.
|
|
189
184
|
*/
|
|
190
185
|
assistantSandwich?: { preamble: string; content: string; postamble: string };
|
|
191
|
-
/**
|
|
192
|
-
* Buffer in-band `notifications send` calls and only flush them after the
|
|
193
|
-
* run completes successfully. See `notifications/deferred-emit.ts`.
|
|
194
|
-
*/
|
|
195
|
-
deferNotifications?: boolean;
|
|
196
186
|
/**
|
|
197
187
|
* Persist the kickoff `prompt` without indexing it — no memory segments,
|
|
198
188
|
* no embeddings, no lexical-index entry. Opt-in for jobs whose prompt is a
|
|
@@ -304,10 +294,6 @@ export async function runBackgroundJob(
|
|
|
304
294
|
...(opts.scheduleJobId ? { scheduleJobId: opts.scheduleJobId } : {}),
|
|
305
295
|
});
|
|
306
296
|
|
|
307
|
-
if (opts.deferNotifications) {
|
|
308
|
-
registerDeferredConversation(conversation.id);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
297
|
// Fire the sidebar-creation callback synchronously after bootstrap so
|
|
312
298
|
// connected clients (macOS sidebar, etc.) see the conversation appear
|
|
313
299
|
// immediately rather than after `processMessage` returns. Wrapped so a
|
|
@@ -382,23 +368,19 @@ export async function runBackgroundJob(
|
|
|
382
368
|
|
|
383
369
|
const runResult = await Promise.race([work, timeout]);
|
|
384
370
|
// Symmetric with the `work.catch` above: once `work` has won the race,
|
|
385
|
-
// the orphan timeout promise can still reject
|
|
386
|
-
//
|
|
387
|
-
//
|
|
371
|
+
// the orphan timeout promise can still reject if the timer fires before
|
|
372
|
+
// the `finally` clears it. Swallow so it doesn't surface as an unhandled
|
|
373
|
+
// rejection that Bun can use to terminate the process.
|
|
388
374
|
timeout.catch(() => {});
|
|
389
375
|
// The turn completed but its LLM call failed (e.g. an invalid provider):
|
|
390
376
|
// `processMessage` reports this via `turnFailure` instead of throwing.
|
|
391
|
-
// Rethrow so it flows through the shared failure path below (
|
|
392
|
-
//
|
|
393
|
-
// returning `ok: true`.
|
|
377
|
+
// Rethrow so it flows through the shared failure path below (logging +
|
|
378
|
+
// `activity.failed` emission) rather than returning `ok: true`.
|
|
394
379
|
if (runResult.turnFailure) {
|
|
395
380
|
throw new BackgroundJobTurnFailureError(
|
|
396
381
|
runResult.turnFailure.failureCode,
|
|
397
382
|
);
|
|
398
383
|
}
|
|
399
|
-
if (opts.deferNotifications) {
|
|
400
|
-
await commitDeferredConversation(conversation.id);
|
|
401
|
-
}
|
|
402
384
|
return { conversationId: conversation.id, ok: true };
|
|
403
385
|
} catch (err) {
|
|
404
386
|
const errorKind = classifyError(err);
|
|
@@ -411,10 +393,6 @@ export async function runBackgroundJob(
|
|
|
411
393
|
// so the structured failure result still flows to the caller.
|
|
412
394
|
const conversationId = conversation?.id ?? "";
|
|
413
395
|
|
|
414
|
-
if (opts.deferNotifications && conversationId) {
|
|
415
|
-
discardDeferredConversation(conversationId);
|
|
416
|
-
}
|
|
417
|
-
|
|
418
396
|
log.error(
|
|
419
397
|
{
|
|
420
398
|
err: error.message,
|
|
@@ -6,7 +6,6 @@ import { eq } from "drizzle-orm";
|
|
|
6
6
|
import { z } from "zod";
|
|
7
7
|
|
|
8
8
|
import { FeedItemSchema } from "../../api/responses/home.js";
|
|
9
|
-
import { bufferIfDeferred } from "../../notifications/deferred-emit.js";
|
|
10
9
|
import { editNotification } from "../../notifications/edit-notification.js";
|
|
11
10
|
import { emitNotificationSignal } from "../../notifications/emit-signal.js";
|
|
12
11
|
import { listEvents } from "../../notifications/events-store.js";
|
|
@@ -72,7 +71,6 @@ const EmitSignalParams = z.object({
|
|
|
72
71
|
conversationAffinityHint: z.record(z.string(), z.string()).optional(),
|
|
73
72
|
dedupeKey: z.string().optional(),
|
|
74
73
|
throwOnError: z.boolean().optional(),
|
|
75
|
-
originatingConversationId: z.string().optional(),
|
|
76
74
|
});
|
|
77
75
|
|
|
78
76
|
const EmitSignalResponse = z.object({
|
|
@@ -90,18 +88,6 @@ async function handleEmitSignal({ body = {} }: RouteHandlerArgs) {
|
|
|
90
88
|
);
|
|
91
89
|
}
|
|
92
90
|
const validated = parsed.data;
|
|
93
|
-
const buffered = bufferIfDeferred(
|
|
94
|
-
validated.originatingConversationId,
|
|
95
|
-
validated,
|
|
96
|
-
);
|
|
97
|
-
if (buffered) {
|
|
98
|
-
return {
|
|
99
|
-
signalId: buffered.signalId,
|
|
100
|
-
dispatched: buffered.dispatched,
|
|
101
|
-
deduplicated: buffered.deduplicated,
|
|
102
|
-
reason: buffered.reason,
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
91
|
const result = await emitNotificationSignal(validated);
|
|
106
92
|
return {
|
|
107
93
|
signalId: result.signalId,
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for `withSqliteRetry()`, the shared retry wrapper for transient
|
|
3
|
+
* SQLite write contention (`SQLITE_BUSY*` / `SQLITE_IOERR*`).
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from "bun:test";
|
|
7
|
+
|
|
8
|
+
import { isRetryableSqliteError, withSqliteRetry } from "../sqlite-retry.js";
|
|
9
|
+
|
|
10
|
+
function sqliteError(code: string): Error {
|
|
11
|
+
const err = new Error(code);
|
|
12
|
+
(err as Error & { code: string }).code = code;
|
|
13
|
+
return err;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
describe("isRetryableSqliteError", () => {
|
|
17
|
+
test("matches the SQLITE_BUSY and SQLITE_IOERR families", () => {
|
|
18
|
+
expect(isRetryableSqliteError(sqliteError("SQLITE_BUSY"))).toBe(true);
|
|
19
|
+
expect(isRetryableSqliteError(sqliteError("SQLITE_BUSY_SNAPSHOT"))).toBe(
|
|
20
|
+
true,
|
|
21
|
+
);
|
|
22
|
+
expect(isRetryableSqliteError(sqliteError("SQLITE_IOERR"))).toBe(true);
|
|
23
|
+
expect(isRetryableSqliteError(sqliteError("SQLITE_IOERR_SHORT_READ"))).toBe(
|
|
24
|
+
true,
|
|
25
|
+
);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("rejects non-transient errors", () => {
|
|
29
|
+
expect(isRetryableSqliteError(sqliteError("SQLITE_CONSTRAINT"))).toBe(
|
|
30
|
+
false,
|
|
31
|
+
);
|
|
32
|
+
expect(isRetryableSqliteError(new Error("database is locked"))).toBe(false);
|
|
33
|
+
expect(isRetryableSqliteError(null)).toBe(false);
|
|
34
|
+
expect(isRetryableSqliteError(undefined)).toBe(false);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
describe("withSqliteRetry", () => {
|
|
39
|
+
test("returns the wrapped value on first success", async () => {
|
|
40
|
+
let calls = 0;
|
|
41
|
+
const result = await withSqliteRetry(
|
|
42
|
+
() => {
|
|
43
|
+
calls += 1;
|
|
44
|
+
return "ok";
|
|
45
|
+
},
|
|
46
|
+
{ op: "test-op" },
|
|
47
|
+
);
|
|
48
|
+
expect(result).toBe("ok");
|
|
49
|
+
expect(calls).toBe(1);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("retries transient contention and returns the eventual success", async () => {
|
|
53
|
+
let calls = 0;
|
|
54
|
+
const result = await withSqliteRetry(
|
|
55
|
+
() => {
|
|
56
|
+
calls += 1;
|
|
57
|
+
if (calls <= 2) {
|
|
58
|
+
throw sqliteError("SQLITE_BUSY");
|
|
59
|
+
}
|
|
60
|
+
return "recovered";
|
|
61
|
+
},
|
|
62
|
+
{ op: "test-op", baseDelayMs: 1 },
|
|
63
|
+
);
|
|
64
|
+
expect(result).toBe("recovered");
|
|
65
|
+
expect(calls).toBe(3);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("awaits an async wrapped function", async () => {
|
|
69
|
+
let calls = 0;
|
|
70
|
+
const result = await withSqliteRetry(
|
|
71
|
+
async () => {
|
|
72
|
+
calls += 1;
|
|
73
|
+
if (calls === 1) {
|
|
74
|
+
throw sqliteError("SQLITE_IOERR");
|
|
75
|
+
}
|
|
76
|
+
return 42;
|
|
77
|
+
},
|
|
78
|
+
{ op: "test-op", baseDelayMs: 1 },
|
|
79
|
+
);
|
|
80
|
+
expect(result).toBe(42);
|
|
81
|
+
expect(calls).toBe(2);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("throws immediately on a non-retryable error", async () => {
|
|
85
|
+
let calls = 0;
|
|
86
|
+
await expect(
|
|
87
|
+
withSqliteRetry(
|
|
88
|
+
() => {
|
|
89
|
+
calls += 1;
|
|
90
|
+
throw sqliteError("SQLITE_CONSTRAINT_PRIMARYKEY");
|
|
91
|
+
},
|
|
92
|
+
{ op: "test-op", baseDelayMs: 1 },
|
|
93
|
+
),
|
|
94
|
+
).rejects.toThrow("SQLITE_CONSTRAINT_PRIMARYKEY");
|
|
95
|
+
expect(calls).toBe(1);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("rethrows after exhausting maxRetries", async () => {
|
|
99
|
+
let calls = 0;
|
|
100
|
+
await expect(
|
|
101
|
+
withSqliteRetry(
|
|
102
|
+
() => {
|
|
103
|
+
calls += 1;
|
|
104
|
+
throw sqliteError("SQLITE_BUSY");
|
|
105
|
+
},
|
|
106
|
+
{ op: "test-op", maxRetries: 2, baseDelayMs: 1 },
|
|
107
|
+
),
|
|
108
|
+
).rejects.toThrow("SQLITE_BUSY");
|
|
109
|
+
// Initial attempt + 2 retries.
|
|
110
|
+
expect(calls).toBe(3);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Per-conversation notification buffer used to keep a background job's
|
|
3
|
-
* "success" notification from racing the runner's `activity.failed` when
|
|
4
|
-
* the job times out after the model already invoked `notifications send`.
|
|
5
|
-
*
|
|
6
|
-
* `registerDeferredConversation` arms the buffer before the LLM turn.
|
|
7
|
-
* `bufferIfDeferred` is called from the IPC route handler — it buffers when
|
|
8
|
-
* armed, swallows when tombstoned (post-discard grace window), and returns
|
|
9
|
-
* null otherwise so the route emits normally.
|
|
10
|
-
* `commitDeferredConversation` flushes on success; `discardDeferredConversation`
|
|
11
|
-
* drops on failure and tombstones briefly to catch late tool calls that
|
|
12
|
-
* arrive after `processMessage` keeps running past the runner's timeout.
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
import { v4 as uuid } from "uuid";
|
|
16
|
-
|
|
17
|
-
import { getLogger } from "../util/logger.js";
|
|
18
|
-
import {
|
|
19
|
-
emitNotificationSignal,
|
|
20
|
-
type EmitSignalParams,
|
|
21
|
-
type EmitSignalResult,
|
|
22
|
-
} from "./emit-signal.js";
|
|
23
|
-
|
|
24
|
-
const log = getLogger("notifications-deferred-emit");
|
|
25
|
-
|
|
26
|
-
// How long after the last observed late notification we keep the tombstone
|
|
27
|
-
// alive. `work` in `runBackgroundJob` is not cancelled on timeout — it can
|
|
28
|
-
// continue running and emit skill calls indefinitely. We refresh the timer
|
|
29
|
-
// on every late arrival, so the tombstone persists as long as the turn is
|
|
30
|
-
// still draining and only expires once the conversation has been quiet for
|
|
31
|
-
// this long.
|
|
32
|
-
const TOMBSTONE_TTL_MS = 5 * 60 * 1000;
|
|
33
|
-
|
|
34
|
-
type BufferEntry =
|
|
35
|
-
| { state: "buffered"; items: EmitSignalParams<string>[] }
|
|
36
|
-
| { state: "tombstoned"; timer: ReturnType<typeof setTimeout> };
|
|
37
|
-
|
|
38
|
-
const buffers = new Map<string, BufferEntry>();
|
|
39
|
-
|
|
40
|
-
function scheduleTombstoneEviction(
|
|
41
|
-
conversationId: string,
|
|
42
|
-
): ReturnType<typeof setTimeout> {
|
|
43
|
-
const timer = setTimeout(() => {
|
|
44
|
-
const cur = buffers.get(conversationId);
|
|
45
|
-
if (cur?.state === "tombstoned") {
|
|
46
|
-
buffers.delete(conversationId);
|
|
47
|
-
}
|
|
48
|
-
}, TOMBSTONE_TTL_MS);
|
|
49
|
-
timer.unref?.();
|
|
50
|
-
return timer;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export function registerDeferredConversation(conversationId: string): void {
|
|
54
|
-
buffers.set(conversationId, { state: "buffered", items: [] });
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Buffer the signal when the originating conversation is armed, swallow it
|
|
59
|
-
* when tombstoned, otherwise return null so the caller emits normally.
|
|
60
|
-
*/
|
|
61
|
-
export function bufferIfDeferred(
|
|
62
|
-
originatingConversationId: string | undefined,
|
|
63
|
-
params: EmitSignalParams<string>,
|
|
64
|
-
): EmitSignalResult | null {
|
|
65
|
-
if (!originatingConversationId) {
|
|
66
|
-
return null;
|
|
67
|
-
}
|
|
68
|
-
const entry = buffers.get(originatingConversationId);
|
|
69
|
-
if (!entry) {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
if (entry.state === "tombstoned") {
|
|
73
|
-
// Refresh the eviction timer so the tombstone outlives any continuing
|
|
74
|
-
// turn activity. Otherwise a long-running orphan `processMessage` could
|
|
75
|
-
// emit a `notifications send` after the fixed TTL elapsed and bypass
|
|
76
|
-
// buffering, recreating the "success + activity.failed" contradiction.
|
|
77
|
-
clearTimeout(entry.timer);
|
|
78
|
-
entry.timer = scheduleTombstoneEviction(originatingConversationId);
|
|
79
|
-
return {
|
|
80
|
-
signalId: uuid(),
|
|
81
|
-
deduplicated: false,
|
|
82
|
-
dispatched: false,
|
|
83
|
-
reason: "Notification dropped: background job did not complete",
|
|
84
|
-
deliveryResults: [],
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
entry.items.push(params);
|
|
88
|
-
return {
|
|
89
|
-
signalId: uuid(),
|
|
90
|
-
deduplicated: false,
|
|
91
|
-
dispatched: false,
|
|
92
|
-
reason: "Notification deferred until background job completes",
|
|
93
|
-
deliveryResults: [],
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
export async function commitDeferredConversation(
|
|
98
|
-
conversationId: string,
|
|
99
|
-
): Promise<void> {
|
|
100
|
-
const entry = buffers.get(conversationId);
|
|
101
|
-
if (!entry || entry.state !== "buffered") {
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
buffers.delete(conversationId);
|
|
105
|
-
for (const params of entry.items) {
|
|
106
|
-
try {
|
|
107
|
-
await emitNotificationSignal(params);
|
|
108
|
-
} catch (err) {
|
|
109
|
-
log.warn(
|
|
110
|
-
{ err, conversationId },
|
|
111
|
-
"Buffered notification failed to emit on commit",
|
|
112
|
-
);
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Drop any buffered signals and tombstone the conversation. The tombstone
|
|
119
|
-
* persists until the turn has been quiet for `TOMBSTONE_TTL_MS`; each late
|
|
120
|
-
* notification observed via `bufferIfDeferred` refreshes the timer.
|
|
121
|
-
*/
|
|
122
|
-
export function discardDeferredConversation(conversationId: string): number {
|
|
123
|
-
const entry = buffers.get(conversationId);
|
|
124
|
-
if (!entry) {
|
|
125
|
-
return 0;
|
|
126
|
-
}
|
|
127
|
-
const droppedCount = entry.state === "buffered" ? entry.items.length : 0;
|
|
128
|
-
if (entry.state === "tombstoned") {
|
|
129
|
-
clearTimeout(entry.timer);
|
|
130
|
-
}
|
|
131
|
-
buffers.set(conversationId, {
|
|
132
|
-
state: "tombstoned",
|
|
133
|
-
timer: scheduleTombstoneEviction(conversationId),
|
|
134
|
-
});
|
|
135
|
-
if (droppedCount > 0) {
|
|
136
|
-
log.info(
|
|
137
|
-
{ conversationId, droppedCount },
|
|
138
|
-
"Discarded buffered notifications for failed background job",
|
|
139
|
-
);
|
|
140
|
-
}
|
|
141
|
-
return droppedCount;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/** @internal Test-only reset hook. */
|
|
145
|
-
export function resetDeferredForTest(): void {
|
|
146
|
-
buffers.clear();
|
|
147
|
-
}
|