@vellumai/assistant 0.11.6-dev.202608261918.60eeea2 → 0.11.6-dev.202608261937.a51ea3f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +1 -1
- package/src/__tests__/schedule-retry.test.ts +7 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +12 -0
- package/src/daemon/conversation-agent-loop.ts +46 -7
- package/src/notifications/__tests__/background-failure-signal.test.ts +107 -0
- package/src/notifications/__tests__/copy-composer.test.ts +83 -0
- package/src/notifications/background-failure-signal.ts +116 -0
- package/src/notifications/copy-composer.ts +41 -8
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +1 -1
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +4 -2
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +9 -42
- package/src/runtime/__tests__/background-job-runner.test.ts +30 -2
- package/src/runtime/background-job-runner.ts +50 -60
- package/src/schedule/scheduler.ts +36 -66
- package/src/telemetry/turn-outcome.ts +75 -4
package/package.json
CHANGED
|
@@ -128,7 +128,7 @@ const BASELINE: Record<string, readonly string[]> = {
|
|
|
128
128
|
"../../../../daemon/embedding-reconcile.js",
|
|
129
129
|
"../../../../daemon/trust-context.js",
|
|
130
130
|
"../../../../daemon/turn-latency-sub-spans.js",
|
|
131
|
-
"../../../../notifications/
|
|
131
|
+
"../../../../notifications/background-failure-signal.js",
|
|
132
132
|
"../../../../persistence/checkpoints.js",
|
|
133
133
|
"../../../../persistence/conversation-types.js",
|
|
134
134
|
"../../../../persistence/db-connection.js",
|
|
@@ -36,11 +36,17 @@ mock.module("../runtime/background-job-runner.js", () => ({
|
|
|
36
36
|
return { conversationId, ok: true };
|
|
37
37
|
} catch (err) {
|
|
38
38
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
39
|
+
const classification = runnerFailureClassification ?? {
|
|
40
|
+
errorKind: "exception" as const,
|
|
41
|
+
};
|
|
39
42
|
return {
|
|
40
43
|
conversationId,
|
|
41
44
|
ok: false,
|
|
42
45
|
error,
|
|
43
|
-
|
|
46
|
+
errorKind: classification.errorKind,
|
|
47
|
+
...(classification.failureCode !== undefined
|
|
48
|
+
? { turnFailure: { failureCode: classification.failureCode } }
|
|
49
|
+
: {}),
|
|
44
50
|
};
|
|
45
51
|
}
|
|
46
52
|
},
|
|
@@ -245,6 +245,14 @@ export interface EventHandlerState {
|
|
|
245
245
|
* the loop persists the failure as an assistant message.
|
|
246
246
|
*/
|
|
247
247
|
providerErrorCategory: string | null;
|
|
248
|
+
/**
|
|
249
|
+
* Connection and profile attribution of the most recent provider error
|
|
250
|
+
* (`classifyConversationError(...).connectionName` / `.profileName`).
|
|
251
|
+
* Carried into the turn's outcome stamp so failure consumers scope
|
|
252
|
+
* identity by the route the call actually resolved.
|
|
253
|
+
*/
|
|
254
|
+
providerErrorConnection: string | null;
|
|
255
|
+
providerErrorProfile: string | null;
|
|
248
256
|
persistProviderErrorAsAssistantMessage: boolean;
|
|
249
257
|
lastAssistantMessageId: string | undefined;
|
|
250
258
|
/**
|
|
@@ -599,6 +607,8 @@ export function createEventHandlerState(): EventHandlerState {
|
|
|
599
607
|
providerErrorUserMessage: null,
|
|
600
608
|
providerErrorCode: null,
|
|
601
609
|
providerErrorCategory: null,
|
|
610
|
+
providerErrorConnection: null,
|
|
611
|
+
providerErrorProfile: null,
|
|
602
612
|
persistProviderErrorAsAssistantMessage: false,
|
|
603
613
|
lastAssistantMessageId: undefined,
|
|
604
614
|
assistantRowAwaitingFinalization: false,
|
|
@@ -2732,6 +2742,8 @@ function handleError(
|
|
|
2732
2742
|
state.providerErrorUserMessage = classified.userMessage;
|
|
2733
2743
|
state.providerErrorCode = classified.code;
|
|
2734
2744
|
state.providerErrorCategory = classified.errorCategory;
|
|
2745
|
+
state.providerErrorConnection = classified.connectionName ?? null;
|
|
2746
|
+
state.providerErrorProfile = classified.profileName ?? null;
|
|
2735
2747
|
state.persistProviderErrorAsAssistantMessage =
|
|
2736
2748
|
shouldPersistProviderErrorAsAssistantMessage(classified);
|
|
2737
2749
|
}
|
|
@@ -733,8 +733,33 @@ export async function runAgentLoopImpl(
|
|
|
733
733
|
// the `finally` (before processing clears, so the reporter's settled-turn
|
|
734
734
|
// barrier guarantees the stamp ships with the turn event). Unset = the turn
|
|
735
735
|
// replied normally and carries no stamp.
|
|
736
|
+
const failedOutcomeFromClassification = (classified: {
|
|
737
|
+
code: string;
|
|
738
|
+
userMessage: string;
|
|
739
|
+
errorCategory: string;
|
|
740
|
+
connectionName?: string;
|
|
741
|
+
profileName?: string;
|
|
742
|
+
}): NonNullable<typeof abnormalOutcome> => ({
|
|
743
|
+
outcome: "failed",
|
|
744
|
+
failureCode: classified.code,
|
|
745
|
+
failureMessage: classified.userMessage,
|
|
746
|
+
failureCategory: classified.errorCategory,
|
|
747
|
+
...(classified.connectionName
|
|
748
|
+
? { failureConnection: classified.connectionName }
|
|
749
|
+
: {}),
|
|
750
|
+
...(classified.profileName
|
|
751
|
+
? { failureProfile: classified.profileName }
|
|
752
|
+
: {}),
|
|
753
|
+
});
|
|
736
754
|
let abnormalOutcome:
|
|
737
|
-
| {
|
|
755
|
+
| {
|
|
756
|
+
outcome: "failed" | "cancelled";
|
|
757
|
+
failureCode?: string;
|
|
758
|
+
failureMessage?: string;
|
|
759
|
+
failureCategory?: string;
|
|
760
|
+
failureConnection?: string;
|
|
761
|
+
failureProfile?: string;
|
|
762
|
+
}
|
|
738
763
|
| undefined;
|
|
739
764
|
// True once a replied terminal SSE (message_complete / generation_handoff)
|
|
740
765
|
// has been emitted. Guards the catch block: an error thrown by the
|
|
@@ -777,6 +802,10 @@ export async function runAgentLoopImpl(
|
|
|
777
802
|
try {
|
|
778
803
|
stampTurnOutcome(userMessageId, abnormalOutcome.outcome, {
|
|
779
804
|
failureCode: abnormalOutcome.failureCode,
|
|
805
|
+
failureMessage: abnormalOutcome.failureMessage,
|
|
806
|
+
failureCategory: abnormalOutcome.failureCategory,
|
|
807
|
+
failureConnection: abnormalOutcome.failureConnection,
|
|
808
|
+
failureProfile: abnormalOutcome.failureProfile,
|
|
780
809
|
});
|
|
781
810
|
} catch (err) {
|
|
782
811
|
rlog.warn(
|
|
@@ -858,6 +887,7 @@ export async function runAgentLoopImpl(
|
|
|
858
887
|
abnormalOutcome = {
|
|
859
888
|
outcome: "failed",
|
|
860
889
|
failureCode: DISK_PRESSURE_ERROR_CODE,
|
|
890
|
+
failureMessage: message,
|
|
861
891
|
};
|
|
862
892
|
rlog.warn(
|
|
863
893
|
{ reason: diskPressureDecision.reason },
|
|
@@ -1382,17 +1412,16 @@ export async function runAgentLoopImpl(
|
|
|
1382
1412
|
);
|
|
1383
1413
|
// Exhausted-overflow exit: the reduction ladder is spent and the turn
|
|
1384
1414
|
// ends without a real reply, so label it failed for telemetry.
|
|
1385
|
-
abnormalOutcome =
|
|
1415
|
+
abnormalOutcome = failedOutcomeFromClassification(classified);
|
|
1386
1416
|
onEvent(buildConversationErrorMessage(ctx.conversationId, classified));
|
|
1387
1417
|
} else if (
|
|
1388
1418
|
overflowTerminalReason === "budget_yield_unrecovered" &&
|
|
1389
1419
|
!abortController.signal.aborted
|
|
1390
1420
|
) {
|
|
1391
1421
|
budgetYieldClassification = budgetYieldUnrecoveredClassification();
|
|
1392
|
-
abnormalOutcome =
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
};
|
|
1422
|
+
abnormalOutcome = failedOutcomeFromClassification(
|
|
1423
|
+
budgetYieldClassification,
|
|
1424
|
+
);
|
|
1396
1425
|
onEvent(
|
|
1397
1426
|
buildConversationErrorMessage(
|
|
1398
1427
|
ctx.conversationId,
|
|
@@ -1548,6 +1577,16 @@ export async function runAgentLoopImpl(
|
|
|
1548
1577
|
...(state.providerErrorCode
|
|
1549
1578
|
? { failureCode: state.providerErrorCode }
|
|
1550
1579
|
: {}),
|
|
1580
|
+
failureMessage: state.providerErrorUserMessage,
|
|
1581
|
+
...(state.providerErrorCategory
|
|
1582
|
+
? { failureCategory: state.providerErrorCategory }
|
|
1583
|
+
: {}),
|
|
1584
|
+
...(state.providerErrorConnection
|
|
1585
|
+
? { failureConnection: state.providerErrorConnection }
|
|
1586
|
+
: {}),
|
|
1587
|
+
...(state.providerErrorProfile
|
|
1588
|
+
? { failureProfile: state.providerErrorProfile }
|
|
1589
|
+
: {}),
|
|
1551
1590
|
};
|
|
1552
1591
|
// Drop any reservation stranded by the failed LLM call. The B3
|
|
1553
1592
|
// pre-allocation path reserves an empty assistant row at
|
|
@@ -1874,7 +1913,7 @@ export async function runAgentLoopImpl(
|
|
|
1874
1913
|
...turnErrorAttribution(),
|
|
1875
1914
|
});
|
|
1876
1915
|
if (!turnReplied) {
|
|
1877
|
-
abnormalOutcome =
|
|
1916
|
+
abnormalOutcome = failedOutcomeFromClassification(classified);
|
|
1878
1917
|
}
|
|
1879
1918
|
onEvent({
|
|
1880
1919
|
type: "error",
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
const emitCalls: Array<Record<string, unknown>> = [];
|
|
4
|
+
mock.module("../emit-signal.js", () => ({
|
|
5
|
+
emitNotificationSignal: (params: Record<string, unknown>) => {
|
|
6
|
+
emitCalls.push(params);
|
|
7
|
+
return Promise.resolve({
|
|
8
|
+
signalId: "sig-1",
|
|
9
|
+
deduplicated: false,
|
|
10
|
+
dispatched: true,
|
|
11
|
+
reason: "ok",
|
|
12
|
+
deliveryResults: [],
|
|
13
|
+
pipelineFailed: false,
|
|
14
|
+
});
|
|
15
|
+
},
|
|
16
|
+
}));
|
|
17
|
+
|
|
18
|
+
const { emitBackgroundFailureSignal } =
|
|
19
|
+
await import("../background-failure-signal.js");
|
|
20
|
+
|
|
21
|
+
describe("emitBackgroundFailureSignal", () => {
|
|
22
|
+
beforeEach(() => {
|
|
23
|
+
emitCalls.length = 0;
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("carried connection leads the dedupe scope and rides the payload", () => {
|
|
27
|
+
emitBackgroundFailureSignal({
|
|
28
|
+
jobName: "schedule:job-1",
|
|
29
|
+
displayName: "schedule:PR scan",
|
|
30
|
+
sourceChannel: "scheduler",
|
|
31
|
+
sourceContextId: "job-1",
|
|
32
|
+
errorKind: "model_provider",
|
|
33
|
+
errorMessage: "Agent turn failed (PROVIDER_BILLING)",
|
|
34
|
+
failureCode: "PROVIDER_BILLING",
|
|
35
|
+
failureSummary: "You're out of credits.",
|
|
36
|
+
errorCategory: "credits_exhausted",
|
|
37
|
+
connectionName: "conn-a",
|
|
38
|
+
profileName: "balanced",
|
|
39
|
+
fallbackProviderScope: "cost-optimized",
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
expect(emitCalls).toHaveLength(1);
|
|
43
|
+
const emitted = emitCalls[0];
|
|
44
|
+
expect(emitted.sourceEventName).toBe("activity.failed");
|
|
45
|
+
expect(emitted.dedupeKey as string).toMatch(
|
|
46
|
+
/^activity-failed:cause:PROVIDER_BILLING:conn-a:\d{4}-\d{2}-\d{2}$/,
|
|
47
|
+
);
|
|
48
|
+
expect(emitted.contextPayload).toMatchObject({
|
|
49
|
+
jobName: "schedule:PR scan",
|
|
50
|
+
errorKind: "model_provider",
|
|
51
|
+
failureCode: "PROVIDER_BILLING",
|
|
52
|
+
failureSummary: "You're out of credits.",
|
|
53
|
+
errorCategory: "credits_exhausted",
|
|
54
|
+
connectionName: "conn-a",
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("carried profile scopes when no connection is carried", () => {
|
|
59
|
+
emitBackgroundFailureSignal({
|
|
60
|
+
jobName: "heartbeat",
|
|
61
|
+
sourceChannel: "assistant_tool",
|
|
62
|
+
sourceContextId: "conv-1",
|
|
63
|
+
errorKind: "model_provider",
|
|
64
|
+
errorMessage: "rate limited",
|
|
65
|
+
failureCode: "PROVIDER_RATE_LIMIT",
|
|
66
|
+
profileName: "balanced",
|
|
67
|
+
fallbackProviderScope: "cost-optimized",
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
expect(emitCalls[0].dedupeKey as string).toMatch(
|
|
71
|
+
/^activity-failed:cause:PROVIDER_RATE_LIMIT:balanced:\d{4}-\d{2}-\d{2}$/,
|
|
72
|
+
);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("recomputed fallback scopes when nothing is carried", () => {
|
|
76
|
+
emitBackgroundFailureSignal({
|
|
77
|
+
jobName: "watcher-a",
|
|
78
|
+
sourceChannel: "assistant_tool",
|
|
79
|
+
sourceContextId: "conv-2",
|
|
80
|
+
errorKind: "model_provider",
|
|
81
|
+
errorMessage: "provider returned 401",
|
|
82
|
+
fallbackProviderScope: "cost-optimized",
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
expect(emitCalls[0].dedupeKey as string).toMatch(
|
|
86
|
+
/^activity-failed:cause:model_provider:cost-optimized:\d{4}-\d{2}-\d{2}$/,
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("no scope at all keys per job", () => {
|
|
91
|
+
emitBackgroundFailureSignal({
|
|
92
|
+
jobName: "memory.v2.sweep",
|
|
93
|
+
sourceChannel: "scheduler",
|
|
94
|
+
sourceContextId: "job-9",
|
|
95
|
+
errorKind: "exception",
|
|
96
|
+
errorMessage: "boom",
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
expect(emitCalls[0].dedupeKey as string).toMatch(
|
|
100
|
+
/^activity-failed:memory\.v2\.sweep:\d{4}-\d{2}-\d{2}$/,
|
|
101
|
+
);
|
|
102
|
+
expect(emitCalls[0].contextPayload).toMatchObject({
|
|
103
|
+
jobName: "memory.v2.sweep",
|
|
104
|
+
errorKind: "exception",
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
});
|
|
@@ -35,6 +35,89 @@ function makeSignal(
|
|
|
35
35
|
|
|
36
36
|
const CHANNELS: NotificationChannel[] = ["vellum" as NotificationChannel];
|
|
37
37
|
|
|
38
|
+
// ── activity.failed rendering ─────────────────────────────────────────
|
|
39
|
+
|
|
40
|
+
describe("activity.failed copy", () => {
|
|
41
|
+
function failedSignal(
|
|
42
|
+
contextPayload: Record<string, unknown>,
|
|
43
|
+
): NotificationSignal {
|
|
44
|
+
return makeSignal({
|
|
45
|
+
sourceEventName: "activity.failed",
|
|
46
|
+
contextPayload,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
test("renders the carried classification message verbatim", () => {
|
|
51
|
+
const copy = composeFallbackCopy(
|
|
52
|
+
failedSignal({
|
|
53
|
+
jobName: "schedule:PR scan",
|
|
54
|
+
errorKind: "model_provider",
|
|
55
|
+
errorMessage: "Agent turn failed (PROVIDER_BILLING)",
|
|
56
|
+
failureSummary:
|
|
57
|
+
"You're out of credits. Add credits in Settings to continue.",
|
|
58
|
+
}),
|
|
59
|
+
CHANNELS,
|
|
60
|
+
);
|
|
61
|
+
expect(copy.vellum?.title).toBe("Background job failed: schedule:PR scan");
|
|
62
|
+
expect(copy.vellum?.body).toBe(
|
|
63
|
+
"You're out of credits. Add credits in Settings to continue.",
|
|
64
|
+
);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("an oversized carried summary is bounded", () => {
|
|
68
|
+
const copy = composeFallbackCopy(
|
|
69
|
+
failedSignal({
|
|
70
|
+
jobName: "job",
|
|
71
|
+
errorKind: "model_provider",
|
|
72
|
+
errorMessage: "irrelevant",
|
|
73
|
+
failureSummary: "x".repeat(400),
|
|
74
|
+
}),
|
|
75
|
+
CHANNELS,
|
|
76
|
+
);
|
|
77
|
+
expect((copy.vellum?.body ?? "").length).toBeLessThanOrEqual(303);
|
|
78
|
+
expect(copy.vellum?.body).toEndWith("...");
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("constant-shaped raw detail never reaches the body", () => {
|
|
82
|
+
const copy = composeFallbackCopy(
|
|
83
|
+
failedSignal({
|
|
84
|
+
jobName: "heartbeat",
|
|
85
|
+
errorKind: "model_provider",
|
|
86
|
+
errorMessage: "Agent turn failed (PROVIDER_BILLING)",
|
|
87
|
+
}),
|
|
88
|
+
CHANNELS,
|
|
89
|
+
);
|
|
90
|
+
expect(copy.vellum?.body).toBe("The model provider did not respond.");
|
|
91
|
+
expect(copy.vellum?.body).not.toContain("PROVIDER_BILLING");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("readable raw detail is appended to the kind prose", () => {
|
|
95
|
+
const copy = composeFallbackCopy(
|
|
96
|
+
failedSignal({
|
|
97
|
+
jobName: "watcher",
|
|
98
|
+
errorKind: "exception",
|
|
99
|
+
errorMessage: "The feed endpoint returned an empty document.",
|
|
100
|
+
}),
|
|
101
|
+
CHANNELS,
|
|
102
|
+
);
|
|
103
|
+
expect(copy.vellum?.body).toBe(
|
|
104
|
+
"It stopped with an error. The feed endpoint returned an empty document.",
|
|
105
|
+
);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("timeout renders its own prose", () => {
|
|
109
|
+
const copy = composeFallbackCopy(
|
|
110
|
+
failedSignal({
|
|
111
|
+
jobName: "filing",
|
|
112
|
+
errorKind: "timeout",
|
|
113
|
+
errorMessage: "Background job 'filing' timed out after 1800000ms",
|
|
114
|
+
}),
|
|
115
|
+
CHANNELS,
|
|
116
|
+
);
|
|
117
|
+
expect(copy.vellum?.body).toContain("It ran out of time before finishing.");
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
38
121
|
// ── composeFallbackCopy with requestedMessage ─────────────────────────
|
|
39
122
|
|
|
40
123
|
describe("composeFallbackCopy honors requestedMessage / requestedTitle", () => {
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import type { BackgroundJobErrorKind } from "../runtime/background-job-runner.js";
|
|
2
|
+
import { getLogger } from "../util/logger.js";
|
|
3
|
+
import { activityFailedDedupeKey } from "./activity-failed-dedupe.js";
|
|
4
|
+
import { emitNotificationSignal } from "./emit-signal.js";
|
|
5
|
+
import type { NotificationSourceChannel } from "./signal.js";
|
|
6
|
+
|
|
7
|
+
const log = getLogger("background-failure-signal");
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* One background failure, as a producer reports it.
|
|
11
|
+
*
|
|
12
|
+
* The classification fields are the authored attribution carried out of the
|
|
13
|
+
* failing turn (`TurnFailure`): `failureSummary` is the classifier's
|
|
14
|
+
* user-facing message, and `profileName`/`connectionName` name the route the
|
|
15
|
+
* call actually resolved. Producers whose failures never ran a turn (a
|
|
16
|
+
* timeout, a bootstrap crash, a data job's own exception) leave them unset
|
|
17
|
+
* and the signal degrades to the classified kind.
|
|
18
|
+
*/
|
|
19
|
+
export interface BackgroundFailureReport {
|
|
20
|
+
/** Stable job identifier for logs and per-job dedupe, e.g. `heartbeat`. */
|
|
21
|
+
jobName: string;
|
|
22
|
+
/** Human-facing job label for notification copy. Defaults to `jobName`. */
|
|
23
|
+
displayName?: string;
|
|
24
|
+
sourceChannel: NotificationSourceChannel;
|
|
25
|
+
/** Conversation or job id the notification links back to. */
|
|
26
|
+
sourceContextId: string;
|
|
27
|
+
errorKind: BackgroundJobErrorKind;
|
|
28
|
+
/** Raw error text, carried for debugging; copy prefers `failureSummary`. */
|
|
29
|
+
errorMessage: string;
|
|
30
|
+
/** Stable classified code (`ConversationErrorCode`), when classified. */
|
|
31
|
+
failureCode?: string;
|
|
32
|
+
/** The classification's authored user-facing message, when carried. */
|
|
33
|
+
failureSummary?: string;
|
|
34
|
+
/** The classification's machine-readable category, when carried. */
|
|
35
|
+
errorCategory?: string;
|
|
36
|
+
/**
|
|
37
|
+
* Connection row the failing call resolved, when carried. The most precise
|
|
38
|
+
* route identity, so it leads the dedupe-scope hierarchy.
|
|
39
|
+
*/
|
|
40
|
+
connectionName?: string;
|
|
41
|
+
/** Profile key the failing call resolved, when carried. */
|
|
42
|
+
profileName?: string;
|
|
43
|
+
/**
|
|
44
|
+
* Recomputed single-route profile key (see `resolveSingleRouteProfileKey`).
|
|
45
|
+
* Used only when the failure carries no attribution of its own.
|
|
46
|
+
*/
|
|
47
|
+
fallbackProviderScope?: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The single emission seam for background-job failure notifications.
|
|
52
|
+
*
|
|
53
|
+
* Owns the payload shape, the dedupe-key derivation, and the attention hints
|
|
54
|
+
* so every producer (the background-job runner, the scheduler's
|
|
55
|
+
* retries-exhausted alert, standalone data jobs) reports the same way and a
|
|
56
|
+
* new producer cannot invent a divergent one. Fire-and-forget: emission
|
|
57
|
+
* errors are logged, never thrown, because a notification failure must not
|
|
58
|
+
* break the job path that reports it.
|
|
59
|
+
*
|
|
60
|
+
* The dedupe scope is the most precise route identity the failure carries:
|
|
61
|
+
* the resolved connection, else the resolved profile, else the recomputed
|
|
62
|
+
* fallback. Failures carrying less precision can split an incident into an
|
|
63
|
+
* extra notification (never merge two incidents into one), and the split
|
|
64
|
+
* shrinks as more paths carry attribution.
|
|
65
|
+
*/
|
|
66
|
+
export function emitBackgroundFailureSignal(
|
|
67
|
+
report: BackgroundFailureReport,
|
|
68
|
+
): void {
|
|
69
|
+
const providerScope =
|
|
70
|
+
report.connectionName ?? report.profileName ?? report.fallbackProviderScope;
|
|
71
|
+
emitNotificationSignal({
|
|
72
|
+
sourceChannel: report.sourceChannel,
|
|
73
|
+
sourceContextId: report.sourceContextId,
|
|
74
|
+
sourceEventName: "activity.failed",
|
|
75
|
+
dedupeKey: activityFailedDedupeKey({
|
|
76
|
+
jobName: report.jobName,
|
|
77
|
+
errorKind: report.errorKind,
|
|
78
|
+
...(report.failureCode !== undefined
|
|
79
|
+
? { failureCode: report.failureCode }
|
|
80
|
+
: {}),
|
|
81
|
+
...(providerScope !== undefined ? { providerScope } : {}),
|
|
82
|
+
}),
|
|
83
|
+
contextPayload: {
|
|
84
|
+
jobName: report.displayName ?? report.jobName,
|
|
85
|
+
errorMessage: report.errorMessage,
|
|
86
|
+
errorKind: report.errorKind,
|
|
87
|
+
...(report.failureCode !== undefined
|
|
88
|
+
? { failureCode: report.failureCode }
|
|
89
|
+
: {}),
|
|
90
|
+
...(report.failureSummary !== undefined
|
|
91
|
+
? { failureSummary: report.failureSummary }
|
|
92
|
+
: {}),
|
|
93
|
+
...(report.errorCategory !== undefined
|
|
94
|
+
? { errorCategory: report.errorCategory }
|
|
95
|
+
: {}),
|
|
96
|
+
...(report.connectionName !== undefined
|
|
97
|
+
? { connectionName: report.connectionName }
|
|
98
|
+
: {}),
|
|
99
|
+
},
|
|
100
|
+
attentionHints: {
|
|
101
|
+
requiresAction: false,
|
|
102
|
+
urgency: "medium",
|
|
103
|
+
isAsyncBackground: true,
|
|
104
|
+
visibleInSourceNow: false,
|
|
105
|
+
},
|
|
106
|
+
}).catch((emitErr) => {
|
|
107
|
+
log.warn(
|
|
108
|
+
{
|
|
109
|
+
err: emitErr instanceof Error ? emitErr.message : String(emitErr),
|
|
110
|
+
jobName: report.jobName,
|
|
111
|
+
sourceContextId: report.sourceContextId,
|
|
112
|
+
},
|
|
113
|
+
"Failed to emit background-failure notification",
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { normalizeTitle } from "../util/short-title.js";
|
|
13
|
+
import { truncate } from "../util/truncate.js";
|
|
13
14
|
import {
|
|
14
15
|
accessRequestCardTitle,
|
|
15
16
|
buildAccessRequestContractText,
|
|
@@ -41,6 +42,33 @@ import type { NotificationChannel, RenderedChannelCopy } from "./types.js";
|
|
|
41
42
|
|
|
42
43
|
type CopyTemplate = (payload: Record<string, unknown>) => RenderedChannelCopy;
|
|
43
44
|
|
|
45
|
+
/**
|
|
46
|
+
* Failure prose for an `activity.failed` payload with no authored
|
|
47
|
+
* classification message. The classified kind supplies the readable part,
|
|
48
|
+
* and the raw error text is appended only when it reads as prose: a
|
|
49
|
+
* stack-shaped or constant-shaped message (PROVIDER_BILLING and friends)
|
|
50
|
+
* says nothing to the person reading the notification and never lands in
|
|
51
|
+
* the body.
|
|
52
|
+
*/
|
|
53
|
+
function describeUnclassifiedFailure(payload: Record<string, unknown>): string {
|
|
54
|
+
const kind = str(payload.errorKind, "exception");
|
|
55
|
+
const opening =
|
|
56
|
+
kind === "timeout"
|
|
57
|
+
? "It ran out of time before finishing."
|
|
58
|
+
: kind === "model_provider"
|
|
59
|
+
? "The model provider did not respond."
|
|
60
|
+
: "It stopped with an error.";
|
|
61
|
+
const raw =
|
|
62
|
+
typeof payload.errorMessage === "string"
|
|
63
|
+
? payload.errorMessage.replace(/\s+/g, " ").trim()
|
|
64
|
+
: "";
|
|
65
|
+
const readable =
|
|
66
|
+
raw.length > 0 &&
|
|
67
|
+
raw.length <= 200 &&
|
|
68
|
+
!/[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+/.test(raw);
|
|
69
|
+
return readable ? `${opening} ${raw}` : opening;
|
|
70
|
+
}
|
|
71
|
+
|
|
44
72
|
function str(value: unknown, fallback: string): string {
|
|
45
73
|
if (typeof value === "string" && value.length > 0) {
|
|
46
74
|
return value;
|
|
@@ -335,16 +363,21 @@ const TEMPLATES: Partial<Record<NotificationSourceEventName, CopyTemplate>> = {
|
|
|
335
363
|
|
|
336
364
|
"activity.failed": (payload) => {
|
|
337
365
|
const jobName = str(payload.jobName, "background job");
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
366
|
+
// The classifier's authored user-facing message rides the payload as
|
|
367
|
+
// `failureSummary` when the failing turn carried its classification;
|
|
368
|
+
// it already says what happened and what to do about it.
|
|
369
|
+
// Authored messages are short by construction, but the classifier's
|
|
370
|
+
// fallback branches can embed provider error text, so bound the body
|
|
371
|
+
// like any other untrusted payload string.
|
|
372
|
+
const summary = truncate(
|
|
373
|
+
typeof payload.failureSummary === "string"
|
|
374
|
+
? payload.failureSummary.trim()
|
|
375
|
+
: "",
|
|
376
|
+
300,
|
|
377
|
+
);
|
|
345
378
|
return {
|
|
346
379
|
title: `Background job failed: ${jobName}`,
|
|
347
|
-
body:
|
|
380
|
+
body: summary !== "" ? summary : describeUnclassifiedFailure(payload),
|
|
348
381
|
};
|
|
349
382
|
},
|
|
350
383
|
|
|
@@ -1138,7 +1138,7 @@ describe("memoryV2ConsolidateJob — consecutive-failure state", () => {
|
|
|
1138
1138
|
ok: false,
|
|
1139
1139
|
error: new Error("provider refused"),
|
|
1140
1140
|
errorKind: "model_provider",
|
|
1141
|
-
...(failureCode !== undefined ? { failureCode } : {}),
|
|
1141
|
+
...(failureCode !== undefined ? { turnFailure: { failureCode } } : {}),
|
|
1142
1142
|
});
|
|
1143
1143
|
}
|
|
1144
1144
|
|
|
@@ -486,12 +486,14 @@ export async function memoryV2ConsolidateJob(
|
|
|
486
486
|
// everything else (network blip, model hiccup, timeout) is transient
|
|
487
487
|
// and stays on the short curve.
|
|
488
488
|
const failureKind: ConsolidationFailureKind =
|
|
489
|
-
runResult.failureCode === "PROVIDER_BILLING"
|
|
489
|
+
runResult.turnFailure?.failureCode === "PROVIDER_BILLING"
|
|
490
|
+
? "billing"
|
|
491
|
+
: "transient";
|
|
490
492
|
log.error(
|
|
491
493
|
{
|
|
492
494
|
conversationId: runResult.conversationId,
|
|
493
495
|
errorKind: runResult.errorKind,
|
|
494
|
-
failureCode: runResult.failureCode,
|
|
496
|
+
failureCode: runResult.turnFailure?.failureCode,
|
|
495
497
|
failureKind,
|
|
496
498
|
err: runResult.error?.message,
|
|
497
499
|
},
|
|
@@ -37,7 +37,7 @@ import { z } from "zod";
|
|
|
37
37
|
import { usesConceptPageMemory } from "../../../../config/memory-v3-gate.js";
|
|
38
38
|
import type { AssistantConfig } from "../../../../config/types.js";
|
|
39
39
|
import { warmGuardianBindings } from "../../../../contacts/guardian-delivery-reader.js";
|
|
40
|
-
import {
|
|
40
|
+
import { emitBackgroundFailureSignal } from "../../../../notifications/background-failure-signal.js";
|
|
41
41
|
import { getDb } from "../../../../persistence/db-connection.js";
|
|
42
42
|
import type { MemoryJob } from "../../../../persistence/jobs-store.js";
|
|
43
43
|
import {
|
|
@@ -199,53 +199,20 @@ export async function memoryV2SweepJob(
|
|
|
199
199
|
} catch (err) {
|
|
200
200
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
201
201
|
log.error({ err }, "memory v2 sweep failed");
|
|
202
|
-
|
|
203
|
-
|
|
202
|
+
// The sweep is a data job with no LLM turn to classify, so its failures
|
|
203
|
+
// report as generic exceptions through the shared background-failure
|
|
204
|
+
// seam, which owns the payload shape and dedupe key.
|
|
205
|
+
emitBackgroundFailureSignal({
|
|
206
|
+
jobName: JOB_NAME,
|
|
207
|
+
sourceChannel: "scheduler",
|
|
208
|
+
sourceContextId: job.id,
|
|
209
|
+
errorKind: "exception",
|
|
204
210
|
errorMessage,
|
|
205
211
|
});
|
|
206
212
|
return 0;
|
|
207
213
|
}
|
|
208
214
|
}
|
|
209
215
|
|
|
210
|
-
/**
|
|
211
|
-
* Emit an `activity.failed` notification for a failed sweep run. Mirrors
|
|
212
|
-
* the shape `runBackgroundJob` produces for its own failures so the home
|
|
213
|
-
* feed and native notifications stay consistent regardless of which code
|
|
214
|
-
* path executed the work. Fire-and-forget — a notification failure must
|
|
215
|
-
* never break sweep operation.
|
|
216
|
-
*/
|
|
217
|
-
function emitSweepActivityFailed(args: {
|
|
218
|
-
jobId: string;
|
|
219
|
-
errorMessage: string;
|
|
220
|
-
}): void {
|
|
221
|
-
const day = new Date().toISOString().slice(0, 10);
|
|
222
|
-
emitNotificationSignal({
|
|
223
|
-
sourceChannel: "scheduler",
|
|
224
|
-
sourceContextId: args.jobId,
|
|
225
|
-
sourceEventName: "activity.failed",
|
|
226
|
-
dedupeKey: `activity-failed:${JOB_NAME}:${day}`,
|
|
227
|
-
contextPayload: {
|
|
228
|
-
jobName: JOB_NAME,
|
|
229
|
-
errorMessage: args.errorMessage,
|
|
230
|
-
errorKind: "exception",
|
|
231
|
-
},
|
|
232
|
-
attentionHints: {
|
|
233
|
-
requiresAction: false,
|
|
234
|
-
urgency: "medium",
|
|
235
|
-
isAsyncBackground: true,
|
|
236
|
-
visibleInSourceNow: false,
|
|
237
|
-
},
|
|
238
|
-
}).catch((emitErr) => {
|
|
239
|
-
log.warn(
|
|
240
|
-
{
|
|
241
|
-
err: emitErr instanceof Error ? emitErr.message : String(emitErr),
|
|
242
|
-
jobId: args.jobId,
|
|
243
|
-
},
|
|
244
|
-
"Failed to emit activity.failed notification for memory v2 sweep",
|
|
245
|
-
);
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
|
|
249
216
|
/**
|
|
250
217
|
* Append each non-empty entry to `memory/buffer.md` + today's archive,
|
|
251
218
|
* using the same format `remember()` produces. Returns the count of entries
|
|
@@ -278,7 +278,7 @@ describe("runBackgroundJob", () => {
|
|
|
278
278
|
expect(result.conversationId).toBe(STUB_CONVERSATION_ID);
|
|
279
279
|
// The stable classified code rides the result so callers can branch on
|
|
280
280
|
// the failure class without parsing the error message.
|
|
281
|
-
expect(result.failureCode).toBe("PROVIDER_BILLING");
|
|
281
|
+
expect(result.turnFailure?.failureCode).toBe("PROVIDER_BILLING");
|
|
282
282
|
|
|
283
283
|
expect(emitCalls).toHaveLength(1);
|
|
284
284
|
expect(emitCalls[0].sourceEventName).toBe("activity.failed");
|
|
@@ -308,6 +308,34 @@ describe("runBackgroundJob", () => {
|
|
|
308
308
|
);
|
|
309
309
|
});
|
|
310
310
|
|
|
311
|
+
test("carried turn attribution scopes the dedupe and rides the payload", async () => {
|
|
312
|
+
processMessageImpl = async () => ({
|
|
313
|
+
messageId: "msg-failed-turn",
|
|
314
|
+
turnFailure: {
|
|
315
|
+
failureCode: "PROVIDER_BILLING",
|
|
316
|
+
userMessage: "You're out of credits.",
|
|
317
|
+
errorCategory: "credits_exhausted",
|
|
318
|
+
connectionName: "conn-a",
|
|
319
|
+
profileName: "balanced",
|
|
320
|
+
},
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
const result = await runBackgroundJob(baseOpts());
|
|
324
|
+
|
|
325
|
+
expect(result.ok).toBe(false);
|
|
326
|
+
expect(result.turnFailure?.userMessage).toBe("You're out of credits.");
|
|
327
|
+
expect(emitCalls).toHaveLength(1);
|
|
328
|
+
// The carried connection outranks the call site's recomputed profile.
|
|
329
|
+
expect(emitCalls[0].dedupeKey as string).toMatch(
|
|
330
|
+
/^activity-failed:cause:PROVIDER_BILLING:conn-a:\d{4}-\d{2}-\d{2}$/,
|
|
331
|
+
);
|
|
332
|
+
expect(emitCalls[0].contextPayload).toMatchObject({
|
|
333
|
+
failureSummary: "You're out of credits.",
|
|
334
|
+
errorCategory: "credits_exhausted",
|
|
335
|
+
connectionName: "conn-a",
|
|
336
|
+
});
|
|
337
|
+
});
|
|
338
|
+
|
|
311
339
|
test("job-specific failure code keeps the per-job dedupe key", async () => {
|
|
312
340
|
// CONTEXT_TOO_LARGE names a defect in this job's own prompt; collapsing
|
|
313
341
|
// it across jobs would hide a second job's unrelated failure.
|
|
@@ -319,7 +347,7 @@ describe("runBackgroundJob", () => {
|
|
|
319
347
|
const result = await runBackgroundJob(baseOpts());
|
|
320
348
|
|
|
321
349
|
expect(result.ok).toBe(false);
|
|
322
|
-
expect(result.failureCode).toBe("CONTEXT_TOO_LARGE");
|
|
350
|
+
expect(result.turnFailure?.failureCode).toBe("CONTEXT_TOO_LARGE");
|
|
323
351
|
expect(emitCalls).toHaveLength(1);
|
|
324
352
|
expect(emitCalls[0].dedupeKey as string).toMatch(
|
|
325
353
|
/^activity-failed:test-job:\d{4}-\d{2}-\d{2}$/,
|
|
@@ -24,13 +24,12 @@ import type { LLMCallSite } from "../config/schemas/llm.js";
|
|
|
24
24
|
import { processMessage } from "../daemon/process-message.js";
|
|
25
25
|
import type { SubagentToolGateMode } from "../daemon/tool-setup-types.js";
|
|
26
26
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
27
|
-
import {
|
|
28
|
-
import { emitNotificationSignal } from "../notifications/emit-signal.js";
|
|
29
|
-
import type { AttentionHints } from "../notifications/signal.js";
|
|
27
|
+
import { emitBackgroundFailureSignal } from "../notifications/background-failure-signal.js";
|
|
30
28
|
import { bootstrapConversation } from "../persistence/conversation-bootstrap.js";
|
|
31
29
|
import { addMessage } from "../persistence/conversation-crud.js";
|
|
32
30
|
import type { TitleOrigin } from "../persistence/conversation-title-service.js";
|
|
33
31
|
import { dispatchProviderResolvable } from "../providers/provider-resolvability.js";
|
|
32
|
+
import type { TurnFailure } from "../telemetry/turn-outcome.js";
|
|
34
33
|
import { getLogger } from "../util/logger.js";
|
|
35
34
|
import { hasReceivedUserMessage } from "./pre-first-message-gate.js";
|
|
36
35
|
|
|
@@ -57,12 +56,14 @@ class BackgroundJobTimeoutError extends Error {
|
|
|
57
56
|
*/
|
|
58
57
|
class BackgroundJobTurnFailureError extends Error {
|
|
59
58
|
override name = "BackgroundJobTurnFailureError";
|
|
60
|
-
readonly
|
|
61
|
-
constructor(
|
|
59
|
+
readonly turnFailure: TurnFailure;
|
|
60
|
+
constructor(turnFailure: TurnFailure) {
|
|
62
61
|
super(
|
|
63
|
-
failureCode
|
|
62
|
+
turnFailure.failureCode
|
|
63
|
+
? `Agent turn failed (${turnFailure.failureCode})`
|
|
64
|
+
: "Agent turn failed",
|
|
64
65
|
);
|
|
65
|
-
this.
|
|
66
|
+
this.turnFailure = turnFailure;
|
|
66
67
|
}
|
|
67
68
|
}
|
|
68
69
|
|
|
@@ -203,13 +204,14 @@ export interface RunBackgroundJobResult {
|
|
|
203
204
|
error?: Error;
|
|
204
205
|
errorKind?: BackgroundJobErrorKind;
|
|
205
206
|
/**
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
207
|
+
* The failing turn's full carried classification (stable failureCode,
|
|
208
|
+
* authored user message, category, connection/profile attribution), when
|
|
209
|
+
* the failure was a non-throwing turn failure. Absent for timeouts and
|
|
210
|
+
* thrown exceptions. The single classification field on the result:
|
|
211
|
+
* callers branch on `turnFailure?.failureCode` and forward the object
|
|
212
|
+
* wholesale when reporting the failure onward.
|
|
211
213
|
*/
|
|
212
|
-
|
|
214
|
+
turnFailure?: TurnFailure;
|
|
213
215
|
/**
|
|
214
216
|
* Set when the runner declined to execute. Callers can distinguish a
|
|
215
217
|
* skipped job from a successful one even though both report `ok: true`.
|
|
@@ -381,18 +383,17 @@ export async function runBackgroundJob(
|
|
|
381
383
|
// Rethrow so it flows through the shared failure path below (logging +
|
|
382
384
|
// `activity.failed` emission) rather than returning `ok: true`.
|
|
383
385
|
if (runResult.turnFailure) {
|
|
384
|
-
throw new BackgroundJobTurnFailureError(
|
|
385
|
-
runResult.turnFailure.failureCode,
|
|
386
|
-
);
|
|
386
|
+
throw new BackgroundJobTurnFailureError(runResult.turnFailure);
|
|
387
387
|
}
|
|
388
388
|
return { conversationId: conversation.id, ok: true };
|
|
389
389
|
} catch (err) {
|
|
390
390
|
const errorKind = classifyError(err);
|
|
391
391
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
392
|
-
const
|
|
392
|
+
const turnFailure =
|
|
393
393
|
err instanceof BackgroundJobTurnFailureError
|
|
394
|
-
? err.
|
|
394
|
+
? err.turnFailure
|
|
395
395
|
: undefined;
|
|
396
|
+
const failureCode = turnFailure?.failureCode;
|
|
396
397
|
// Bootstrap can fail before `conversation` is assigned; fall back to ""
|
|
397
398
|
// so the structured failure result still flows to the caller.
|
|
398
399
|
const conversationId = conversation?.id ?? "";
|
|
@@ -408,24 +409,17 @@ export async function runBackgroundJob(
|
|
|
408
409
|
);
|
|
409
410
|
|
|
410
411
|
if (!opts.suppressFailureNotifications) {
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
//
|
|
418
|
-
//
|
|
419
|
-
|
|
420
|
-
// dispatch uses, so this recomputation cannot accept a profile that
|
|
421
|
-
// dispatch rejected (e.g. a pin whose connection was force-deleted).
|
|
422
|
-
// A mix winner, no named winner, or a config read failing on this path
|
|
423
|
-
// all yield no scope, and the key falls back to per-job: a mix's arm
|
|
424
|
-
// is picked per conversation and can span providers, so no single
|
|
425
|
-
// route can be named for it after the fact.
|
|
426
|
-
let providerScope: string | undefined;
|
|
412
|
+
// Fallback scope for failures that carry no attribution: the profile
|
|
413
|
+
// key the resolver would use for this call site, computed with the
|
|
414
|
+
// same provider-resolvability predicate dispatch uses so the
|
|
415
|
+
// recomputation cannot accept a profile dispatch rejected (e.g. a pin
|
|
416
|
+
// whose connection was force-deleted). A mix winner, no named winner,
|
|
417
|
+
// or a config read failing on this path all yield no scope, and the
|
|
418
|
+
// key falls back to per-job. A carried `turnFailure.profileName` is
|
|
419
|
+
// dispatch's own attribution and takes precedence over all of this.
|
|
420
|
+
let fallbackProviderScope: string | undefined;
|
|
427
421
|
try {
|
|
428
|
-
|
|
422
|
+
fallbackProviderScope = resolveSingleRouteProfileKey(
|
|
429
423
|
opts.callSite,
|
|
430
424
|
getConfig().llm,
|
|
431
425
|
{
|
|
@@ -436,34 +430,30 @@ export async function runBackgroundJob(
|
|
|
436
430
|
},
|
|
437
431
|
);
|
|
438
432
|
} catch {
|
|
439
|
-
|
|
433
|
+
fallbackProviderScope = undefined;
|
|
440
434
|
}
|
|
441
|
-
|
|
435
|
+
emitBackgroundFailureSignal({
|
|
442
436
|
jobName: opts.jobName,
|
|
443
|
-
errorKind,
|
|
444
|
-
...(failureCode !== undefined ? { failureCode } : {}),
|
|
445
|
-
...(providerScope !== undefined ? { providerScope } : {}),
|
|
446
|
-
});
|
|
447
|
-
emitNotificationSignal({
|
|
448
437
|
sourceChannel: "assistant_tool",
|
|
449
438
|
sourceContextId: conversationId,
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
{
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
},
|
|
465
|
-
|
|
466
|
-
|
|
439
|
+
errorKind,
|
|
440
|
+
errorMessage: error.message,
|
|
441
|
+
...(failureCode !== undefined ? { failureCode } : {}),
|
|
442
|
+
...(turnFailure?.userMessage !== undefined
|
|
443
|
+
? { failureSummary: turnFailure.userMessage }
|
|
444
|
+
: {}),
|
|
445
|
+
...(turnFailure?.errorCategory !== undefined
|
|
446
|
+
? { errorCategory: turnFailure.errorCategory }
|
|
447
|
+
: {}),
|
|
448
|
+
...(turnFailure?.connectionName !== undefined
|
|
449
|
+
? { connectionName: turnFailure.connectionName }
|
|
450
|
+
: {}),
|
|
451
|
+
...(turnFailure?.profileName !== undefined
|
|
452
|
+
? { profileName: turnFailure.profileName }
|
|
453
|
+
: {}),
|
|
454
|
+
...(fallbackProviderScope !== undefined
|
|
455
|
+
? { fallbackProviderScope }
|
|
456
|
+
: {}),
|
|
467
457
|
});
|
|
468
458
|
}
|
|
469
459
|
|
|
@@ -472,7 +462,7 @@ export async function runBackgroundJob(
|
|
|
472
462
|
ok: false,
|
|
473
463
|
error,
|
|
474
464
|
errorKind,
|
|
475
|
-
...(
|
|
465
|
+
...(turnFailure !== undefined ? { turnFailure } : {}),
|
|
476
466
|
};
|
|
477
467
|
} finally {
|
|
478
468
|
if (timer) {
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
} from "../daemon/disk-pressure-background-gate.js";
|
|
9
9
|
import { processMessage } from "../daemon/process-message.js";
|
|
10
10
|
import { INTERNAL_GUARDIAN_TRUST_CONTEXT } from "../daemon/trust-context.js";
|
|
11
|
-
import {
|
|
11
|
+
import { emitBackgroundFailureSignal } from "../notifications/background-failure-signal.js";
|
|
12
12
|
import { emitNotificationSignal } from "../notifications/emit-signal.js";
|
|
13
13
|
import { getConversation } from "../persistence/conversation-crud.js";
|
|
14
14
|
import { isLifecycleQuiesced } from "../persistence/lifecycle-quiesce.js";
|
|
@@ -235,7 +235,12 @@ async function handleExecutionFailure(params: {
|
|
|
235
235
|
* collapse; absent means a generic exception, which keys per schedule.
|
|
236
236
|
*/
|
|
237
237
|
errorKind?: BackgroundJobErrorKind;
|
|
238
|
-
|
|
238
|
+
/**
|
|
239
|
+
* The failing turn's carried classification, when the failure was a
|
|
240
|
+
* non-throwing turn failure. The exhaustion alert renders its authored
|
|
241
|
+
* message and scopes by the route that actually served the runs.
|
|
242
|
+
*/
|
|
243
|
+
turnFailure?: TurnFailure;
|
|
239
244
|
}): Promise<void> {
|
|
240
245
|
const decision = decideRetry(params.job);
|
|
241
246
|
await applyRetryDecision({
|
|
@@ -246,16 +251,33 @@ async function handleExecutionFailure(params: {
|
|
|
246
251
|
failOneShotPermanently,
|
|
247
252
|
resetRetryCount,
|
|
248
253
|
emitAlert: () => {
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
jobName: params.job.
|
|
253
|
-
|
|
254
|
+
const fallbackProviderScope = resolveScheduleProviderScope(params.job);
|
|
255
|
+
const carried = params.turnFailure;
|
|
256
|
+
emitBackgroundFailureSignal({
|
|
257
|
+
jobName: `schedule:${params.job.id}`,
|
|
258
|
+
displayName: `schedule:${params.job.name}`,
|
|
259
|
+
sourceChannel: "scheduler",
|
|
260
|
+
sourceContextId: params.job.id,
|
|
254
261
|
errorKind: params.errorKind ?? "exception",
|
|
255
|
-
|
|
256
|
-
|
|
262
|
+
errorMessage: params.errorMsg,
|
|
263
|
+
...(carried?.failureCode !== undefined
|
|
264
|
+
? { failureCode: carried.failureCode }
|
|
265
|
+
: {}),
|
|
266
|
+
...(carried?.userMessage !== undefined
|
|
267
|
+
? { failureSummary: carried.userMessage }
|
|
268
|
+
: {}),
|
|
269
|
+
...(carried?.errorCategory !== undefined
|
|
270
|
+
? { errorCategory: carried.errorCategory }
|
|
271
|
+
: {}),
|
|
272
|
+
...(carried?.connectionName !== undefined
|
|
273
|
+
? { connectionName: carried.connectionName }
|
|
274
|
+
: {}),
|
|
275
|
+
...(carried?.profileName !== undefined
|
|
276
|
+
? { profileName: carried.profileName }
|
|
277
|
+
: {}),
|
|
278
|
+
...(fallbackProviderScope !== undefined
|
|
279
|
+
? { fallbackProviderScope }
|
|
257
280
|
: {}),
|
|
258
|
-
...(providerScope !== undefined ? { providerScope } : {}),
|
|
259
281
|
});
|
|
260
282
|
},
|
|
261
283
|
log,
|
|
@@ -938,7 +960,7 @@ export async function runDueSchedulesOnce(
|
|
|
938
960
|
let ok: boolean;
|
|
939
961
|
let errorMsg: string | undefined;
|
|
940
962
|
let errorKind: BackgroundJobErrorKind | undefined;
|
|
941
|
-
let
|
|
963
|
+
let failedTurn: TurnFailure | undefined;
|
|
942
964
|
const conversationReused = reusedConversationId != null;
|
|
943
965
|
let runConversationId = reusedConversationId;
|
|
944
966
|
const runId = await createScheduleRun(job.id, reusedConversationId);
|
|
@@ -973,7 +995,7 @@ export async function runDueSchedulesOnce(
|
|
|
973
995
|
ok = false;
|
|
974
996
|
errorMsg = describeTurnFailure(turnFailure);
|
|
975
997
|
errorKind = "model_provider";
|
|
976
|
-
|
|
998
|
+
failedTurn = turnFailure;
|
|
977
999
|
} else {
|
|
978
1000
|
ok = true;
|
|
979
1001
|
}
|
|
@@ -1040,7 +1062,7 @@ export async function runDueSchedulesOnce(
|
|
|
1040
1062
|
ok = result.ok;
|
|
1041
1063
|
errorMsg = result.error?.message;
|
|
1042
1064
|
errorKind = result.errorKind;
|
|
1043
|
-
|
|
1065
|
+
failedTurn = result.turnFailure;
|
|
1044
1066
|
}
|
|
1045
1067
|
|
|
1046
1068
|
if (ok) {
|
|
@@ -1070,7 +1092,7 @@ export async function runDueSchedulesOnce(
|
|
|
1070
1092
|
errorMsg: errorMsg ?? "Schedule run failed",
|
|
1071
1093
|
isOneShot,
|
|
1072
1094
|
...(errorKind !== undefined ? { errorKind } : {}),
|
|
1073
|
-
...(
|
|
1095
|
+
...(failedTurn !== undefined ? { turnFailure: failedTurn } : {}),
|
|
1074
1096
|
});
|
|
1075
1097
|
|
|
1076
1098
|
// Only skip invalidation when the conversation was *actually* reused,
|
|
@@ -1094,55 +1116,3 @@ export async function runDueSchedulesOnce(
|
|
|
1094
1116
|
|
|
1095
1117
|
return result;
|
|
1096
1118
|
}
|
|
1097
|
-
|
|
1098
|
-
/**
|
|
1099
|
-
* Emit an `activity.failed` notification for a schedule whose retries have
|
|
1100
|
-
* been exhausted. Fires once when the retry policy has given up. The dedupe
|
|
1101
|
-
* key comes from the shared cause-first derivation, so every schedule (and
|
|
1102
|
-
* background job) failing for the same classified cause on the same UTC day
|
|
1103
|
-
* collapses into one notification, and cause-less failures collapse per
|
|
1104
|
-
* schedule per day.
|
|
1105
|
-
*/
|
|
1106
|
-
function emitScheduleActivityFailed(args: {
|
|
1107
|
-
jobId: string;
|
|
1108
|
-
jobName: string;
|
|
1109
|
-
errorMessage: string;
|
|
1110
|
-
errorKind: BackgroundJobErrorKind;
|
|
1111
|
-
failureCode?: string;
|
|
1112
|
-
providerScope?: string;
|
|
1113
|
-
}): void {
|
|
1114
|
-
emitNotificationSignal({
|
|
1115
|
-
sourceChannel: "scheduler",
|
|
1116
|
-
sourceContextId: args.jobId,
|
|
1117
|
-
sourceEventName: "activity.failed",
|
|
1118
|
-
dedupeKey: activityFailedDedupeKey({
|
|
1119
|
-
jobName: `schedule:${args.jobId}`,
|
|
1120
|
-
errorKind: args.errorKind,
|
|
1121
|
-
...(args.failureCode !== undefined
|
|
1122
|
-
? { failureCode: args.failureCode }
|
|
1123
|
-
: {}),
|
|
1124
|
-
...(args.providerScope !== undefined
|
|
1125
|
-
? { providerScope: args.providerScope }
|
|
1126
|
-
: {}),
|
|
1127
|
-
}),
|
|
1128
|
-
contextPayload: {
|
|
1129
|
-
jobName: `schedule:${args.jobName}`,
|
|
1130
|
-
errorMessage: args.errorMessage,
|
|
1131
|
-
errorKind: args.errorKind,
|
|
1132
|
-
},
|
|
1133
|
-
attentionHints: {
|
|
1134
|
-
requiresAction: false,
|
|
1135
|
-
urgency: "medium",
|
|
1136
|
-
isAsyncBackground: true,
|
|
1137
|
-
visibleInSourceNow: false,
|
|
1138
|
-
},
|
|
1139
|
-
}).catch((emitErr) => {
|
|
1140
|
-
log.warn(
|
|
1141
|
-
{
|
|
1142
|
-
err: emitErr instanceof Error ? emitErr.message : String(emitErr),
|
|
1143
|
-
jobId: args.jobId,
|
|
1144
|
-
},
|
|
1145
|
-
"Failed to emit activity.failed notification for exhausted schedule",
|
|
1146
|
-
);
|
|
1147
|
-
});
|
|
1148
|
-
}
|
|
@@ -31,6 +31,26 @@ export interface TurnOutcomeExtras {
|
|
|
31
31
|
batchedInto?: string;
|
|
32
32
|
/** For `"failed"`: stable classified error code (never free-form text). */
|
|
33
33
|
failureCode?: string;
|
|
34
|
+
/**
|
|
35
|
+
* For `"failed"`: the classification's authored user-facing message
|
|
36
|
+
* (`classifyConversationError(...).userMessage`), so downstream failure
|
|
37
|
+
* reporting (background-job alerts) can render the actionable prose the
|
|
38
|
+
* chat banner shows instead of reconstructing degraded text from the code.
|
|
39
|
+
*/
|
|
40
|
+
failureMessage?: string;
|
|
41
|
+
/** For `"failed"`: the classification's machine-readable category. */
|
|
42
|
+
failureCategory?: string;
|
|
43
|
+
/**
|
|
44
|
+
* For `"failed"`: name of the `provider_connections` row in play when the
|
|
45
|
+
* failure occurred, when the classifier had it in scope.
|
|
46
|
+
*/
|
|
47
|
+
failureConnection?: string;
|
|
48
|
+
/**
|
|
49
|
+
* For `"failed"`: the resolved profile key the failing call ran under,
|
|
50
|
+
* when the classifier had it in scope. This is dispatch's own attribution,
|
|
51
|
+
* so consumers scope failure identity by it instead of re-resolving.
|
|
52
|
+
*/
|
|
53
|
+
failureProfile?: string;
|
|
34
54
|
}
|
|
35
55
|
|
|
36
56
|
/**
|
|
@@ -49,10 +69,25 @@ export function stampTurnOutcome(
|
|
|
49
69
|
extras: TurnOutcomeExtras = {},
|
|
50
70
|
): void {
|
|
51
71
|
try {
|
|
72
|
+
// The failure* keys are read back by `readTurnFailure` only; the turn
|
|
73
|
+
// telemetry projection (`queryUnreportedTurnEvents`) extracts none of
|
|
74
|
+
// them, so nothing here changes the platform wire contract.
|
|
52
75
|
updateMessageMetadata(userMessageId, {
|
|
53
76
|
turnOutcome: outcome,
|
|
54
77
|
...(extras.batchedInto ? { turnBatchedInto: extras.batchedInto } : {}),
|
|
55
78
|
...(extras.failureCode ? { turnFailureCode: extras.failureCode } : {}),
|
|
79
|
+
...(extras.failureMessage
|
|
80
|
+
? { turnFailureMessage: extras.failureMessage }
|
|
81
|
+
: {}),
|
|
82
|
+
...(extras.failureCategory
|
|
83
|
+
? { turnFailureCategory: extras.failureCategory }
|
|
84
|
+
: {}),
|
|
85
|
+
...(extras.failureConnection
|
|
86
|
+
? { turnFailureConnection: extras.failureConnection }
|
|
87
|
+
: {}),
|
|
88
|
+
...(extras.failureProfile
|
|
89
|
+
? { turnFailureProfile: extras.failureProfile }
|
|
90
|
+
: {}),
|
|
56
91
|
});
|
|
57
92
|
} catch (err) {
|
|
58
93
|
log.warn(
|
|
@@ -75,6 +110,14 @@ export function stampTurnOutcome(
|
|
|
75
110
|
export interface TurnFailure {
|
|
76
111
|
/** Stable classified error code (never free-form text). */
|
|
77
112
|
failureCode?: string;
|
|
113
|
+
/** The classification's authored user-facing message, when stamped. */
|
|
114
|
+
userMessage?: string;
|
|
115
|
+
/** The classification's machine-readable category, when stamped. */
|
|
116
|
+
errorCategory?: string;
|
|
117
|
+
/** Connection row in play when the failure occurred, when stamped. */
|
|
118
|
+
connectionName?: string;
|
|
119
|
+
/** Resolved profile key the failing call ran under, when stamped. */
|
|
120
|
+
profileName?: string;
|
|
78
121
|
}
|
|
79
122
|
|
|
80
123
|
/**
|
|
@@ -106,11 +149,39 @@ export function readTurnFailure(userMessageId: string): TurnFailure | null {
|
|
|
106
149
|
if (typeof parsed !== "object" || parsed === null) {
|
|
107
150
|
return null;
|
|
108
151
|
}
|
|
109
|
-
const meta = parsed as {
|
|
152
|
+
const meta = parsed as {
|
|
153
|
+
turnOutcome?: unknown;
|
|
154
|
+
turnFailureCode?: unknown;
|
|
155
|
+
turnFailureMessage?: unknown;
|
|
156
|
+
turnFailureCategory?: unknown;
|
|
157
|
+
turnFailureConnection?: unknown;
|
|
158
|
+
turnFailureProfile?: unknown;
|
|
159
|
+
};
|
|
110
160
|
if (meta.turnOutcome !== "failed") {
|
|
111
161
|
return null;
|
|
112
162
|
}
|
|
113
|
-
|
|
114
|
-
?
|
|
115
|
-
|
|
163
|
+
const str = (value: unknown): string | undefined =>
|
|
164
|
+
typeof value === "string" ? value : undefined;
|
|
165
|
+
const failure: TurnFailure = {};
|
|
166
|
+
const failureCode = str(meta.turnFailureCode);
|
|
167
|
+
if (failureCode !== undefined) {
|
|
168
|
+
failure.failureCode = failureCode;
|
|
169
|
+
}
|
|
170
|
+
const userMessage = str(meta.turnFailureMessage);
|
|
171
|
+
if (userMessage !== undefined) {
|
|
172
|
+
failure.userMessage = userMessage;
|
|
173
|
+
}
|
|
174
|
+
const errorCategory = str(meta.turnFailureCategory);
|
|
175
|
+
if (errorCategory !== undefined) {
|
|
176
|
+
failure.errorCategory = errorCategory;
|
|
177
|
+
}
|
|
178
|
+
const connectionName = str(meta.turnFailureConnection);
|
|
179
|
+
if (connectionName !== undefined) {
|
|
180
|
+
failure.connectionName = connectionName;
|
|
181
|
+
}
|
|
182
|
+
const profileName = str(meta.turnFailureProfile);
|
|
183
|
+
if (profileName !== undefined) {
|
|
184
|
+
failure.profileName = profileName;
|
|
185
|
+
}
|
|
186
|
+
return failure;
|
|
116
187
|
}
|