@opengeni/worker-bundle 0.22.4-canary.1 → 0.22.7-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +20 -0
- package/dist/activities/agent-turn/governance-model.d.ts +9 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +14 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +8 -0
- package/dist/activities/generated-images.d.ts +3 -1
- package/dist/activities/image-generation-references.d.ts +39 -7
- package/dist/activities/sandbox-lease.d.ts +12 -0
- package/dist/activities/types.d.ts +24 -0
- package/dist/{activities-control-4XVBA3Q5.js → activities-control-VZIIV7RX.js} +77 -12
- package/dist/activities-control-VZIIV7RX.js.map +1 -0
- package/dist/{activities-turn-J3CR27E6.js → activities-turn-5RGQDBFV.js} +253 -51
- package/dist/activities-turn-5RGQDBFV.js.map +1 -0
- package/dist/{chunk-FA35XBBC.js → chunk-NAVSAZ56.js} +1 -1
- package/dist/chunk-NAVSAZ56.js.map +1 -0
- package/dist/{chunk-XWQPS6GG.js → chunk-O4HDX5SC.js} +59 -10
- package/dist/{chunk-XWQPS6GG.js.map → chunk-O4HDX5SC.js.map} +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/workflow-bundle.js +62 -9
- package/dist/workflows/session.d.ts +4 -1
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +29 -18
- package/src/activities/agent-turn/claim.ts +6 -6
- package/src/activities/agent-turn/errors.ts +104 -21
- package/src/activities/agent-turn/failure-settlement.ts +40 -0
- package/src/activities/agent-turn/governance-model.ts +31 -4
- package/src/activities/agent-turn/history.ts +7 -3
- package/src/activities/agent-turn/image-reference-sandbox.ts +35 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +13 -0
- package/src/activities/agent-turn/run.ts +7 -0
- package/src/activities/agent-turn/stream-attempt.ts +25 -2
- package/src/activities/generated-images.ts +5 -2
- package/src/activities/image-generation-references.ts +184 -37
- package/src/activities/sandbox-lease.ts +48 -9
- package/src/activities/session-state.ts +69 -1
- package/src/activities/types.ts +26 -0
- package/src/observability-metrics.ts +2 -0
- package/src/workflows/session.ts +95 -21
- package/dist/activities-control-4XVBA3Q5.js.map +0 -1
- package/dist/activities-turn-J3CR27E6.js.map +0 -1
- package/dist/chunk-FA35XBBC.js.map +0 -1
|
@@ -93,6 +93,7 @@ import {
|
|
|
93
93
|
safeErrorDiagnostic,
|
|
94
94
|
compactionFailureReasonFromError,
|
|
95
95
|
isCompactionSummaryFailure,
|
|
96
|
+
PostCompactionContinuationEmptyError,
|
|
96
97
|
shouldRecoverCompactionProviderFailure,
|
|
97
98
|
classifyContextWindowOverflowError,
|
|
98
99
|
} from "./errors";
|
|
@@ -101,6 +102,7 @@ import {
|
|
|
101
102
|
toolCallProducesRetainableSessionImage,
|
|
102
103
|
completedToolCallFromSdkEvent,
|
|
103
104
|
} from "./history";
|
|
105
|
+
import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
|
|
104
106
|
import {
|
|
105
107
|
modelUsageSourceKey,
|
|
106
108
|
recordCompletedModelCallBeforeOwnershipFences,
|
|
@@ -457,10 +459,13 @@ export async function runTurnStreamAttempt(
|
|
|
457
459
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
458
460
|
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
459
461
|
let workerPreparationTotalRecorded = false;
|
|
460
|
-
const runStreamAttempt = async (
|
|
462
|
+
const runStreamAttempt = async (options: {
|
|
463
|
+
requireTerminalModelResponse: boolean;
|
|
464
|
+
}): Promise<RunAgentTurnResult> => {
|
|
461
465
|
if (!runInput) {
|
|
462
466
|
throw new Error("Run input was not prepared");
|
|
463
467
|
}
|
|
468
|
+
const responseCountBeforeStream = modelResponseState.responseCount;
|
|
464
469
|
eventing.stream = undefined;
|
|
465
470
|
eventing.batcher = null;
|
|
466
471
|
// The SDK emits every processed call item for one model response before
|
|
@@ -481,6 +486,7 @@ export async function runTurnStreamAttempt(
|
|
|
481
486
|
resolvedModel?.provider.kind === "xai-subscription";
|
|
482
487
|
let fallbackProviderRequestStartedAt: number | null = null;
|
|
483
488
|
const recordFallbackProviderDispatchAtWire = async (): Promise<void> => {
|
|
489
|
+
await checkpointHistoryBeforeProviderDispatch(historySink);
|
|
484
490
|
if (
|
|
485
491
|
providerPublishesNativeRequestEvents ||
|
|
486
492
|
eventing.firstModelRequestPreparationRecorded ||
|
|
@@ -1226,6 +1232,21 @@ export async function runTurnStreamAttempt(
|
|
|
1226
1232
|
eventing.stream.completed.catch(() => undefined),
|
|
1227
1233
|
cancellationSignal,
|
|
1228
1234
|
);
|
|
1235
|
+
if (
|
|
1236
|
+
options.requireTerminalModelResponse &&
|
|
1237
|
+
eventing.stream.interruptions.length === 0 &&
|
|
1238
|
+
modelResponseState.responseCount === responseCountBeforeStream
|
|
1239
|
+
) {
|
|
1240
|
+
// A fresh post-compaction stream may be returned already cancelled by
|
|
1241
|
+
// the SDK/runtime without yielding an event or terminal response. That
|
|
1242
|
+
// is not a completed logical turn: accepting finalOutput's undefined ->
|
|
1243
|
+
// empty-string fallback would release the queue and start newer user
|
|
1244
|
+
// work. Cancellation retains priority; otherwise checkpoint and recover
|
|
1245
|
+
// this exact turn from the durable compacted history.
|
|
1246
|
+
throwIfWorkerShuttingDown();
|
|
1247
|
+
throwIfTurnCancelled();
|
|
1248
|
+
throw new PostCompactionContinuationEmptyError();
|
|
1249
|
+
}
|
|
1229
1250
|
if (!streamSawPerResponseUsage) {
|
|
1230
1251
|
const aggregateUsage = eventing.stream.state.usage;
|
|
1231
1252
|
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
@@ -1529,7 +1550,9 @@ export async function runTurnStreamAttempt(
|
|
|
1529
1550
|
let retriedAfterCompaction = false;
|
|
1530
1551
|
while (true) {
|
|
1531
1552
|
try {
|
|
1532
|
-
const result = await runStreamAttempt(
|
|
1553
|
+
const result = await runStreamAttempt({
|
|
1554
|
+
requireTerminalModelResponse: retriedAfterCompaction,
|
|
1555
|
+
});
|
|
1533
1556
|
if (retriedAfterCompaction) {
|
|
1534
1557
|
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1535
1558
|
sessionId: input.sessionId,
|
|
@@ -50,7 +50,10 @@ export type RetainedGeneratedImage = {
|
|
|
50
50
|
};
|
|
51
51
|
|
|
52
52
|
export class GeneratedImageValidationError extends Error {
|
|
53
|
-
constructor(
|
|
53
|
+
constructor(
|
|
54
|
+
message: string,
|
|
55
|
+
readonly reason?: ScreenshotValidationError["reason"],
|
|
56
|
+
) {
|
|
54
57
|
super(message);
|
|
55
58
|
this.name = "GeneratedImageValidationError";
|
|
56
59
|
}
|
|
@@ -211,7 +214,7 @@ export function validateGeneratedImage(input: {
|
|
|
211
214
|
);
|
|
212
215
|
} catch (error) {
|
|
213
216
|
if (error instanceof ScreenshotValidationError) {
|
|
214
|
-
throw new GeneratedImageValidationError(error.message);
|
|
217
|
+
throw new GeneratedImageValidationError(error.message, error.reason);
|
|
215
218
|
}
|
|
216
219
|
throw error;
|
|
217
220
|
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { type ImageGenerationReference, type FileAsset } from "@opengeni/contracts";
|
|
2
2
|
import { getGeneratedImageArtifact, requireFileForSubject, type Database } from "@opengeni/db";
|
|
3
3
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
GeneratedImageValidationError,
|
|
6
|
+
validateGeneratedImage,
|
|
7
|
+
type GeneratedImageMediaType,
|
|
8
|
+
} from "./generated-images";
|
|
5
9
|
|
|
6
10
|
export const IMAGE_GENERATION_REFERENCE_MAX_BYTES = 30 * 1024 * 1024;
|
|
7
11
|
export const IMAGE_GENERATION_REFERENCES_MAX_TOTAL_BYTES = 64 * 1024 * 1024;
|
|
@@ -15,12 +19,36 @@ export type ResolvedImageGenerationReference = Readonly<{
|
|
|
15
19
|
|
|
16
20
|
export type SandboxImageReferenceReader = (path: string, maxBytes: number) => Promise<Uint8Array>;
|
|
17
21
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
export type ImageGenerationReferenceErrorCode =
|
|
23
|
+
| "invalid_reference"
|
|
24
|
+
| "unsupported_reference_media"
|
|
25
|
+
| "reference_too_large"
|
|
26
|
+
| "reference_not_ready"
|
|
27
|
+
| "reference_unavailable"
|
|
28
|
+
| "reference_integrity_mismatch";
|
|
29
|
+
|
|
30
|
+
/** A deterministic input rejection that occurs before any image provider request. */
|
|
31
|
+
export class ImageGenerationReferenceError extends Error {
|
|
32
|
+
constructor(
|
|
33
|
+
readonly code: ImageGenerationReferenceErrorCode,
|
|
34
|
+
message: string,
|
|
35
|
+
options?: ErrorOptions,
|
|
36
|
+
) {
|
|
37
|
+
super(message, options);
|
|
38
|
+
this.name = "ImageGenerationReferenceError";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type ImageGenerationReferenceRejectedResult = Readonly<{
|
|
43
|
+
isError: true;
|
|
44
|
+
status: "rejected";
|
|
45
|
+
code: ImageGenerationReferenceErrorCode;
|
|
46
|
+
message: string;
|
|
47
|
+
operationCreated: false;
|
|
48
|
+
content: readonly [{ readonly type: "text"; readonly text: string }];
|
|
49
|
+
}>;
|
|
50
|
+
|
|
51
|
+
export type ResolveImageGenerationReferencesInput = Readonly<{
|
|
24
52
|
db: Database;
|
|
25
53
|
objectStorage: ObjectStorage;
|
|
26
54
|
accountId: string;
|
|
@@ -28,21 +56,77 @@ export async function resolveImageGenerationReferences(input: {
|
|
|
28
56
|
subjectId: string | null;
|
|
29
57
|
references: readonly ImageGenerationReference[];
|
|
30
58
|
readSandboxFile?: SandboxImageReferenceReader;
|
|
31
|
-
}
|
|
59
|
+
}>;
|
|
60
|
+
|
|
61
|
+
export type ImageGenerationReferenceResolution =
|
|
62
|
+
| Readonly<{
|
|
63
|
+
status: "resolved";
|
|
64
|
+
references: ResolvedImageGenerationReference[];
|
|
65
|
+
}>
|
|
66
|
+
| Readonly<{
|
|
67
|
+
status: "rejected";
|
|
68
|
+
result: ImageGenerationReferenceRejectedResult;
|
|
69
|
+
}>;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Carry deterministic pre-provider input failures on the tool output. Unknown
|
|
73
|
+
* database and object-storage failures still throw, and provider execution
|
|
74
|
+
* remains outside this boundary so an outcome-unknown operation can never be
|
|
75
|
+
* reported rejected.
|
|
76
|
+
*/
|
|
77
|
+
export async function resolveImageGenerationReferencesForTool(
|
|
78
|
+
input: ResolveImageGenerationReferencesInput,
|
|
79
|
+
): Promise<ImageGenerationReferenceResolution> {
|
|
80
|
+
try {
|
|
81
|
+
return {
|
|
82
|
+
status: "resolved",
|
|
83
|
+
references: await resolveImageGenerationReferences(input),
|
|
84
|
+
};
|
|
85
|
+
} catch (error) {
|
|
86
|
+
if (!(error instanceof ImageGenerationReferenceError)) throw error;
|
|
87
|
+
const text = `Image generation was not started: ${error.message}`;
|
|
88
|
+
return {
|
|
89
|
+
status: "rejected",
|
|
90
|
+
result: {
|
|
91
|
+
isError: true,
|
|
92
|
+
status: "rejected",
|
|
93
|
+
code: error.code,
|
|
94
|
+
message: error.message,
|
|
95
|
+
operationCreated: false,
|
|
96
|
+
content: [{ type: "text", text }],
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Resolve ordered model-facing references without accepting arbitrary URLs.
|
|
104
|
+
* Stored files remain workspace-RLS scoped; sandbox reads stay rooted beneath
|
|
105
|
+
* /workspace. Provider adapters receive only validated immutable bytes.
|
|
106
|
+
*/
|
|
107
|
+
export async function resolveImageGenerationReferences(
|
|
108
|
+
input: ResolveImageGenerationReferencesInput,
|
|
109
|
+
): Promise<ResolvedImageGenerationReference[]> {
|
|
32
110
|
const resolved: ResolvedImageGenerationReference[] = [];
|
|
33
111
|
let totalBytes = 0;
|
|
34
112
|
|
|
35
113
|
for (const reference of input.references) {
|
|
36
114
|
const source = await referenceBytes(input, reference);
|
|
37
115
|
if (source.bytes.byteLength > IMAGE_GENERATION_REFERENCE_MAX_BYTES) {
|
|
38
|
-
throw new
|
|
116
|
+
throw new ImageGenerationReferenceError(
|
|
117
|
+
"reference_too_large",
|
|
118
|
+
"An image reference exceeds the per-image byte limit.",
|
|
119
|
+
);
|
|
39
120
|
}
|
|
40
121
|
totalBytes += source.bytes.byteLength;
|
|
41
122
|
if (totalBytes > IMAGE_GENERATION_REFERENCES_MAX_TOTAL_BYTES) {
|
|
42
|
-
throw new
|
|
123
|
+
throw new ImageGenerationReferenceError(
|
|
124
|
+
"reference_too_large",
|
|
125
|
+
"The image references exceed the combined byte limit.",
|
|
126
|
+
);
|
|
43
127
|
}
|
|
44
128
|
|
|
45
|
-
const image =
|
|
129
|
+
const image = validateReferenceImage(source.bytes);
|
|
46
130
|
if (source.file) assertStoredFileMatches(source.file, image);
|
|
47
131
|
resolved.push({
|
|
48
132
|
mediaType: image.mediaType,
|
|
@@ -56,39 +140,64 @@ export async function resolveImageGenerationReferences(input: {
|
|
|
56
140
|
}
|
|
57
141
|
|
|
58
142
|
async function referenceBytes(
|
|
59
|
-
input:
|
|
60
|
-
db: Database;
|
|
61
|
-
objectStorage: ObjectStorage;
|
|
62
|
-
accountId: string;
|
|
63
|
-
workspaceId: string;
|
|
64
|
-
subjectId: string | null;
|
|
65
|
-
readSandboxFile?: SandboxImageReferenceReader;
|
|
66
|
-
},
|
|
143
|
+
input: ResolveImageGenerationReferencesInput,
|
|
67
144
|
reference: ImageGenerationReference,
|
|
68
145
|
): Promise<{ bytes: Uint8Array; file?: FileAsset }> {
|
|
69
146
|
if (reference.kind === "sandbox_path") {
|
|
70
147
|
if (!input.readSandboxFile) {
|
|
71
|
-
throw new
|
|
148
|
+
throw new ImageGenerationReferenceError(
|
|
149
|
+
"reference_unavailable",
|
|
150
|
+
"Sandbox image references require an available workspace sandbox.",
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
return {
|
|
155
|
+
bytes: await input.readSandboxFile(
|
|
156
|
+
reference.path,
|
|
157
|
+
IMAGE_GENERATION_REFERENCE_MAX_BYTES + 1,
|
|
158
|
+
),
|
|
159
|
+
};
|
|
160
|
+
} catch (error) {
|
|
161
|
+
if (error instanceof ImageGenerationReferenceError) throw error;
|
|
162
|
+
throw new ImageGenerationReferenceError(
|
|
163
|
+
"reference_unavailable",
|
|
164
|
+
"The sandbox image reference could not be read. Verify that the file still exists in the current workspace.",
|
|
165
|
+
{ cause: error },
|
|
166
|
+
);
|
|
72
167
|
}
|
|
73
|
-
return {
|
|
74
|
-
bytes: await input.readSandboxFile(reference.path, IMAGE_GENERATION_REFERENCE_MAX_BYTES + 1),
|
|
75
|
-
};
|
|
76
168
|
}
|
|
77
169
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
170
|
+
let file: FileAsset;
|
|
171
|
+
if (reference.kind === "file") {
|
|
172
|
+
try {
|
|
173
|
+
file = await requireFileForSubject(input.db, {
|
|
174
|
+
accountId: input.accountId,
|
|
175
|
+
workspaceId: input.workspaceId,
|
|
176
|
+
subjectId: input.subjectId,
|
|
177
|
+
fileId: reference.fileId,
|
|
178
|
+
});
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (!(error instanceof Error) || !error.message.startsWith("File not found:")) throw error;
|
|
181
|
+
throw new ImageGenerationReferenceError(
|
|
182
|
+
"reference_unavailable",
|
|
183
|
+
"The workspace File image reference is unavailable.",
|
|
184
|
+
{ cause: error },
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
file = await artifactFile(input.db, input.workspaceId, reference.artifactId);
|
|
189
|
+
}
|
|
87
190
|
if (file.status !== "ready" || !file.sha256) {
|
|
88
|
-
throw new
|
|
191
|
+
throw new ImageGenerationReferenceError(
|
|
192
|
+
"reference_not_ready",
|
|
193
|
+
"The image reference is not durably ready yet.",
|
|
194
|
+
);
|
|
89
195
|
}
|
|
90
196
|
if (file.sizeBytes <= 0 || file.sizeBytes > IMAGE_GENERATION_REFERENCE_MAX_BYTES) {
|
|
91
|
-
throw new
|
|
197
|
+
throw new ImageGenerationReferenceError(
|
|
198
|
+
"reference_too_large",
|
|
199
|
+
"The image reference file exceeds the supported size.",
|
|
200
|
+
);
|
|
92
201
|
}
|
|
93
202
|
return {
|
|
94
203
|
bytes: await readStoredFile(input.objectStorage, file),
|
|
@@ -101,7 +210,12 @@ async function readStoredFile(objectStorage: ObjectStorage, file: FileAsset): Pr
|
|
|
101
210
|
start: 0,
|
|
102
211
|
end: file.sizeBytes - 1,
|
|
103
212
|
});
|
|
104
|
-
if (!bytes)
|
|
213
|
+
if (!bytes) {
|
|
214
|
+
throw new ImageGenerationReferenceError(
|
|
215
|
+
"reference_unavailable",
|
|
216
|
+
"The durable image reference bytes are unavailable.",
|
|
217
|
+
);
|
|
218
|
+
}
|
|
105
219
|
return bytes;
|
|
106
220
|
}
|
|
107
221
|
|
|
@@ -112,7 +226,10 @@ async function artifactFile(
|
|
|
112
226
|
): Promise<FileAsset> {
|
|
113
227
|
const artifact = await getGeneratedImageArtifact(db, workspaceId, artifactId);
|
|
114
228
|
if (!artifact || artifact.status !== "ready") {
|
|
115
|
-
throw new
|
|
229
|
+
throw new ImageGenerationReferenceError(
|
|
230
|
+
"reference_unavailable",
|
|
231
|
+
"The generated-image artifact reference is unavailable.",
|
|
232
|
+
);
|
|
116
233
|
}
|
|
117
234
|
return artifact.file;
|
|
118
235
|
}
|
|
@@ -122,6 +239,36 @@ function assertStoredFileMatches(
|
|
|
122
239
|
image: Pick<ResolvedImageGenerationReference, "sizeBytes" | "sha256">,
|
|
123
240
|
): void {
|
|
124
241
|
if (file.sizeBytes !== image.sizeBytes || file.sha256 !== image.sha256) {
|
|
125
|
-
throw new
|
|
242
|
+
throw new ImageGenerationReferenceError(
|
|
243
|
+
"reference_integrity_mismatch",
|
|
244
|
+
"The image reference bytes do not match their durable file metadata.",
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function validateReferenceImage(bytes: Uint8Array) {
|
|
250
|
+
try {
|
|
251
|
+
return validateGeneratedImage({ bytes });
|
|
252
|
+
} catch (error) {
|
|
253
|
+
if (!(error instanceof GeneratedImageValidationError)) throw error;
|
|
254
|
+
if (error.reason === "unsupported") {
|
|
255
|
+
throw new ImageGenerationReferenceError(
|
|
256
|
+
"unsupported_reference_media",
|
|
257
|
+
"Image references must be PNG, JPEG, or WebP. Convert SVG or other formats before calling generate_image.",
|
|
258
|
+
{ cause: error },
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
if (error.reason === "oversized") {
|
|
262
|
+
throw new ImageGenerationReferenceError(
|
|
263
|
+
"reference_too_large",
|
|
264
|
+
"The image reference exceeds the supported dimensions or byte size.",
|
|
265
|
+
{ cause: error },
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
throw new ImageGenerationReferenceError(
|
|
269
|
+
"invalid_reference",
|
|
270
|
+
"The image reference is not a valid PNG, JPEG, or WebP image.",
|
|
271
|
+
{ cause: error },
|
|
272
|
+
);
|
|
126
273
|
}
|
|
127
274
|
}
|
|
@@ -309,6 +309,8 @@ export type DrainableProviderProbeFn = (
|
|
|
309
309
|
export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
310
310
|
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
311
311
|
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
312
|
+
export const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
|
|
313
|
+
export const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;
|
|
312
314
|
export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
313
315
|
export const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
314
316
|
|
|
@@ -1475,13 +1477,9 @@ async function deferRetainedProcessClaim(
|
|
|
1475
1477
|
process: SandboxRetainedProcess,
|
|
1476
1478
|
expected: ReturnType<typeof retainedProcessSettlementIdentity>,
|
|
1477
1479
|
claimId: string,
|
|
1478
|
-
outcome:
|
|
1480
|
+
outcome: RetainedProcessDeferralOutcome,
|
|
1479
1481
|
): Promise<void> {
|
|
1480
|
-
const
|
|
1481
|
-
const retryAfterMs = Math.min(
|
|
1482
|
-
5 * 60_000,
|
|
1483
|
-
Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
|
|
1484
|
-
);
|
|
1482
|
+
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
|
|
1485
1483
|
try {
|
|
1486
1484
|
await deferRetainedProcessReconciliation(db, {
|
|
1487
1485
|
accountId: process.accountId,
|
|
@@ -1490,10 +1488,10 @@ async function deferRetainedProcessClaim(
|
|
|
1490
1488
|
processId: process.id,
|
|
1491
1489
|
expected,
|
|
1492
1490
|
claimId,
|
|
1493
|
-
outcome,
|
|
1494
|
-
retryAfterMs,
|
|
1491
|
+
outcome: deferral.durableOutcome,
|
|
1492
|
+
retryAfterMs: deferral.retryAfterMs,
|
|
1495
1493
|
});
|
|
1496
|
-
recordRetainedProcessReconciliation(observability,
|
|
1494
|
+
recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
|
|
1497
1495
|
} catch (error) {
|
|
1498
1496
|
recordRetainedProcessReconciliation(observability, "defer_failed");
|
|
1499
1497
|
observability.warn("sandbox reaper: retained-process defer failed", {
|
|
@@ -1503,6 +1501,47 @@ async function deferRetainedProcessClaim(
|
|
|
1503
1501
|
}
|
|
1504
1502
|
}
|
|
1505
1503
|
|
|
1504
|
+
type RetainedProcessDeferralOutcome =
|
|
1505
|
+
| Extract<RetainedProcessProbeResult, { status: "deferred" }>["reason"]
|
|
1506
|
+
| "settlement_failed";
|
|
1507
|
+
|
|
1508
|
+
export function retainedProcessReconciliationDeferral(
|
|
1509
|
+
settings: Pick<ActivityServices["settings"], "sandboxLeaseReaperPeriodMs">,
|
|
1510
|
+
process: Pick<SandboxRetainedProcess, "reconcileAttempts">,
|
|
1511
|
+
outcome: RetainedProcessDeferralOutcome,
|
|
1512
|
+
): {
|
|
1513
|
+
durableOutcome: string;
|
|
1514
|
+
metricOutcome: RetainedProcessReconciliationOutcome;
|
|
1515
|
+
retryAfterMs: number;
|
|
1516
|
+
quarantined: boolean;
|
|
1517
|
+
} {
|
|
1518
|
+
const bindingQuarantine =
|
|
1519
|
+
process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS &&
|
|
1520
|
+
(outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch");
|
|
1521
|
+
if (bindingQuarantine) {
|
|
1522
|
+
return {
|
|
1523
|
+
durableOutcome: `quarantined_${outcome}`,
|
|
1524
|
+
metricOutcome:
|
|
1525
|
+
outcome === "provider_binding_missing"
|
|
1526
|
+
? "quarantined_binding_missing"
|
|
1527
|
+
: "quarantined_binding_mismatch",
|
|
1528
|
+
retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
|
|
1529
|
+
quarantined: true,
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1532
|
+
const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
1533
|
+
const retryAfterMs = Math.min(
|
|
1534
|
+
5 * 60_000,
|
|
1535
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
|
|
1536
|
+
);
|
|
1537
|
+
return {
|
|
1538
|
+
durableOutcome: outcome,
|
|
1539
|
+
metricOutcome: outcome,
|
|
1540
|
+
retryAfterMs,
|
|
1541
|
+
quarantined: false,
|
|
1542
|
+
};
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1506
1545
|
type RetainedProcessProbeClient = {
|
|
1507
1546
|
backendId: string;
|
|
1508
1547
|
resume?: (state: unknown) => Promise<unknown>;
|
|
@@ -143,7 +143,9 @@ export function createSessionStateActivities(
|
|
|
143
143
|
});
|
|
144
144
|
if (attempt) return { action: "stale" };
|
|
145
145
|
|
|
146
|
-
|
|
146
|
+
const preClaimFailureDisposition =
|
|
147
|
+
input.preClaimFailure?.disposition ?? input.preClaimFailureDisposition;
|
|
148
|
+
if (preClaimFailureDisposition === "permanent" && input.trigger) {
|
|
147
149
|
const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
|
|
148
150
|
accountId: input.accountId,
|
|
149
151
|
sessionId: input.sessionId,
|
|
@@ -180,6 +182,72 @@ export function createSessionStateActivities(
|
|
|
180
182
|
});
|
|
181
183
|
return { action: "unclaimed" };
|
|
182
184
|
}
|
|
185
|
+
const postClaimRecovery = input.postClaimDatabaseRecovery;
|
|
186
|
+
const postClaimIdentityMatches = Boolean(
|
|
187
|
+
postClaimRecovery &&
|
|
188
|
+
turn.id === postClaimRecovery.turnId &&
|
|
189
|
+
turn.triggerEventId === postClaimRecovery.triggerEventId &&
|
|
190
|
+
turn.executionGeneration === postClaimRecovery.executionGeneration,
|
|
191
|
+
);
|
|
192
|
+
const providerRecoveryCount = postClaimIdentityMatches
|
|
193
|
+
? postClaimRecovery?.providerRecoveryCount
|
|
194
|
+
: undefined;
|
|
195
|
+
const providerFailureCode = postClaimIdentityMatches
|
|
196
|
+
? postClaimRecovery?.providerFailureCode
|
|
197
|
+
: undefined;
|
|
198
|
+
const hasProviderRecoveryCount = providerRecoveryCount !== undefined;
|
|
199
|
+
const hasProviderFailureCode = providerFailureCode !== undefined;
|
|
200
|
+
if (hasProviderRecoveryCount !== hasProviderFailureCode) {
|
|
201
|
+
return { action: "stale" };
|
|
202
|
+
}
|
|
203
|
+
if (
|
|
204
|
+
providerRecoveryCount !== undefined &&
|
|
205
|
+
(!Number.isSafeInteger(providerRecoveryCount) ||
|
|
206
|
+
providerRecoveryCount <= 0 ||
|
|
207
|
+
providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 ||
|
|
208
|
+
providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
|
|
209
|
+
typeof providerFailureCode !== "string" ||
|
|
210
|
+
!/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))
|
|
211
|
+
) {
|
|
212
|
+
return { action: "stale" };
|
|
213
|
+
}
|
|
214
|
+
const recoveredClaimCode = postClaimIdentityMatches
|
|
215
|
+
? postClaimRecovery!.code
|
|
216
|
+
: input.preClaimFailure?.disposition === "retryable" &&
|
|
217
|
+
input.preClaimFailure.code !== "claim_invariant"
|
|
218
|
+
? input.preClaimFailure.code
|
|
219
|
+
: null;
|
|
220
|
+
if (recoveredClaimCode) {
|
|
221
|
+
const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
|
|
222
|
+
sessionId: input.sessionId,
|
|
223
|
+
turnId: turn.id,
|
|
224
|
+
triggerEventId: turn.triggerEventId,
|
|
225
|
+
attemptId: input.attemptId,
|
|
226
|
+
reason: providerFailureCode ?? "claimed_attempt_database_failure",
|
|
227
|
+
...(providerRecoveryCount !== undefined ? { providerRecoveryCount } : {}),
|
|
228
|
+
detail: {
|
|
229
|
+
code: providerFailureCode ?? recoveredClaimCode,
|
|
230
|
+
retryable: true,
|
|
231
|
+
...(providerRecoveryCount !== undefined
|
|
232
|
+
? {
|
|
233
|
+
databaseFailureCode: recoveredClaimCode,
|
|
234
|
+
providerRecoveryCount,
|
|
235
|
+
}
|
|
236
|
+
: {}),
|
|
237
|
+
recoverySource: "workflow_activity_failure",
|
|
238
|
+
},
|
|
239
|
+
fromStatuses: ["running"],
|
|
240
|
+
});
|
|
241
|
+
if (recovery.action !== "recovering") return { action: "stale" };
|
|
242
|
+
await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
243
|
+
await refreshQueuedTurnsGauge(
|
|
244
|
+
db,
|
|
245
|
+
observability,
|
|
246
|
+
countQueuedTurnsFn,
|
|
247
|
+
recordTurnsQueuedGaugeFn,
|
|
248
|
+
);
|
|
249
|
+
return { action: "recovering" };
|
|
250
|
+
}
|
|
183
251
|
const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
|
|
184
252
|
const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
|
|
185
253
|
sessionId: input.sessionId,
|
package/src/activities/types.ts
CHANGED
|
@@ -240,6 +240,14 @@ export type FailSessionAttemptInput = {
|
|
|
240
240
|
* admission transaction; omission is a rolling-deploy/legacy unknown and
|
|
241
241
|
* deliberately keeps the recoverable wake behavior. */
|
|
242
242
|
preClaimFailureDisposition?: PreClaimFailureDisposition;
|
|
243
|
+
/** Added in v4. Retains the safe classified DB code so an ambiguously
|
|
244
|
+
* committed claim discovered by the control lane can recover the exact
|
|
245
|
+
* attempt instead of terminally failing it. */
|
|
246
|
+
preClaimFailure?: PreClaimFailureDetail;
|
|
247
|
+
/** Added in v4. A claimed attempt that lost operational database access
|
|
248
|
+
* before turn-start completion carries its exact immutable turn identity so
|
|
249
|
+
* the DB-only control lane can recover it instead of terminally failing it. */
|
|
250
|
+
postClaimDatabaseRecovery?: PostClaimDatabaseRecoveryDetail;
|
|
243
251
|
/** The workflow admission trigger is required to re-evaluate the same
|
|
244
252
|
* durable admission branch before terminally settling a permanent failure. */
|
|
245
253
|
trigger?: RunAgentTurnInput["trigger"];
|
|
@@ -248,6 +256,7 @@ export type FailSessionAttemptInput = {
|
|
|
248
256
|
|
|
249
257
|
export type FailSessionAttemptResult =
|
|
250
258
|
| { action: "failed" }
|
|
259
|
+
| { action: "recovering" }
|
|
251
260
|
| { action: "unclaimed" }
|
|
252
261
|
| { action: "terminal" }
|
|
253
262
|
| { action: "stale" };
|
|
@@ -262,6 +271,23 @@ export type PreClaimFailureDetail = {
|
|
|
262
271
|
export const PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
|
|
263
272
|
export const PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
|
|
264
273
|
|
|
274
|
+
export type PostClaimDatabaseRecoveryDetail = {
|
|
275
|
+
turnId: string;
|
|
276
|
+
triggerEventId: string;
|
|
277
|
+
executionGeneration: number;
|
|
278
|
+
code: "db_deadlock" | "db_serialization_failure" | "db_failure";
|
|
279
|
+
/** Present when the database outage interrupted a retryable provider
|
|
280
|
+
* recovery checkpoint. The control lane must advance this exact durable
|
|
281
|
+
* count instead of reopening the same recovery generation. */
|
|
282
|
+
providerRecoveryCount?: number;
|
|
283
|
+
/** Safe classified provider cause paired with providerRecoveryCount. */
|
|
284
|
+
providerFailureCode?: string;
|
|
285
|
+
};
|
|
286
|
+
|
|
287
|
+
export const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";
|
|
288
|
+
export const POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE =
|
|
289
|
+
"Agent turn database recovery required after attempt claim.";
|
|
290
|
+
|
|
265
291
|
export type RecoverDispatchInput = {
|
|
266
292
|
accountId: string;
|
|
267
293
|
workspaceId: string;
|
|
@@ -992,6 +992,8 @@ export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
|
|
|
992
992
|
"provider_binding_missing",
|
|
993
993
|
"provider_binding_mismatch",
|
|
994
994
|
"provider_binding_adopted",
|
|
995
|
+
"quarantined_binding_missing",
|
|
996
|
+
"quarantined_binding_mismatch",
|
|
995
997
|
"defer_failed",
|
|
996
998
|
] as const;
|
|
997
999
|
|