@opengeni/worker-bundle 0.22.4-canary.2 → 0.22.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +20 -0
  2. package/dist/activities/agent-turn/governance-model.d.ts +9 -0
  3. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +14 -0
  4. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +8 -0
  5. package/dist/activities/generated-images.d.ts +3 -1
  6. package/dist/activities/image-generation-references.d.ts +39 -7
  7. package/dist/activities/sandbox-lease.d.ts +12 -0
  8. package/dist/activities/types.d.ts +24 -0
  9. package/dist/{activities-control-4XVBA3Q5.js → activities-control-VZIIV7RX.js} +77 -12
  10. package/dist/activities-control-VZIIV7RX.js.map +1 -0
  11. package/dist/{activities-turn-J3CR27E6.js → activities-turn-5RGQDBFV.js} +253 -51
  12. package/dist/activities-turn-5RGQDBFV.js.map +1 -0
  13. package/dist/{chunk-FA35XBBC.js → chunk-NAVSAZ56.js} +1 -1
  14. package/dist/chunk-NAVSAZ56.js.map +1 -0
  15. package/dist/{chunk-XWQPS6GG.js → chunk-O4HDX5SC.js} +59 -10
  16. package/dist/{chunk-XWQPS6GG.js.map → chunk-O4HDX5SC.js.map} +1 -1
  17. package/dist/index.js +2 -2
  18. package/dist/observability-metrics.d.ts +1 -1
  19. package/dist/workflow-bundle.js +62 -9
  20. package/dist/workflows/session.d.ts +4 -1
  21. package/package.json +17 -17
  22. package/src/activities/agent-turn/agent-build.ts +29 -18
  23. package/src/activities/agent-turn/claim.ts +6 -6
  24. package/src/activities/agent-turn/errors.ts +104 -21
  25. package/src/activities/agent-turn/failure-settlement.ts +40 -0
  26. package/src/activities/agent-turn/governance-model.ts +31 -4
  27. package/src/activities/agent-turn/history.ts +7 -3
  28. package/src/activities/agent-turn/image-reference-sandbox.ts +35 -0
  29. package/src/activities/agent-turn/provider-dispatch-barrier.ts +13 -0
  30. package/src/activities/agent-turn/run.ts +7 -0
  31. package/src/activities/agent-turn/stream-attempt.ts +25 -2
  32. package/src/activities/generated-images.ts +5 -2
  33. package/src/activities/image-generation-references.ts +184 -37
  34. package/src/activities/sandbox-lease.ts +48 -9
  35. package/src/activities/session-state.ts +69 -1
  36. package/src/activities/types.ts +26 -0
  37. package/src/observability-metrics.ts +2 -0
  38. package/src/workflows/session.ts +95 -21
  39. package/dist/activities-control-4XVBA3Q5.js.map +0 -1
  40. package/dist/activities-turn-J3CR27E6.js.map +0 -1
  41. package/dist/chunk-FA35XBBC.js.map +0 -1
@@ -93,6 +93,7 @@ import {
93
93
  safeErrorDiagnostic,
94
94
  compactionFailureReasonFromError,
95
95
  isCompactionSummaryFailure,
96
+ PostCompactionContinuationEmptyError,
96
97
  shouldRecoverCompactionProviderFailure,
97
98
  classifyContextWindowOverflowError,
98
99
  } from "./errors";
@@ -101,6 +102,7 @@ import {
101
102
  toolCallProducesRetainableSessionImage,
102
103
  completedToolCallFromSdkEvent,
103
104
  } from "./history";
105
+ import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
104
106
  import {
105
107
  modelUsageSourceKey,
106
108
  recordCompletedModelCallBeforeOwnershipFences,
@@ -457,10 +459,13 @@ export async function runTurnStreamAttempt(
457
459
  // the first no-response-ID fallback key and suppress a real model call.
458
460
  const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
459
461
  let workerPreparationTotalRecorded = false;
460
- const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
462
+ const runStreamAttempt = async (options: {
463
+ requireTerminalModelResponse: boolean;
464
+ }): Promise<RunAgentTurnResult> => {
461
465
  if (!runInput) {
462
466
  throw new Error("Run input was not prepared");
463
467
  }
468
+ const responseCountBeforeStream = modelResponseState.responseCount;
464
469
  eventing.stream = undefined;
465
470
  eventing.batcher = null;
466
471
  // The SDK emits every processed call item for one model response before
@@ -481,6 +486,7 @@ export async function runTurnStreamAttempt(
481
486
  resolvedModel?.provider.kind === "xai-subscription";
482
487
  let fallbackProviderRequestStartedAt: number | null = null;
483
488
  const recordFallbackProviderDispatchAtWire = async (): Promise<void> => {
489
+ await checkpointHistoryBeforeProviderDispatch(historySink);
484
490
  if (
485
491
  providerPublishesNativeRequestEvents ||
486
492
  eventing.firstModelRequestPreparationRecorded ||
@@ -1226,6 +1232,21 @@ export async function runTurnStreamAttempt(
1226
1232
  eventing.stream.completed.catch(() => undefined),
1227
1233
  cancellationSignal,
1228
1234
  );
1235
+ if (
1236
+ options.requireTerminalModelResponse &&
1237
+ eventing.stream.interruptions.length === 0 &&
1238
+ modelResponseState.responseCount === responseCountBeforeStream
1239
+ ) {
1240
+ // A fresh post-compaction stream may be returned already cancelled by
1241
+ // the SDK/runtime without yielding an event or terminal response. That
1242
+ // is not a completed logical turn: accepting finalOutput's undefined ->
1243
+ // empty-string fallback would release the queue and start newer user
1244
+ // work. Cancellation retains priority; otherwise checkpoint and recover
1245
+ // this exact turn from the durable compacted history.
1246
+ throwIfWorkerShuttingDown();
1247
+ throwIfTurnCancelled();
1248
+ throw new PostCompactionContinuationEmptyError();
1249
+ }
1229
1250
  if (!streamSawPerResponseUsage) {
1230
1251
  const aggregateUsage = eventing.stream.state.usage;
1231
1252
  const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
@@ -1529,7 +1550,9 @@ export async function runTurnStreamAttempt(
1529
1550
  let retriedAfterCompaction = false;
1530
1551
  while (true) {
1531
1552
  try {
1532
- const result = await runStreamAttempt();
1553
+ const result = await runStreamAttempt({
1554
+ requireTerminalModelResponse: retriedAfterCompaction,
1555
+ });
1533
1556
  if (retriedAfterCompaction) {
1534
1557
  observability.info("context compaction recovery succeeded after in-activity retry", {
1535
1558
  sessionId: input.sessionId,
@@ -50,7 +50,10 @@ export type RetainedGeneratedImage = {
50
50
  };
51
51
 
52
52
  export class GeneratedImageValidationError extends Error {
53
- constructor(message: string) {
53
+ constructor(
54
+ message: string,
55
+ readonly reason?: ScreenshotValidationError["reason"],
56
+ ) {
54
57
  super(message);
55
58
  this.name = "GeneratedImageValidationError";
56
59
  }
@@ -211,7 +214,7 @@ export function validateGeneratedImage(input: {
211
214
  );
212
215
  } catch (error) {
213
216
  if (error instanceof ScreenshotValidationError) {
214
- throw new GeneratedImageValidationError(error.message);
217
+ throw new GeneratedImageValidationError(error.message, error.reason);
215
218
  }
216
219
  throw error;
217
220
  }
@@ -1,7 +1,11 @@
1
1
  import { type ImageGenerationReference, type FileAsset } from "@opengeni/contracts";
2
2
  import { getGeneratedImageArtifact, requireFileForSubject, type Database } from "@opengeni/db";
3
3
  import type { ObjectStorage } from "@opengeni/storage";
4
- import { validateGeneratedImage, type GeneratedImageMediaType } from "./generated-images";
4
+ import {
5
+ GeneratedImageValidationError,
6
+ validateGeneratedImage,
7
+ type GeneratedImageMediaType,
8
+ } from "./generated-images";
5
9
 
6
10
  export const IMAGE_GENERATION_REFERENCE_MAX_BYTES = 30 * 1024 * 1024;
7
11
  export const IMAGE_GENERATION_REFERENCES_MAX_TOTAL_BYTES = 64 * 1024 * 1024;
@@ -15,12 +19,36 @@ export type ResolvedImageGenerationReference = Readonly<{
15
19
 
16
20
  export type SandboxImageReferenceReader = (path: string, maxBytes: number) => Promise<Uint8Array>;
17
21
 
18
- /**
19
- * Resolve ordered model-facing references without accepting arbitrary URLs.
20
- * Stored files remain workspace-RLS scoped; sandbox reads stay rooted beneath
21
- * /workspace. Provider adapters receive only validated immutable bytes.
22
- */
23
- export async function resolveImageGenerationReferences(input: {
22
+ export type ImageGenerationReferenceErrorCode =
23
+ | "invalid_reference"
24
+ | "unsupported_reference_media"
25
+ | "reference_too_large"
26
+ | "reference_not_ready"
27
+ | "reference_unavailable"
28
+ | "reference_integrity_mismatch";
29
+
30
+ /** A deterministic input rejection that occurs before any image provider request. */
31
+ export class ImageGenerationReferenceError extends Error {
32
+ constructor(
33
+ readonly code: ImageGenerationReferenceErrorCode,
34
+ message: string,
35
+ options?: ErrorOptions,
36
+ ) {
37
+ super(message, options);
38
+ this.name = "ImageGenerationReferenceError";
39
+ }
40
+ }
41
+
42
+ export type ImageGenerationReferenceRejectedResult = Readonly<{
43
+ isError: true;
44
+ status: "rejected";
45
+ code: ImageGenerationReferenceErrorCode;
46
+ message: string;
47
+ operationCreated: false;
48
+ content: readonly [{ readonly type: "text"; readonly text: string }];
49
+ }>;
50
+
51
+ export type ResolveImageGenerationReferencesInput = Readonly<{
24
52
  db: Database;
25
53
  objectStorage: ObjectStorage;
26
54
  accountId: string;
@@ -28,21 +56,77 @@ export async function resolveImageGenerationReferences(input: {
28
56
  subjectId: string | null;
29
57
  references: readonly ImageGenerationReference[];
30
58
  readSandboxFile?: SandboxImageReferenceReader;
31
- }): Promise<ResolvedImageGenerationReference[]> {
59
+ }>;
60
+
61
+ export type ImageGenerationReferenceResolution =
62
+ | Readonly<{
63
+ status: "resolved";
64
+ references: ResolvedImageGenerationReference[];
65
+ }>
66
+ | Readonly<{
67
+ status: "rejected";
68
+ result: ImageGenerationReferenceRejectedResult;
69
+ }>;
70
+
71
+ /**
72
+ * Carry deterministic pre-provider input failures on the tool output. Unknown
73
+ * database and object-storage failures still throw, and provider execution
74
+ * remains outside this boundary so an outcome-unknown operation can never be
75
+ * reported rejected.
76
+ */
77
+ export async function resolveImageGenerationReferencesForTool(
78
+ input: ResolveImageGenerationReferencesInput,
79
+ ): Promise<ImageGenerationReferenceResolution> {
80
+ try {
81
+ return {
82
+ status: "resolved",
83
+ references: await resolveImageGenerationReferences(input),
84
+ };
85
+ } catch (error) {
86
+ if (!(error instanceof ImageGenerationReferenceError)) throw error;
87
+ const text = `Image generation was not started: ${error.message}`;
88
+ return {
89
+ status: "rejected",
90
+ result: {
91
+ isError: true,
92
+ status: "rejected",
93
+ code: error.code,
94
+ message: error.message,
95
+ operationCreated: false,
96
+ content: [{ type: "text", text }],
97
+ },
98
+ };
99
+ }
100
+ }
101
+
102
+ /**
103
+ * Resolve ordered model-facing references without accepting arbitrary URLs.
104
+ * Stored files remain workspace-RLS scoped; sandbox reads stay rooted beneath
105
+ * /workspace. Provider adapters receive only validated immutable bytes.
106
+ */
107
+ export async function resolveImageGenerationReferences(
108
+ input: ResolveImageGenerationReferencesInput,
109
+ ): Promise<ResolvedImageGenerationReference[]> {
32
110
  const resolved: ResolvedImageGenerationReference[] = [];
33
111
  let totalBytes = 0;
34
112
 
35
113
  for (const reference of input.references) {
36
114
  const source = await referenceBytes(input, reference);
37
115
  if (source.bytes.byteLength > IMAGE_GENERATION_REFERENCE_MAX_BYTES) {
38
- throw new Error("Image reference exceeds the per-image byte limit");
116
+ throw new ImageGenerationReferenceError(
117
+ "reference_too_large",
118
+ "An image reference exceeds the per-image byte limit.",
119
+ );
39
120
  }
40
121
  totalBytes += source.bytes.byteLength;
41
122
  if (totalBytes > IMAGE_GENERATION_REFERENCES_MAX_TOTAL_BYTES) {
42
- throw new Error("Image references exceed the combined byte limit");
123
+ throw new ImageGenerationReferenceError(
124
+ "reference_too_large",
125
+ "The image references exceed the combined byte limit.",
126
+ );
43
127
  }
44
128
 
45
- const image = validateGeneratedImage({ bytes: source.bytes });
129
+ const image = validateReferenceImage(source.bytes);
46
130
  if (source.file) assertStoredFileMatches(source.file, image);
47
131
  resolved.push({
48
132
  mediaType: image.mediaType,
@@ -56,39 +140,64 @@ export async function resolveImageGenerationReferences(input: {
56
140
  }
57
141
 
58
142
  async function referenceBytes(
59
- input: {
60
- db: Database;
61
- objectStorage: ObjectStorage;
62
- accountId: string;
63
- workspaceId: string;
64
- subjectId: string | null;
65
- readSandboxFile?: SandboxImageReferenceReader;
66
- },
143
+ input: ResolveImageGenerationReferencesInput,
67
144
  reference: ImageGenerationReference,
68
145
  ): Promise<{ bytes: Uint8Array; file?: FileAsset }> {
69
146
  if (reference.kind === "sandbox_path") {
70
147
  if (!input.readSandboxFile) {
71
- throw new Error("Sandbox image references require an active sandbox");
148
+ throw new ImageGenerationReferenceError(
149
+ "reference_unavailable",
150
+ "Sandbox image references require an available workspace sandbox.",
151
+ );
152
+ }
153
+ try {
154
+ return {
155
+ bytes: await input.readSandboxFile(
156
+ reference.path,
157
+ IMAGE_GENERATION_REFERENCE_MAX_BYTES + 1,
158
+ ),
159
+ };
160
+ } catch (error) {
161
+ if (error instanceof ImageGenerationReferenceError) throw error;
162
+ throw new ImageGenerationReferenceError(
163
+ "reference_unavailable",
164
+ "The sandbox image reference could not be read. Verify that the file still exists in the current workspace.",
165
+ { cause: error },
166
+ );
72
167
  }
73
- return {
74
- bytes: await input.readSandboxFile(reference.path, IMAGE_GENERATION_REFERENCE_MAX_BYTES + 1),
75
- };
76
168
  }
77
169
 
78
- const file =
79
- reference.kind === "file"
80
- ? await requireFileForSubject(input.db, {
81
- accountId: input.accountId,
82
- workspaceId: input.workspaceId,
83
- subjectId: input.subjectId,
84
- fileId: reference.fileId,
85
- })
86
- : await artifactFile(input.db, input.workspaceId, reference.artifactId);
170
+ let file: FileAsset;
171
+ if (reference.kind === "file") {
172
+ try {
173
+ file = await requireFileForSubject(input.db, {
174
+ accountId: input.accountId,
175
+ workspaceId: input.workspaceId,
176
+ subjectId: input.subjectId,
177
+ fileId: reference.fileId,
178
+ });
179
+ } catch (error) {
180
+ if (!(error instanceof Error) || !error.message.startsWith("File not found:")) throw error;
181
+ throw new ImageGenerationReferenceError(
182
+ "reference_unavailable",
183
+ "The workspace File image reference is unavailable.",
184
+ { cause: error },
185
+ );
186
+ }
187
+ } else {
188
+ file = await artifactFile(input.db, input.workspaceId, reference.artifactId);
189
+ }
87
190
  if (file.status !== "ready" || !file.sha256) {
88
- throw new Error("Image reference file is not durably ready");
191
+ throw new ImageGenerationReferenceError(
192
+ "reference_not_ready",
193
+ "The image reference is not durably ready yet.",
194
+ );
89
195
  }
90
196
  if (file.sizeBytes <= 0 || file.sizeBytes > IMAGE_GENERATION_REFERENCE_MAX_BYTES) {
91
- throw new Error("Image reference file exceeds the supported size");
197
+ throw new ImageGenerationReferenceError(
198
+ "reference_too_large",
199
+ "The image reference file exceeds the supported size.",
200
+ );
92
201
  }
93
202
  return {
94
203
  bytes: await readStoredFile(input.objectStorage, file),
@@ -101,7 +210,12 @@ async function readStoredFile(objectStorage: ObjectStorage, file: FileAsset): Pr
101
210
  start: 0,
102
211
  end: file.sizeBytes - 1,
103
212
  });
104
- if (!bytes) throw new Error("Image reference file bytes are unavailable");
213
+ if (!bytes) {
214
+ throw new ImageGenerationReferenceError(
215
+ "reference_unavailable",
216
+ "The durable image reference bytes are unavailable.",
217
+ );
218
+ }
105
219
  return bytes;
106
220
  }
107
221
 
@@ -112,7 +226,10 @@ async function artifactFile(
112
226
  ): Promise<FileAsset> {
113
227
  const artifact = await getGeneratedImageArtifact(db, workspaceId, artifactId);
114
228
  if (!artifact || artifact.status !== "ready") {
115
- throw new Error(`Generated image artifact not found or unavailable: ${artifactId}`);
229
+ throw new ImageGenerationReferenceError(
230
+ "reference_unavailable",
231
+ "The generated-image artifact reference is unavailable.",
232
+ );
116
233
  }
117
234
  return artifact.file;
118
235
  }
@@ -122,6 +239,36 @@ function assertStoredFileMatches(
122
239
  image: Pick<ResolvedImageGenerationReference, "sizeBytes" | "sha256">,
123
240
  ): void {
124
241
  if (file.sizeBytes !== image.sizeBytes || file.sha256 !== image.sha256) {
125
- throw new Error("Image reference bytes do not match their durable file metadata");
242
+ throw new ImageGenerationReferenceError(
243
+ "reference_integrity_mismatch",
244
+ "The image reference bytes do not match their durable file metadata.",
245
+ );
246
+ }
247
+ }
248
+
249
+ function validateReferenceImage(bytes: Uint8Array) {
250
+ try {
251
+ return validateGeneratedImage({ bytes });
252
+ } catch (error) {
253
+ if (!(error instanceof GeneratedImageValidationError)) throw error;
254
+ if (error.reason === "unsupported") {
255
+ throw new ImageGenerationReferenceError(
256
+ "unsupported_reference_media",
257
+ "Image references must be PNG, JPEG, or WebP. Convert SVG or other formats before calling generate_image.",
258
+ { cause: error },
259
+ );
260
+ }
261
+ if (error.reason === "oversized") {
262
+ throw new ImageGenerationReferenceError(
263
+ "reference_too_large",
264
+ "The image reference exceeds the supported dimensions or byte size.",
265
+ { cause: error },
266
+ );
267
+ }
268
+ throw new ImageGenerationReferenceError(
269
+ "invalid_reference",
270
+ "The image reference is not a valid PNG, JPEG, or WebP image.",
271
+ { cause: error },
272
+ );
126
273
  }
127
274
  }
@@ -309,6 +309,8 @@ export type DrainableProviderProbeFn = (
309
309
  export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
310
310
  export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
311
311
  export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
312
+ export const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
313
+ export const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;
312
314
  export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
313
315
  export const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
314
316
 
@@ -1475,13 +1477,9 @@ async function deferRetainedProcessClaim(
1475
1477
  process: SandboxRetainedProcess,
1476
1478
  expected: ReturnType<typeof retainedProcessSettlementIdentity>,
1477
1479
  claimId: string,
1478
- outcome: RetainedProcessReconciliationOutcome,
1480
+ outcome: RetainedProcessDeferralOutcome,
1479
1481
  ): Promise<void> {
1480
- const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
1481
- const retryAfterMs = Math.min(
1482
- 5 * 60_000,
1483
- Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
1484
- );
1482
+ const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
1485
1483
  try {
1486
1484
  await deferRetainedProcessReconciliation(db, {
1487
1485
  accountId: process.accountId,
@@ -1490,10 +1488,10 @@ async function deferRetainedProcessClaim(
1490
1488
  processId: process.id,
1491
1489
  expected,
1492
1490
  claimId,
1493
- outcome,
1494
- retryAfterMs,
1491
+ outcome: deferral.durableOutcome,
1492
+ retryAfterMs: deferral.retryAfterMs,
1495
1493
  });
1496
- recordRetainedProcessReconciliation(observability, outcome);
1494
+ recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
1497
1495
  } catch (error) {
1498
1496
  recordRetainedProcessReconciliation(observability, "defer_failed");
1499
1497
  observability.warn("sandbox reaper: retained-process defer failed", {
@@ -1503,6 +1501,47 @@ async function deferRetainedProcessClaim(
1503
1501
  }
1504
1502
  }
1505
1503
 
1504
+ type RetainedProcessDeferralOutcome =
1505
+ | Extract<RetainedProcessProbeResult, { status: "deferred" }>["reason"]
1506
+ | "settlement_failed";
1507
+
1508
+ export function retainedProcessReconciliationDeferral(
1509
+ settings: Pick<ActivityServices["settings"], "sandboxLeaseReaperPeriodMs">,
1510
+ process: Pick<SandboxRetainedProcess, "reconcileAttempts">,
1511
+ outcome: RetainedProcessDeferralOutcome,
1512
+ ): {
1513
+ durableOutcome: string;
1514
+ metricOutcome: RetainedProcessReconciliationOutcome;
1515
+ retryAfterMs: number;
1516
+ quarantined: boolean;
1517
+ } {
1518
+ const bindingQuarantine =
1519
+ process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS &&
1520
+ (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch");
1521
+ if (bindingQuarantine) {
1522
+ return {
1523
+ durableOutcome: `quarantined_${outcome}`,
1524
+ metricOutcome:
1525
+ outcome === "provider_binding_missing"
1526
+ ? "quarantined_binding_missing"
1527
+ : "quarantined_binding_mismatch",
1528
+ retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
1529
+ quarantined: true,
1530
+ };
1531
+ }
1532
+ const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
1533
+ const retryAfterMs = Math.min(
1534
+ 5 * 60_000,
1535
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
1536
+ );
1537
+ return {
1538
+ durableOutcome: outcome,
1539
+ metricOutcome: outcome,
1540
+ retryAfterMs,
1541
+ quarantined: false,
1542
+ };
1543
+ }
1544
+
1506
1545
  type RetainedProcessProbeClient = {
1507
1546
  backendId: string;
1508
1547
  resume?: (state: unknown) => Promise<unknown>;
@@ -143,7 +143,9 @@ export function createSessionStateActivities(
143
143
  });
144
144
  if (attempt) return { action: "stale" };
145
145
 
146
- if (input.preClaimFailureDisposition === "permanent" && input.trigger) {
146
+ const preClaimFailureDisposition =
147
+ input.preClaimFailure?.disposition ?? input.preClaimFailureDisposition;
148
+ if (preClaimFailureDisposition === "permanent" && input.trigger) {
147
149
  const failed = await failSessionWorkBeforeAttemptClaimFn(db, input.workspaceId, {
148
150
  accountId: input.accountId,
149
151
  sessionId: input.sessionId,
@@ -180,6 +182,72 @@ export function createSessionStateActivities(
180
182
  });
181
183
  return { action: "unclaimed" };
182
184
  }
185
+ const postClaimRecovery = input.postClaimDatabaseRecovery;
186
+ const postClaimIdentityMatches = Boolean(
187
+ postClaimRecovery &&
188
+ turn.id === postClaimRecovery.turnId &&
189
+ turn.triggerEventId === postClaimRecovery.triggerEventId &&
190
+ turn.executionGeneration === postClaimRecovery.executionGeneration,
191
+ );
192
+ const providerRecoveryCount = postClaimIdentityMatches
193
+ ? postClaimRecovery?.providerRecoveryCount
194
+ : undefined;
195
+ const providerFailureCode = postClaimIdentityMatches
196
+ ? postClaimRecovery?.providerFailureCode
197
+ : undefined;
198
+ const hasProviderRecoveryCount = providerRecoveryCount !== undefined;
199
+ const hasProviderFailureCode = providerFailureCode !== undefined;
200
+ if (hasProviderRecoveryCount !== hasProviderFailureCode) {
201
+ return { action: "stale" };
202
+ }
203
+ if (
204
+ providerRecoveryCount !== undefined &&
205
+ (!Number.isSafeInteger(providerRecoveryCount) ||
206
+ providerRecoveryCount <= 0 ||
207
+ providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 ||
208
+ providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
209
+ typeof providerFailureCode !== "string" ||
210
+ !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))
211
+ ) {
212
+ return { action: "stale" };
213
+ }
214
+ const recoveredClaimCode = postClaimIdentityMatches
215
+ ? postClaimRecovery!.code
216
+ : input.preClaimFailure?.disposition === "retryable" &&
217
+ input.preClaimFailure.code !== "claim_invariant"
218
+ ? input.preClaimFailure.code
219
+ : null;
220
+ if (recoveredClaimCode) {
221
+ const recovery = await requestSessionTurnRecoveryFn(db, input.workspaceId, {
222
+ sessionId: input.sessionId,
223
+ turnId: turn.id,
224
+ triggerEventId: turn.triggerEventId,
225
+ attemptId: input.attemptId,
226
+ reason: providerFailureCode ?? "claimed_attempt_database_failure",
227
+ ...(providerRecoveryCount !== undefined ? { providerRecoveryCount } : {}),
228
+ detail: {
229
+ code: providerFailureCode ?? recoveredClaimCode,
230
+ retryable: true,
231
+ ...(providerRecoveryCount !== undefined
232
+ ? {
233
+ databaseFailureCode: recoveredClaimCode,
234
+ providerRecoveryCount,
235
+ }
236
+ : {}),
237
+ recoverySource: "workflow_activity_failure",
238
+ },
239
+ fromStatuses: ["running"],
240
+ });
241
+ if (recovery.action !== "recovering") return { action: "stale" };
242
+ await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, recovery.events);
243
+ await refreshQueuedTurnsGauge(
244
+ db,
245
+ observability,
246
+ countQueuedTurnsFn,
247
+ recordTurnsQueuedGaugeFn,
248
+ );
249
+ return { action: "recovering" };
250
+ }
183
251
  const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
184
252
  const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
185
253
  sessionId: input.sessionId,
@@ -240,6 +240,14 @@ export type FailSessionAttemptInput = {
240
240
  * admission transaction; omission is a rolling-deploy/legacy unknown and
241
241
  * deliberately keeps the recoverable wake behavior. */
242
242
  preClaimFailureDisposition?: PreClaimFailureDisposition;
243
+ /** Added in v4. Retains the safe classified DB code so an ambiguously
244
+ * committed claim discovered by the control lane can recover the exact
245
+ * attempt instead of terminally failing it. */
246
+ preClaimFailure?: PreClaimFailureDetail;
247
+ /** Added in v4. A claimed attempt that lost operational database access
248
+ * before turn-start completion carries its exact immutable turn identity so
249
+ * the DB-only control lane can recover it instead of terminally failing it. */
250
+ postClaimDatabaseRecovery?: PostClaimDatabaseRecoveryDetail;
243
251
  /** The workflow admission trigger is required to re-evaluate the same
244
252
  * durable admission branch before terminally settling a permanent failure. */
245
253
  trigger?: RunAgentTurnInput["trigger"];
@@ -248,6 +256,7 @@ export type FailSessionAttemptInput = {
248
256
 
249
257
  export type FailSessionAttemptResult =
250
258
  | { action: "failed" }
259
+ | { action: "recovering" }
251
260
  | { action: "unclaimed" }
252
261
  | { action: "terminal" }
253
262
  | { action: "stale" };
@@ -262,6 +271,23 @@ export type PreClaimFailureDetail = {
262
271
  export const PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
263
272
  export const PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
264
273
 
274
+ export type PostClaimDatabaseRecoveryDetail = {
275
+ turnId: string;
276
+ triggerEventId: string;
277
+ executionGeneration: number;
278
+ code: "db_deadlock" | "db_serialization_failure" | "db_failure";
279
+ /** Present when the database outage interrupted a retryable provider
280
+ * recovery checkpoint. The control lane must advance this exact durable
281
+ * count instead of reopening the same recovery generation. */
282
+ providerRecoveryCount?: number;
283
+ /** Safe classified provider cause paired with providerRecoveryCount. */
284
+ providerFailureCode?: string;
285
+ };
286
+
287
+ export const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";
288
+ export const POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE =
289
+ "Agent turn database recovery required after attempt claim.";
290
+
265
291
  export type RecoverDispatchInput = {
266
292
  accountId: string;
267
293
  workspaceId: string;
@@ -992,6 +992,8 @@ export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
992
992
  "provider_binding_missing",
993
993
  "provider_binding_mismatch",
994
994
  "provider_binding_adopted",
995
+ "quarantined_binding_missing",
996
+ "quarantined_binding_mismatch",
995
997
  "defer_failed",
996
998
  ] as const;
997
999