@opengeni/worker-bundle 0.22.4-canary.2 → 0.22.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +20 -0
  2. package/dist/activities/agent-turn/governance-model.d.ts +9 -0
  3. package/dist/activities/agent-turn/image-reference-sandbox.d.ts +14 -0
  4. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +8 -0
  5. package/dist/activities/generated-images.d.ts +3 -1
  6. package/dist/activities/image-generation-references.d.ts +39 -7
  7. package/dist/activities/sandbox-lease.d.ts +12 -0
  8. package/dist/activities/types.d.ts +24 -0
  9. package/dist/{activities-control-4XVBA3Q5.js → activities-control-VZIIV7RX.js} +77 -12
  10. package/dist/activities-control-VZIIV7RX.js.map +1 -0
  11. package/dist/{activities-turn-J3CR27E6.js → activities-turn-5RGQDBFV.js} +253 -51
  12. package/dist/activities-turn-5RGQDBFV.js.map +1 -0
  13. package/dist/{chunk-FA35XBBC.js → chunk-NAVSAZ56.js} +1 -1
  14. package/dist/chunk-NAVSAZ56.js.map +1 -0
  15. package/dist/{chunk-XWQPS6GG.js → chunk-O4HDX5SC.js} +59 -10
  16. package/dist/{chunk-XWQPS6GG.js.map → chunk-O4HDX5SC.js.map} +1 -1
  17. package/dist/index.js +2 -2
  18. package/dist/observability-metrics.d.ts +1 -1
  19. package/dist/workflow-bundle.js +62 -9
  20. package/dist/workflows/session.d.ts +4 -1
  21. package/package.json +17 -17
  22. package/src/activities/agent-turn/agent-build.ts +29 -18
  23. package/src/activities/agent-turn/claim.ts +6 -6
  24. package/src/activities/agent-turn/errors.ts +104 -21
  25. package/src/activities/agent-turn/failure-settlement.ts +40 -0
  26. package/src/activities/agent-turn/governance-model.ts +31 -4
  27. package/src/activities/agent-turn/history.ts +7 -3
  28. package/src/activities/agent-turn/image-reference-sandbox.ts +35 -0
  29. package/src/activities/agent-turn/provider-dispatch-barrier.ts +13 -0
  30. package/src/activities/agent-turn/run.ts +7 -0
  31. package/src/activities/agent-turn/stream-attempt.ts +25 -2
  32. package/src/activities/generated-images.ts +5 -2
  33. package/src/activities/image-generation-references.ts +184 -37
  34. package/src/activities/sandbox-lease.ts +48 -9
  35. package/src/activities/session-state.ts +69 -1
  36. package/src/activities/types.ts +26 -0
  37. package/src/observability-metrics.ts +2 -0
  38. package/src/workflows/session.ts +95 -21
  39. package/dist/activities-control-4XVBA3Q5.js.map +0 -1
  40. package/dist/activities-turn-J3CR27E6.js.map +0 -1
  41. package/dist/chunk-FA35XBBC.js.map +0 -1
@@ -1,5 +1,5 @@
1
1
  import type * as activities from "../activities.js";
2
- import { type EscapedMcpTimeoutRecoveryDetail, type PreClaimFailureDisposition } from "../activities/types.js";
2
+ import { type EscapedMcpTimeoutRecoveryDetail, type PostClaimDatabaseRecoveryDetail, type PreClaimFailureDetail, type PreClaimFailureDisposition } from "../activities/types.js";
3
3
  /**
4
4
  * How long the continuation loop must hold before re-admitting the next turn. 0 ⇒ no
5
5
  * hold (re-dispatch immediately — a rotation candidate is ready, or no idle delay was
@@ -46,7 +46,10 @@ export declare function humanInputDeadlineWaitMs(expiresAt: string, nowMs?: numb
46
46
  */
47
47
  export declare function escapedMcpTimeoutRecoveryDetail(error: unknown): EscapedMcpTimeoutRecoveryDetail | null;
48
48
  /** Read only the upgraded turn worker's explicit pre-claim wire contract. */
49
+ export declare function preClaimFailureDetail(error: unknown): PreClaimFailureDetail | undefined;
49
50
  export declare function preClaimFailureDisposition(error: unknown): PreClaimFailureDisposition | undefined;
51
+ /** Read only the upgraded turn worker's exact post-claim DB recovery wire. */
52
+ export declare function postClaimDatabaseRecoveryDetail(error: unknown): PostClaimDatabaseRecoveryDetail | null;
50
53
  /**
51
54
  * Classify only the exact turn-fence cancellation protocol used to arbitrate a
52
55
  * database control commit that reaches the activity just before its Temporal
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.22.4-canary.2",
3
+ "version": "0.22.8",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -54,22 +54,22 @@
54
54
  },
55
55
  "dependencies": {
56
56
  "@llamaindex/liteparse": "^1.5.3",
57
- "@opengeni/agent-proto": "^0.5.1-canary.4",
58
- "@opengeni/capabilities": "^0.3.1-canary.3",
59
- "@opengeni/codemode": "^0.4.15-canary.2",
60
- "@opengeni/codex": "^0.2.19-canary.3",
61
- "@opengeni/config": "^0.20.1-canary.2",
62
- "@opengeni/contracts": "^2.5.0-canary.2",
63
- "@opengeni/core": "^2.4.0-canary.2",
64
- "@opengeni/db": "^3.4.0-canary.2",
65
- "@opengeni/documents": "^0.8.2-canary.2",
66
- "@opengeni/events": "^0.4.0-canary.2",
67
- "@opengeni/github": "^0.5.5-canary.2",
68
- "@opengeni/network": "^0.2.3-canary.3",
69
- "@opengeni/observability": "^0.8.6-canary.2",
70
- "@opengeni/runtime": "^1.4.0-canary.2",
71
- "@opengeni/storage": "^0.2.108-canary.2",
72
- "@opengeni/xai-subscription": "^0.1.2-canary.3",
57
+ "@opengeni/agent-proto": "^0.5.1",
58
+ "@opengeni/capabilities": "^0.3.1",
59
+ "@opengeni/codemode": "^0.4.17",
60
+ "@opengeni/codex": "^0.2.19",
61
+ "@opengeni/config": "^0.22.0",
62
+ "@opengeni/contracts": "^2.7.0",
63
+ "@opengeni/core": "^2.5.3",
64
+ "@opengeni/db": "^3.6.0",
65
+ "@opengeni/documents": "^0.8.6",
66
+ "@opengeni/events": "^0.4.4",
67
+ "@opengeni/github": "^0.6.1",
68
+ "@opengeni/network": "^0.2.3",
69
+ "@opengeni/observability": "^0.8.9",
70
+ "@opengeni/runtime": "^1.4.2",
71
+ "@opengeni/storage": "^0.2.110",
72
+ "@opengeni/xai-subscription": "^0.1.2",
73
73
  "@temporalio/activity": "^1.17.0",
74
74
  "@temporalio/client": "^1.17.0",
75
75
  "@temporalio/worker": "^1.17.0",
@@ -45,8 +45,11 @@ import { createTurnCredentialLeases } from "./credential-leases";
45
45
  import { createTurnMediaArtifacts } from "./media-artifacts";
46
46
  import { executeGatewayImageGeneration } from "../gateway-image-generation";
47
47
  import { executeCodexImageGeneration } from "../codex-image-generation";
48
- import { resolveImageGenerationReferences } from "../image-generation-references";
49
- import { SandboxChannelAService, type ChannelASession } from "@opengeni/runtime/sandbox";
48
+ import {
49
+ ImageGenerationReferenceError,
50
+ resolveImageGenerationReferencesForTool,
51
+ } from "../image-generation-references";
52
+ import { SandboxChannelAService } from "@opengeni/runtime/sandbox";
50
53
  import { sandboxRunAs } from "@opengeni/runtime";
51
54
  import { VideoGenerationRejectedResult } from "@opengeni/contracts";
52
55
 
@@ -74,6 +77,7 @@ import type {
74
77
  SandboxRuntimeState,
75
78
  } from "./turn-context";
76
79
  import { SESSION_TITLE_MODEL_TOOL_NAME } from "./session-title";
80
+ import { resolveImageReferenceSandboxSession } from "./image-reference-sandbox";
77
81
 
78
82
  export type BuildTurnAgentDeps = {
79
83
  input: RunAgentTurnInput;
@@ -240,9 +244,9 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
240
244
  : {};
241
245
  const hostedWebSearch = hostedWebSearchForTurn(resolvedModel, runSettings.webSearchEnabled);
242
246
  const resolveImageReferences = async (
243
- references: Parameters<typeof resolveImageGenerationReferences>[0]["references"],
247
+ references: Parameters<typeof resolveImageGenerationReferencesForTool>[0]["references"],
244
248
  ) =>
245
- await resolveImageGenerationReferences({
249
+ await resolveImageGenerationReferencesForTool({
246
250
  db,
247
251
  objectStorage: objectStorage!,
248
252
  accountId: input.accountId,
@@ -250,17 +254,16 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
250
254
  subjectId: fileAuthoritySubjectId,
251
255
  references,
252
256
  readSandboxFile: async (path, maxBytes) => {
253
- const imageReferenceSession = (sandboxState.setupBoxSession ??
254
- media.sdkOwnedSandboxSession) as ChannelASession | null;
255
- if (!imageReferenceSession) {
256
- throw new Error("Sandbox image reference is unavailable");
257
- }
257
+ const imageReferenceSandbox = await resolveImageReferenceSandboxSession(
258
+ sandboxState,
259
+ media.sdkOwnedSandboxSession,
260
+ );
258
261
  const relativePath = path.slice("/workspace/".length);
259
262
  const referenceRunAs = sandboxRunAs(eventing.modelRunSettings);
260
263
  const channel = new SandboxChannelAService({
261
- session: imageReferenceSession,
264
+ session: imageReferenceSandbox.session,
262
265
  workspaceRoot: "/workspace",
263
- leaseEpoch: sandboxState.resolvedSandbox?.leaseEpoch ?? 0,
266
+ leaseEpoch: imageReferenceSandbox.leaseEpoch,
264
267
  ...(referenceRunAs ? { runAs: referenceRunAs } : {}),
265
268
  });
266
269
  const read = await channel.fsRead({
@@ -268,7 +271,12 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
268
271
  encoding: "base64",
269
272
  maxBytes,
270
273
  });
271
- if (read.truncated) throw new Error("Sandbox image reference exceeds the byte limit");
274
+ if (read.truncated) {
275
+ throw new ImageGenerationReferenceError(
276
+ "reference_too_large",
277
+ "The sandbox image reference exceeds the per-image byte limit.",
278
+ );
279
+ }
272
280
  return Uint8Array.from(Buffer.from(read.content, "base64"));
273
281
  },
274
282
  });
@@ -297,7 +305,8 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
297
305
  imageGeneration: {
298
306
  kind: "provider_adapter",
299
307
  execute: async ({ prompt, references }, { toolCallId }) => {
300
- const resolvedReferences = await resolveImageReferences(references);
308
+ const referenceResolution = await resolveImageReferences(references);
309
+ if (referenceResolution.status === "rejected") return referenceResolution.result;
301
310
  const receipt = await executeCodexImageGeneration({
302
311
  db,
303
312
  objectStorage,
@@ -308,7 +317,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
308
317
  attemptId: input.attemptId,
309
318
  toolCallId,
310
319
  prompt,
311
- references: resolvedReferences,
320
+ references: referenceResolution.references,
312
321
  credentialId: imageAuthority.credentialId,
313
322
  codexContext: imageAuthority.credentialContext,
314
323
  ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
@@ -331,7 +340,8 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
331
340
  imageGeneration: {
332
341
  kind: "provider_adapter",
333
342
  execute: async ({ prompt, references }, { toolCallId }) => {
334
- const resolvedReferences = await resolveImageReferences(references);
343
+ const referenceResolution = await resolveImageReferences(references);
344
+ if (referenceResolution.status === "rejected") return referenceResolution.result;
335
345
  const receipt = await executeXaiSubscriptionImageGeneration({
336
346
  db,
337
347
  objectStorage,
@@ -342,7 +352,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
342
352
  attemptId: input.attemptId,
343
353
  toolCallId,
344
354
  prompt,
345
- references: resolvedReferences,
355
+ references: referenceResolution.references,
346
356
  credentialId: imageAuthority.credentialId,
347
357
  xaiContext: imageAuthority.credentialContext,
348
358
  ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
@@ -366,7 +376,8 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
366
376
  imageGeneration: {
367
377
  kind: "provider_adapter",
368
378
  execute: async ({ prompt, references }, { toolCallId }) => {
369
- const resolvedReferences = await resolveImageReferences(references);
379
+ const referenceResolution = await resolveImageReferences(references);
380
+ if (referenceResolution.status === "rejected") return referenceResolution.result;
370
381
  const receipt = await executeGatewayImageGeneration({
371
382
  db,
372
383
  objectStorage,
@@ -378,7 +389,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
378
389
  apiKey: gatewayApiKey,
379
390
  modelId: capabilitySettings.imageGenerationModel,
380
391
  prompt,
381
- references: resolvedReferences,
392
+ references: referenceResolution.references,
382
393
  toolCallId,
383
394
  ...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
384
395
  });
@@ -185,13 +185,17 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
185
185
  }
186
186
  const turn = claim.turn;
187
187
  attempt.turnId = turn.id;
188
+ attempt.executionGeneration = turn.executionGeneration;
189
+ attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
190
+ attempt.triggerEventId = turn.triggerEventId;
191
+ attempt.redispatchesAtDispatch = Number(
192
+ (turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ?? 0,
193
+ );
188
194
  // Establish durable attempt ownership before any later read can fail.
189
195
  // Therefore every failure with no turnId came from the one atomic claim
190
196
  // transaction and can be classified without conflating ordinary runtime
191
197
  // or transport failures with admission failures.
192
198
  const session = await requireSession(db, input.workspaceId, input.sessionId);
193
- attempt.executionGeneration = turn.executionGeneration;
194
- attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
195
199
  let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
196
200
  const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
197
201
  const fileAuthoritySubjectId = turn.initiatingHumanSubjectId ?? null;
@@ -268,7 +272,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
268
272
  billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
269
273
  billingState.isCodexTurn = billingIdentity.codexSubscription;
270
274
  billingState.isXaiTurn = billingIdentity.xaiSubscription;
271
- attempt.triggerEventId = turn.triggerEventId;
272
275
  const trigger = await getSessionEvent(db, input.workspaceId, attempt.triggerEventId);
273
276
  if (!trigger) {
274
277
  throw new Error(`Trigger event not found: ${attempt.triggerEventId}`);
@@ -286,9 +289,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
286
289
  trigger,
287
290
  );
288
291
  attempt.triggerType = trigger.type;
289
- attempt.redispatchesAtDispatch = Number(
290
- (turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ?? 0,
291
- );
292
292
  turnLifecycleMetricsFor(observability).start(attempt.turnId);
293
293
  // §7.5 P3 — pass the accepted billing attribution (externally funded turns
294
294
  // bypass OpenGeni credit/token gates)
@@ -35,10 +35,16 @@ import {
35
35
  isXaiSubscriptionTransportError,
36
36
  XaiSubscriptionReloginRequired,
37
37
  } from "@opengeni/xai-subscription";
38
- import type { EscapedMcpTimeoutRecoveryDetail, PreClaimFailureDetail } from "../types";
38
+ import type {
39
+ EscapedMcpTimeoutRecoveryDetail,
40
+ PostClaimDatabaseRecoveryDetail,
41
+ PreClaimFailureDetail,
42
+ } from "../types";
39
43
  import {
40
44
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
41
45
  ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
46
+ POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
47
+ POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
42
48
  PRE_CLAIM_FAILURE_MESSAGE,
43
49
  PRE_CLAIM_FAILURE_TYPE,
44
50
  } from "../types";
@@ -52,6 +58,16 @@ import {
52
58
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
53
59
  export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
54
60
  export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
61
+ export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
62
+
63
+ export class PostCompactionContinuationEmptyError extends Error {
64
+ readonly code = POST_COMPACTION_CONTINUATION_EMPTY_CODE;
65
+
66
+ constructor() {
67
+ super("Post-compaction continuation stream ended before a terminal model response");
68
+ this.name = "PostCompactionContinuationEmptyError";
69
+ }
70
+ }
55
71
 
56
72
  export type ProviderRecoveryResult =
57
73
  | {
@@ -89,7 +105,8 @@ export function providerRecoveryResult(input: {
89
105
  : input.failureCode === "provider_unavailable" ||
90
106
  input.failureCode === "upstream_connectivity_unavailable" ||
91
107
  input.failureCode === "mcp_transport_timeout" ||
92
- input.failureCode === "mcp_transport_unavailable"
108
+ input.failureCode === "mcp_transport_unavailable" ||
109
+ input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
93
110
  ? Math.max(
94
111
  providerDelay ?? 0,
95
112
  PROVIDER_CONNECTIVITY_BACKOFF_MS[
@@ -246,32 +263,16 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
246
263
  */
247
264
  export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
248
265
  const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
249
- const sqlState = persistenceFailure?.details.sqlState ?? null;
250
- const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
266
+ const retryableCode = retryableDatabaseFailureCode(error);
251
267
  // The database transaction itself retries only the two contention failures
252
268
  // proven safe for immediate replay. Once the activity has failed, the
253
269
  // workflow may also retry operational outages after a durable re-read and
254
270
  // bounded delay. Unknown driver/database failures stay recoverable because
255
271
  // they commonly represent a lost connection; known constraint, auth, and
256
272
  // application SQLSTATEs are permanent and must not create an infinite loop.
257
- const retryable = Boolean(
258
- rawTransportFailure ||
259
- (persistenceFailure &&
260
- (sqlState === null ||
261
- sqlState.startsWith("08") ||
262
- sqlState.startsWith("40") ||
263
- sqlState.startsWith("53") ||
264
- sqlState === "55P03" ||
265
- sqlState === "57014" ||
266
- sqlState === "57P01" ||
267
- sqlState === "57P02" ||
268
- sqlState === "57P03" ||
269
- sqlState.startsWith("58"))),
270
- );
271
273
  const detail: PreClaimFailureDetail = {
272
- disposition: retryable ? "retryable" : "permanent",
273
- code:
274
- persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
274
+ disposition: retryableCode ? "retryable" : "permanent",
275
+ code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
275
276
  };
276
277
  return ApplicationFailure.create({
277
278
  message: PRE_CLAIM_FAILURE_MESSAGE,
@@ -281,6 +282,80 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
281
282
  });
282
283
  }
283
284
 
285
+ function retryableDatabaseFailureCode(
286
+ error: unknown,
287
+ ): PostClaimDatabaseRecoveryDetail["code"] | null {
288
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
289
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
290
+ if (sqlState === null && isRetryableDatabaseTransportFailure(error)) {
291
+ return "db_failure";
292
+ }
293
+ if (
294
+ !persistenceFailure ||
295
+ !(
296
+ sqlState === null ||
297
+ sqlState.startsWith("08") ||
298
+ sqlState.startsWith("40") ||
299
+ sqlState.startsWith("53") ||
300
+ sqlState === "55P03" ||
301
+ sqlState === "57014" ||
302
+ sqlState === "57P01" ||
303
+ sqlState === "57P02" ||
304
+ sqlState === "57P03" ||
305
+ sqlState.startsWith("58")
306
+ )
307
+ ) {
308
+ return null;
309
+ }
310
+ return persistenceFailure.details.code;
311
+ }
312
+
313
+ /**
314
+ * Carry one exact claimed-but-not-started attempt into the workflow's DB-only
315
+ * recovery lane. Permanent database/state failures remain terminal; only the
316
+ * same operational outage classes that are safe before claim are admitted.
317
+ */
318
+ export function postClaimDatabaseRecoveryFailure(input: {
319
+ error: unknown;
320
+ turnId: string;
321
+ triggerEventId: string;
322
+ executionGeneration: number;
323
+ providerRecovery?: {
324
+ failureCode: string;
325
+ providerRecoveryCount: number;
326
+ };
327
+ }): ApplicationFailure | null {
328
+ const code = retryableDatabaseFailureCode(input.error);
329
+ if (!code || input.executionGeneration < 1) return null;
330
+ if (
331
+ input.providerRecovery &&
332
+ (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
333
+ input.providerRecovery.providerRecoveryCount <= 0 ||
334
+ input.providerRecovery.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
335
+ !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode))
336
+ ) {
337
+ return null;
338
+ }
339
+ const detail: PostClaimDatabaseRecoveryDetail = {
340
+ turnId: input.turnId,
341
+ triggerEventId: input.triggerEventId,
342
+ executionGeneration: input.executionGeneration,
343
+ code,
344
+ ...(input.providerRecovery
345
+ ? {
346
+ providerFailureCode: input.providerRecovery.failureCode,
347
+ providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
348
+ }
349
+ : {}),
350
+ };
351
+ return ApplicationFailure.create({
352
+ message: POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
353
+ type: POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
354
+ nonRetryable: true,
355
+ details: [detail],
356
+ });
357
+ }
358
+
284
359
  /**
285
360
  * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
286
361
  * workspace-active. No rotation in P1. The selected id must still be in the
@@ -729,6 +804,14 @@ export function agentRunFailurePayload(
729
804
  detail: error.message,
730
805
  };
731
806
  }
807
+ if (error instanceof PostCompactionContinuationEmptyError) {
808
+ return {
809
+ error:
810
+ "Context compaction completed, but the continuation ended before a new model response. The same turn will retry from the compacted checkpoint.",
811
+ code: POST_COMPACTION_CONTINUATION_EMPTY_CODE,
812
+ retryable: true,
813
+ };
814
+ }
732
815
  // An accepted Codex stream with no terminal response is malformed/partial,
733
816
  // not provider backpressure. Replaying the same accepted turn could repeat
734
817
  // model or tool effects, so this marked transport failure must outrank the
@@ -55,6 +55,7 @@ import { createTurnHistorySink } from "./history-sink";
55
55
  import { BudgetExhaustedError } from "./admission";
56
56
  import {
57
57
  providerRecoveryExhaustedFailure,
58
+ postClaimDatabaseRecoveryFailure,
58
59
  providerRecoveryResult,
59
60
  providerRetryAfterMs,
60
61
  escapedMcpTimeoutRecoveryFailure,
@@ -1387,9 +1388,48 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
1387
1388
  control.activityError = error;
1388
1389
  throw escaped;
1389
1390
  }
1391
+ const postClaimRecovery =
1392
+ recoveryResult.status === "recovering"
1393
+ ? postClaimDatabaseRecoveryFailure({
1394
+ error: recoveryError,
1395
+ turnId: attempt.turnId,
1396
+ triggerEventId: attempt.triggerEventId!,
1397
+ executionGeneration: attempt.executionGeneration,
1398
+ providerRecovery: {
1399
+ failureCode: failure.code ?? "provider_unavailable",
1400
+ providerRecoveryCount: nextProviderRecoveryCount,
1401
+ },
1402
+ })
1403
+ : null;
1404
+ if (postClaimRecovery) {
1405
+ control.activityStatus = "recovering";
1406
+ control.turnMetricOutcome = "recovering";
1407
+ control.activityError = error;
1408
+ throw postClaimRecovery;
1409
+ }
1390
1410
  throw recoveryError;
1391
1411
  }
1392
1412
  }
1413
+ if (
1414
+ attempt.turnId &&
1415
+ attempt.triggerEventId &&
1416
+ attempt.executionGeneration > 0 &&
1417
+ !eventing.turnStartedPublished &&
1418
+ !attempt.modelRequestStarted
1419
+ ) {
1420
+ const recoveryFailure = postClaimDatabaseRecoveryFailure({
1421
+ error,
1422
+ turnId: attempt.turnId,
1423
+ triggerEventId: attempt.triggerEventId,
1424
+ executionGeneration: attempt.executionGeneration,
1425
+ });
1426
+ if (recoveryFailure) {
1427
+ control.activityStatus = "recovering";
1428
+ control.turnMetricOutcome = "recovering";
1429
+ control.activityError = error;
1430
+ throw recoveryFailure;
1431
+ }
1432
+ }
1393
1433
  control.activityStatus = "failed";
1394
1434
  control.activityError = error;
1395
1435
  if (!attempt.turnId) {
@@ -128,6 +128,33 @@ export type GovernanceModelOk = {
128
128
 
129
129
  export type GovernanceModelOutcome = { exit: RunAgentTurnResult } | { ok: GovernanceModelOk };
130
130
 
131
+ type PreferenceSnapshotTurn = Pick<ClaimTurnOk["turn"], "initiatingHumanSubjectId" | "initiator">;
132
+
133
+ export function preferenceSnapshotHumanSubjectId(turn: PreferenceSnapshotTurn): string | null {
134
+ return (
135
+ turn.initiatingHumanSubjectId ??
136
+ (turn.initiator.kind === "subject" ? turn.initiator.subjectId : null)
137
+ );
138
+ }
139
+
140
+ /**
141
+ * Preferences are scoped partly to the frozen initiating human. A service-only
142
+ * turn has no such scope, so avoid calling the human-bound snapshot capability
143
+ * merely to catch its expected authority rejection.
144
+ */
145
+ export async function preferenceSnapshotForTurn<T>(
146
+ turn: PreferenceSnapshotTurn,
147
+ load: () => Promise<T>,
148
+ ): Promise<T | null> {
149
+ if (!preferenceSnapshotHumanSubjectId(turn)) return null;
150
+ try {
151
+ return await load();
152
+ } catch (error) {
153
+ if (error instanceof PreferenceRegistryInitiatorError) return null;
154
+ throw error;
155
+ }
156
+ }
157
+
131
158
  export async function prepareGovernanceAndModel(
132
159
  deps: GovernanceModelDeps,
133
160
  ): Promise<GovernanceModelOutcome> {
@@ -197,10 +224,10 @@ export async function prepareGovernanceAndModel(
197
224
  getWorkspace(db, input.workspaceId),
198
225
  getOrCreateCompanyProfileSnapshot(db, governanceClaims),
199
226
  getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
200
- getOrCreatePreferenceRegistrySnapshot(db, governanceClaims).catch((error) => {
201
- if (error instanceof PreferenceRegistryInitiatorError) return null;
202
- throw error;
203
- }),
227
+ preferenceSnapshotForTurn(
228
+ turn,
229
+ async () => await getOrCreatePreferenceRegistrySnapshot(db, governanceClaims),
230
+ ),
204
231
  ]),
205
232
  getWorkspaceModelPolicy(db, input.workspaceId),
206
233
  ]);
@@ -1,4 +1,8 @@
1
- import { normalizeProtocolJsonValue, sanitizeHistoryItemsForModel } from "@opengeni/runtime";
1
+ import {
2
+ normalizeProtocolJsonValue,
3
+ sanitizeHistoryItemsForModel,
4
+ toolCallIdFromSdkItem,
5
+ } from "@opengeni/runtime";
2
6
  import { opaqueProviderArtifactFingerprint } from "@opengeni/codex";
3
7
 
4
8
  export function historyRowsToAppend(
@@ -149,7 +153,7 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
149
153
  // never receives a separate function result and therefore must not enter the
150
154
  // pending function-call ledger.
151
155
  if (raw.type === "hosted_tool_call" && raw.name === "image_generation_call") return null;
152
- const callId = raw.callId ?? raw.call_id ?? raw.id;
156
+ const callId = toolCallIdFromSdkItem(raw) ?? raw.id;
153
157
  const callType = raw.type;
154
158
  if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
155
159
  return null;
@@ -182,7 +186,7 @@ export function completedToolCallFromSdkEvent(event: unknown): {
182
186
  item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
183
187
  ? (item.rawItem as Record<string, unknown>)
184
188
  : {};
185
- const callId = raw.callId ?? raw.call_id ?? item.id;
189
+ const callId = toolCallIdFromSdkItem(raw) ?? item.id;
186
190
  if (typeof callId !== "string" || callId.length === 0) return null;
187
191
  return {
188
192
  callId,
@@ -0,0 +1,35 @@
1
+ import type { ChannelASession } from "@opengeni/runtime/sandbox";
2
+ import type { SandboxRuntimeState } from "./turn-context";
3
+
4
+ type ImageReferenceSandboxState = Pick<
5
+ SandboxRuntimeState,
6
+ "resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner"
7
+ >;
8
+
9
+ /**
10
+ * Join the turn's canonical lazy sandbox boundary before reading a
11
+ * model-supplied /workspace image reference. Durable File and artifact
12
+ * references never call this helper because their bytes come from object
13
+ * storage instead.
14
+ */
15
+ export async function resolveImageReferenceSandboxSession(
16
+ sandboxState: ImageReferenceSandboxState,
17
+ sdkOwnedSandboxSession: unknown,
18
+ ): Promise<{ session: ChannelASession; leaseEpoch: number }> {
19
+ let sandbox = sandboxState.resolvedSandbox;
20
+ if (!sandbox && sandboxState.turnSandboxProvisioner) {
21
+ sandbox = await sandboxState.turnSandboxProvisioner.get();
22
+ }
23
+
24
+ const session = (sandboxState.lazyOwnedSandbox?.session ??
25
+ sandbox?.established.session ??
26
+ sdkOwnedSandboxSession) as ChannelASession | null;
27
+ if (!session) {
28
+ throw new Error("Sandbox image reference is unavailable");
29
+ }
30
+
31
+ return {
32
+ session,
33
+ leaseEpoch: sandbox?.leaseEpoch ?? 0,
34
+ };
35
+ }
@@ -0,0 +1,13 @@
1
+ import type { TurnHistorySink } from "./history-sink";
2
+
3
+ /**
4
+ * Persist the SDK's complete prior model/tool history before another provider
5
+ * request can start. The first request has no prior model/tool rows to append.
6
+ * Follow-up requests run only after the preceding complete call/result batch is
7
+ * replay-safe.
8
+ */
9
+ export async function checkpointHistoryBeforeProviderDispatch(
10
+ historySink: Pick<TurnHistorySink, "reconcileConversationTruth">,
11
+ ): Promise<void> {
12
+ await historySink.reconcileConversationTruth({ requireDurable: true });
13
+ }
@@ -44,6 +44,7 @@ import { ToolResultSpill } from "./tool-result-spill";
44
44
  import { createTurnCredentialLeases } from "./credential-leases";
45
45
  import { createTurnMediaArtifacts } from "./media-artifacts";
46
46
  import { createTurnHistorySink } from "./history-sink";
47
+ import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
47
48
  import { sandboxRunAs } from "@opengeni/runtime";
48
49
  import { randomUUID } from "node:crypto";
49
50
  import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
@@ -562,6 +563,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
562
563
  if (!eventing.publish || !attempt.turnId) {
563
564
  throw new Error("Codex model request started before the turn event producer");
564
565
  }
566
+ if (event.phase === "started") {
567
+ await checkpointHistoryBeforeProviderDispatch(historySink);
568
+ }
565
569
  const shouldRecordStartedAudit =
566
570
  event.phase === "started" && !firstModelRequestAuditRecorded;
567
571
  if (shouldRecordStartedAudit) {
@@ -707,6 +711,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
707
711
  if (!eventing.publish || !attempt.turnId) {
708
712
  throw new Error("SuperGrok model request started before the turn event producer");
709
713
  }
714
+ if (event.phase === "started") {
715
+ await checkpointHistoryBeforeProviderDispatch(historySink);
716
+ }
710
717
  const shouldRecordStartedAudit =
711
718
  event.phase === "started" && !firstModelRequestAuditRecorded;
712
719
  if (shouldRecordStartedAudit) firstModelRequestAuditRecorded = true;