@opengeni/worker-bundle 0.22.4-canary.2 → 0.22.7-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +20 -0
- package/dist/activities/agent-turn/governance-model.d.ts +9 -0
- package/dist/activities/agent-turn/image-reference-sandbox.d.ts +14 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +8 -0
- package/dist/activities/generated-images.d.ts +3 -1
- package/dist/activities/image-generation-references.d.ts +39 -7
- package/dist/activities/sandbox-lease.d.ts +12 -0
- package/dist/activities/types.d.ts +24 -0
- package/dist/{activities-control-4XVBA3Q5.js → activities-control-VZIIV7RX.js} +77 -12
- package/dist/activities-control-VZIIV7RX.js.map +1 -0
- package/dist/{activities-turn-J3CR27E6.js → activities-turn-5RGQDBFV.js} +253 -51
- package/dist/activities-turn-5RGQDBFV.js.map +1 -0
- package/dist/{chunk-FA35XBBC.js → chunk-NAVSAZ56.js} +1 -1
- package/dist/chunk-NAVSAZ56.js.map +1 -0
- package/dist/{chunk-XWQPS6GG.js → chunk-O4HDX5SC.js} +59 -10
- package/dist/{chunk-XWQPS6GG.js.map → chunk-O4HDX5SC.js.map} +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/workflow-bundle.js +62 -9
- package/dist/workflows/session.d.ts +4 -1
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +29 -18
- package/src/activities/agent-turn/claim.ts +6 -6
- package/src/activities/agent-turn/errors.ts +104 -21
- package/src/activities/agent-turn/failure-settlement.ts +40 -0
- package/src/activities/agent-turn/governance-model.ts +31 -4
- package/src/activities/agent-turn/history.ts +7 -3
- package/src/activities/agent-turn/image-reference-sandbox.ts +35 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +13 -0
- package/src/activities/agent-turn/run.ts +7 -0
- package/src/activities/agent-turn/stream-attempt.ts +25 -2
- package/src/activities/generated-images.ts +5 -2
- package/src/activities/image-generation-references.ts +184 -37
- package/src/activities/sandbox-lease.ts +48 -9
- package/src/activities/session-state.ts +69 -1
- package/src/activities/types.ts +26 -0
- package/src/observability-metrics.ts +2 -0
- package/src/workflows/session.ts +95 -21
- package/dist/activities-control-4XVBA3Q5.js.map +0 -1
- package/dist/activities-turn-J3CR27E6.js.map +0 -1
- package/dist/chunk-FA35XBBC.js.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type * as activities from "../activities.js";
|
|
2
|
-
import { type EscapedMcpTimeoutRecoveryDetail, type PreClaimFailureDisposition } from "../activities/types.js";
|
|
2
|
+
import { type EscapedMcpTimeoutRecoveryDetail, type PostClaimDatabaseRecoveryDetail, type PreClaimFailureDetail, type PreClaimFailureDisposition } from "../activities/types.js";
|
|
3
3
|
/**
|
|
4
4
|
* How long the continuation loop must hold before re-admitting the next turn. 0 ⇒ no
|
|
5
5
|
* hold (re-dispatch immediately — a rotation candidate is ready, or no idle delay was
|
|
@@ -46,7 +46,10 @@ export declare function humanInputDeadlineWaitMs(expiresAt: string, nowMs?: numb
|
|
|
46
46
|
*/
|
|
47
47
|
export declare function escapedMcpTimeoutRecoveryDetail(error: unknown): EscapedMcpTimeoutRecoveryDetail | null;
|
|
48
48
|
/** Read only the upgraded turn worker's explicit pre-claim wire contract. */
|
|
49
|
+
export declare function preClaimFailureDetail(error: unknown): PreClaimFailureDetail | undefined;
|
|
49
50
|
export declare function preClaimFailureDisposition(error: unknown): PreClaimFailureDisposition | undefined;
|
|
51
|
+
/** Read only the upgraded turn worker's exact post-claim DB recovery wire. */
|
|
52
|
+
export declare function postClaimDatabaseRecoveryDetail(error: unknown): PostClaimDatabaseRecoveryDetail | null;
|
|
50
53
|
/**
|
|
51
54
|
* Classify only the exact turn-fence cancellation protocol used to arbitrate a
|
|
52
55
|
* database control commit that reaches the activity just before its Temporal
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.22.
|
|
3
|
+
"version": "0.22.7-canary.0",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -54,22 +54,22 @@
|
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@llamaindex/liteparse": "^1.5.3",
|
|
57
|
-
"@opengeni/agent-proto": "^0.5.1-canary.
|
|
58
|
-
"@opengeni/capabilities": "^0.3.1-canary.
|
|
59
|
-
"@opengeni/codemode": "^0.4.
|
|
60
|
-
"@opengeni/codex": "^0.2.19-canary.
|
|
61
|
-
"@opengeni/config": "^0.
|
|
62
|
-
"@opengeni/contracts": "^2.
|
|
63
|
-
"@opengeni/core": "^2.
|
|
64
|
-
"@opengeni/db": "^3.
|
|
65
|
-
"@opengeni/documents": "^0.8.
|
|
66
|
-
"@opengeni/events": "^0.4.
|
|
67
|
-
"@opengeni/github": "^0.
|
|
68
|
-
"@opengeni/network": "^0.2.3-canary.
|
|
69
|
-
"@opengeni/observability": "^0.8.
|
|
70
|
-
"@opengeni/runtime": "^1.4.
|
|
71
|
-
"@opengeni/storage": "^0.2.
|
|
72
|
-
"@opengeni/xai-subscription": "^0.1.2-canary.
|
|
57
|
+
"@opengeni/agent-proto": "^0.5.1-canary.5",
|
|
58
|
+
"@opengeni/capabilities": "^0.3.1-canary.4",
|
|
59
|
+
"@opengeni/codemode": "^0.4.16-canary.0",
|
|
60
|
+
"@opengeni/codex": "^0.2.19-canary.4",
|
|
61
|
+
"@opengeni/config": "^0.21.0-canary.0",
|
|
62
|
+
"@opengeni/contracts": "^2.6.0-canary.0",
|
|
63
|
+
"@opengeni/core": "^2.5.2-canary.0",
|
|
64
|
+
"@opengeni/db": "^3.5.2-canary.0",
|
|
65
|
+
"@opengeni/documents": "^0.8.5-canary.0",
|
|
66
|
+
"@opengeni/events": "^0.4.3-canary.0",
|
|
67
|
+
"@opengeni/github": "^0.6.0-canary.0",
|
|
68
|
+
"@opengeni/network": "^0.2.3-canary.4",
|
|
69
|
+
"@opengeni/observability": "^0.8.8-canary.0",
|
|
70
|
+
"@opengeni/runtime": "^1.4.1-canary.0",
|
|
71
|
+
"@opengeni/storage": "^0.2.109-canary.0",
|
|
72
|
+
"@opengeni/xai-subscription": "^0.1.2-canary.4",
|
|
73
73
|
"@temporalio/activity": "^1.17.0",
|
|
74
74
|
"@temporalio/client": "^1.17.0",
|
|
75
75
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -45,8 +45,11 @@ import { createTurnCredentialLeases } from "./credential-leases";
|
|
|
45
45
|
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
46
46
|
import { executeGatewayImageGeneration } from "../gateway-image-generation";
|
|
47
47
|
import { executeCodexImageGeneration } from "../codex-image-generation";
|
|
48
|
-
import {
|
|
49
|
-
|
|
48
|
+
import {
|
|
49
|
+
ImageGenerationReferenceError,
|
|
50
|
+
resolveImageGenerationReferencesForTool,
|
|
51
|
+
} from "../image-generation-references";
|
|
52
|
+
import { SandboxChannelAService } from "@opengeni/runtime/sandbox";
|
|
50
53
|
import { sandboxRunAs } from "@opengeni/runtime";
|
|
51
54
|
import { VideoGenerationRejectedResult } from "@opengeni/contracts";
|
|
52
55
|
|
|
@@ -74,6 +77,7 @@ import type {
|
|
|
74
77
|
SandboxRuntimeState,
|
|
75
78
|
} from "./turn-context";
|
|
76
79
|
import { SESSION_TITLE_MODEL_TOOL_NAME } from "./session-title";
|
|
80
|
+
import { resolveImageReferenceSandboxSession } from "./image-reference-sandbox";
|
|
77
81
|
|
|
78
82
|
export type BuildTurnAgentDeps = {
|
|
79
83
|
input: RunAgentTurnInput;
|
|
@@ -240,9 +244,9 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
240
244
|
: {};
|
|
241
245
|
const hostedWebSearch = hostedWebSearchForTurn(resolvedModel, runSettings.webSearchEnabled);
|
|
242
246
|
const resolveImageReferences = async (
|
|
243
|
-
references: Parameters<typeof
|
|
247
|
+
references: Parameters<typeof resolveImageGenerationReferencesForTool>[0]["references"],
|
|
244
248
|
) =>
|
|
245
|
-
await
|
|
249
|
+
await resolveImageGenerationReferencesForTool({
|
|
246
250
|
db,
|
|
247
251
|
objectStorage: objectStorage!,
|
|
248
252
|
accountId: input.accountId,
|
|
@@ -250,17 +254,16 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
250
254
|
subjectId: fileAuthoritySubjectId,
|
|
251
255
|
references,
|
|
252
256
|
readSandboxFile: async (path, maxBytes) => {
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
}
|
|
257
|
+
const imageReferenceSandbox = await resolveImageReferenceSandboxSession(
|
|
258
|
+
sandboxState,
|
|
259
|
+
media.sdkOwnedSandboxSession,
|
|
260
|
+
);
|
|
258
261
|
const relativePath = path.slice("/workspace/".length);
|
|
259
262
|
const referenceRunAs = sandboxRunAs(eventing.modelRunSettings);
|
|
260
263
|
const channel = new SandboxChannelAService({
|
|
261
|
-
session:
|
|
264
|
+
session: imageReferenceSandbox.session,
|
|
262
265
|
workspaceRoot: "/workspace",
|
|
263
|
-
leaseEpoch:
|
|
266
|
+
leaseEpoch: imageReferenceSandbox.leaseEpoch,
|
|
264
267
|
...(referenceRunAs ? { runAs: referenceRunAs } : {}),
|
|
265
268
|
});
|
|
266
269
|
const read = await channel.fsRead({
|
|
@@ -268,7 +271,12 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
268
271
|
encoding: "base64",
|
|
269
272
|
maxBytes,
|
|
270
273
|
});
|
|
271
|
-
if (read.truncated)
|
|
274
|
+
if (read.truncated) {
|
|
275
|
+
throw new ImageGenerationReferenceError(
|
|
276
|
+
"reference_too_large",
|
|
277
|
+
"The sandbox image reference exceeds the per-image byte limit.",
|
|
278
|
+
);
|
|
279
|
+
}
|
|
272
280
|
return Uint8Array.from(Buffer.from(read.content, "base64"));
|
|
273
281
|
},
|
|
274
282
|
});
|
|
@@ -297,7 +305,8 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
297
305
|
imageGeneration: {
|
|
298
306
|
kind: "provider_adapter",
|
|
299
307
|
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
300
|
-
const
|
|
308
|
+
const referenceResolution = await resolveImageReferences(references);
|
|
309
|
+
if (referenceResolution.status === "rejected") return referenceResolution.result;
|
|
301
310
|
const receipt = await executeCodexImageGeneration({
|
|
302
311
|
db,
|
|
303
312
|
objectStorage,
|
|
@@ -308,7 +317,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
308
317
|
attemptId: input.attemptId,
|
|
309
318
|
toolCallId,
|
|
310
319
|
prompt,
|
|
311
|
-
references:
|
|
320
|
+
references: referenceResolution.references,
|
|
312
321
|
credentialId: imageAuthority.credentialId,
|
|
313
322
|
codexContext: imageAuthority.credentialContext,
|
|
314
323
|
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
@@ -331,7 +340,8 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
331
340
|
imageGeneration: {
|
|
332
341
|
kind: "provider_adapter",
|
|
333
342
|
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
334
|
-
const
|
|
343
|
+
const referenceResolution = await resolveImageReferences(references);
|
|
344
|
+
if (referenceResolution.status === "rejected") return referenceResolution.result;
|
|
335
345
|
const receipt = await executeXaiSubscriptionImageGeneration({
|
|
336
346
|
db,
|
|
337
347
|
objectStorage,
|
|
@@ -342,7 +352,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
342
352
|
attemptId: input.attemptId,
|
|
343
353
|
toolCallId,
|
|
344
354
|
prompt,
|
|
345
|
-
references:
|
|
355
|
+
references: referenceResolution.references,
|
|
346
356
|
credentialId: imageAuthority.credentialId,
|
|
347
357
|
xaiContext: imageAuthority.credentialContext,
|
|
348
358
|
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
@@ -366,7 +376,8 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
366
376
|
imageGeneration: {
|
|
367
377
|
kind: "provider_adapter",
|
|
368
378
|
execute: async ({ prompt, references }, { toolCallId }) => {
|
|
369
|
-
const
|
|
379
|
+
const referenceResolution = await resolveImageReferences(references);
|
|
380
|
+
if (referenceResolution.status === "rejected") return referenceResolution.result;
|
|
370
381
|
const receipt = await executeGatewayImageGeneration({
|
|
371
382
|
db,
|
|
372
383
|
objectStorage,
|
|
@@ -378,7 +389,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
378
389
|
apiKey: gatewayApiKey,
|
|
379
390
|
modelId: capabilitySettings.imageGenerationModel,
|
|
380
391
|
prompt,
|
|
381
|
-
references:
|
|
392
|
+
references: referenceResolution.references,
|
|
382
393
|
toolCallId,
|
|
383
394
|
...(runtimeCancellationSignal ? { abortSignal: runtimeCancellationSignal } : {}),
|
|
384
395
|
});
|
|
@@ -185,13 +185,17 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
185
185
|
}
|
|
186
186
|
const turn = claim.turn;
|
|
187
187
|
attempt.turnId = turn.id;
|
|
188
|
+
attempt.executionGeneration = turn.executionGeneration;
|
|
189
|
+
attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
190
|
+
attempt.triggerEventId = turn.triggerEventId;
|
|
191
|
+
attempt.redispatchesAtDispatch = Number(
|
|
192
|
+
(turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ?? 0,
|
|
193
|
+
);
|
|
188
194
|
// Establish durable attempt ownership before any later read can fail.
|
|
189
195
|
// Therefore every failure with no turnId came from the one atomic claim
|
|
190
196
|
// transaction and can be classified without conflating ordinary runtime
|
|
191
197
|
// or transport failures with admission failures.
|
|
192
198
|
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
193
|
-
attempt.executionGeneration = turn.executionGeneration;
|
|
194
|
-
attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
195
199
|
let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
|
|
196
200
|
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
|
|
197
201
|
const fileAuthoritySubjectId = turn.initiatingHumanSubjectId ?? null;
|
|
@@ -268,7 +272,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
268
272
|
billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
|
|
269
273
|
billingState.isCodexTurn = billingIdentity.codexSubscription;
|
|
270
274
|
billingState.isXaiTurn = billingIdentity.xaiSubscription;
|
|
271
|
-
attempt.triggerEventId = turn.triggerEventId;
|
|
272
275
|
const trigger = await getSessionEvent(db, input.workspaceId, attempt.triggerEventId);
|
|
273
276
|
if (!trigger) {
|
|
274
277
|
throw new Error(`Trigger event not found: ${attempt.triggerEventId}`);
|
|
@@ -286,9 +289,6 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
|
|
|
286
289
|
trigger,
|
|
287
290
|
);
|
|
288
291
|
attempt.triggerType = trigger.type;
|
|
289
|
-
attempt.redispatchesAtDispatch = Number(
|
|
290
|
-
(turn.metadata as { workerDeathRedispatches?: number } | null)?.workerDeathRedispatches ?? 0,
|
|
291
|
-
);
|
|
292
292
|
turnLifecycleMetricsFor(observability).start(attempt.turnId);
|
|
293
293
|
// §7.5 P3 — pass the accepted billing attribution (externally funded turns
|
|
294
294
|
// bypass OpenGeni credit/token gates)
|
|
@@ -35,10 +35,16 @@ import {
|
|
|
35
35
|
isXaiSubscriptionTransportError,
|
|
36
36
|
XaiSubscriptionReloginRequired,
|
|
37
37
|
} from "@opengeni/xai-subscription";
|
|
38
|
-
import type {
|
|
38
|
+
import type {
|
|
39
|
+
EscapedMcpTimeoutRecoveryDetail,
|
|
40
|
+
PostClaimDatabaseRecoveryDetail,
|
|
41
|
+
PreClaimFailureDetail,
|
|
42
|
+
} from "../types";
|
|
39
43
|
import {
|
|
40
44
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
41
45
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
46
|
+
POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
|
|
47
|
+
POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
|
|
42
48
|
PRE_CLAIM_FAILURE_MESSAGE,
|
|
43
49
|
PRE_CLAIM_FAILURE_TYPE,
|
|
44
50
|
} from "../types";
|
|
@@ -52,6 +58,16 @@ import {
|
|
|
52
58
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
53
59
|
export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
54
60
|
export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
61
|
+
export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
|
|
62
|
+
|
|
63
|
+
export class PostCompactionContinuationEmptyError extends Error {
|
|
64
|
+
readonly code = POST_COMPACTION_CONTINUATION_EMPTY_CODE;
|
|
65
|
+
|
|
66
|
+
constructor() {
|
|
67
|
+
super("Post-compaction continuation stream ended before a terminal model response");
|
|
68
|
+
this.name = "PostCompactionContinuationEmptyError";
|
|
69
|
+
}
|
|
70
|
+
}
|
|
55
71
|
|
|
56
72
|
export type ProviderRecoveryResult =
|
|
57
73
|
| {
|
|
@@ -89,7 +105,8 @@ export function providerRecoveryResult(input: {
|
|
|
89
105
|
: input.failureCode === "provider_unavailable" ||
|
|
90
106
|
input.failureCode === "upstream_connectivity_unavailable" ||
|
|
91
107
|
input.failureCode === "mcp_transport_timeout" ||
|
|
92
|
-
input.failureCode === "mcp_transport_unavailable"
|
|
108
|
+
input.failureCode === "mcp_transport_unavailable" ||
|
|
109
|
+
input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
|
|
93
110
|
? Math.max(
|
|
94
111
|
providerDelay ?? 0,
|
|
95
112
|
PROVIDER_CONNECTIVITY_BACKOFF_MS[
|
|
@@ -246,32 +263,16 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
246
263
|
*/
|
|
247
264
|
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
248
265
|
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
249
|
-
const
|
|
250
|
-
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
266
|
+
const retryableCode = retryableDatabaseFailureCode(error);
|
|
251
267
|
// The database transaction itself retries only the two contention failures
|
|
252
268
|
// proven safe for immediate replay. Once the activity has failed, the
|
|
253
269
|
// workflow may also retry operational outages after a durable re-read and
|
|
254
270
|
// bounded delay. Unknown driver/database failures stay recoverable because
|
|
255
271
|
// they commonly represent a lost connection; known constraint, auth, and
|
|
256
272
|
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
257
|
-
const retryable = Boolean(
|
|
258
|
-
rawTransportFailure ||
|
|
259
|
-
(persistenceFailure &&
|
|
260
|
-
(sqlState === null ||
|
|
261
|
-
sqlState.startsWith("08") ||
|
|
262
|
-
sqlState.startsWith("40") ||
|
|
263
|
-
sqlState.startsWith("53") ||
|
|
264
|
-
sqlState === "55P03" ||
|
|
265
|
-
sqlState === "57014" ||
|
|
266
|
-
sqlState === "57P01" ||
|
|
267
|
-
sqlState === "57P02" ||
|
|
268
|
-
sqlState === "57P03" ||
|
|
269
|
-
sqlState.startsWith("58"))),
|
|
270
|
-
);
|
|
271
273
|
const detail: PreClaimFailureDetail = {
|
|
272
|
-
disposition:
|
|
273
|
-
code:
|
|
274
|
-
persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
|
|
274
|
+
disposition: retryableCode ? "retryable" : "permanent",
|
|
275
|
+
code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
|
|
275
276
|
};
|
|
276
277
|
return ApplicationFailure.create({
|
|
277
278
|
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
@@ -281,6 +282,80 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
|
281
282
|
});
|
|
282
283
|
}
|
|
283
284
|
|
|
285
|
+
function retryableDatabaseFailureCode(
|
|
286
|
+
error: unknown,
|
|
287
|
+
): PostClaimDatabaseRecoveryDetail["code"] | null {
|
|
288
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
289
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
290
|
+
if (sqlState === null && isRetryableDatabaseTransportFailure(error)) {
|
|
291
|
+
return "db_failure";
|
|
292
|
+
}
|
|
293
|
+
if (
|
|
294
|
+
!persistenceFailure ||
|
|
295
|
+
!(
|
|
296
|
+
sqlState === null ||
|
|
297
|
+
sqlState.startsWith("08") ||
|
|
298
|
+
sqlState.startsWith("40") ||
|
|
299
|
+
sqlState.startsWith("53") ||
|
|
300
|
+
sqlState === "55P03" ||
|
|
301
|
+
sqlState === "57014" ||
|
|
302
|
+
sqlState === "57P01" ||
|
|
303
|
+
sqlState === "57P02" ||
|
|
304
|
+
sqlState === "57P03" ||
|
|
305
|
+
sqlState.startsWith("58")
|
|
306
|
+
)
|
|
307
|
+
) {
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
return persistenceFailure.details.code;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Carry one exact claimed-but-not-started attempt into the workflow's DB-only
|
|
315
|
+
* recovery lane. Permanent database/state failures remain terminal; only the
|
|
316
|
+
* same operational outage classes that are safe before claim are admitted.
|
|
317
|
+
*/
|
|
318
|
+
export function postClaimDatabaseRecoveryFailure(input: {
|
|
319
|
+
error: unknown;
|
|
320
|
+
turnId: string;
|
|
321
|
+
triggerEventId: string;
|
|
322
|
+
executionGeneration: number;
|
|
323
|
+
providerRecovery?: {
|
|
324
|
+
failureCode: string;
|
|
325
|
+
providerRecoveryCount: number;
|
|
326
|
+
};
|
|
327
|
+
}): ApplicationFailure | null {
|
|
328
|
+
const code = retryableDatabaseFailureCode(input.error);
|
|
329
|
+
if (!code || input.executionGeneration < 1) return null;
|
|
330
|
+
if (
|
|
331
|
+
input.providerRecovery &&
|
|
332
|
+
(!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
|
|
333
|
+
input.providerRecovery.providerRecoveryCount <= 0 ||
|
|
334
|
+
input.providerRecovery.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
|
|
335
|
+
!/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode))
|
|
336
|
+
) {
|
|
337
|
+
return null;
|
|
338
|
+
}
|
|
339
|
+
const detail: PostClaimDatabaseRecoveryDetail = {
|
|
340
|
+
turnId: input.turnId,
|
|
341
|
+
triggerEventId: input.triggerEventId,
|
|
342
|
+
executionGeneration: input.executionGeneration,
|
|
343
|
+
code,
|
|
344
|
+
...(input.providerRecovery
|
|
345
|
+
? {
|
|
346
|
+
providerFailureCode: input.providerRecovery.failureCode,
|
|
347
|
+
providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
|
|
348
|
+
}
|
|
349
|
+
: {}),
|
|
350
|
+
};
|
|
351
|
+
return ApplicationFailure.create({
|
|
352
|
+
message: POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
|
|
353
|
+
type: POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
|
|
354
|
+
nonRetryable: true,
|
|
355
|
+
details: [detail],
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
|
|
284
359
|
/**
|
|
285
360
|
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
286
361
|
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
@@ -729,6 +804,14 @@ export function agentRunFailurePayload(
|
|
|
729
804
|
detail: error.message,
|
|
730
805
|
};
|
|
731
806
|
}
|
|
807
|
+
if (error instanceof PostCompactionContinuationEmptyError) {
|
|
808
|
+
return {
|
|
809
|
+
error:
|
|
810
|
+
"Context compaction completed, but the continuation ended before a new model response. The same turn will retry from the compacted checkpoint.",
|
|
811
|
+
code: POST_COMPACTION_CONTINUATION_EMPTY_CODE,
|
|
812
|
+
retryable: true,
|
|
813
|
+
};
|
|
814
|
+
}
|
|
732
815
|
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
733
816
|
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
734
817
|
// model or tool effects, so this marked transport failure must outrank the
|
|
@@ -55,6 +55,7 @@ import { createTurnHistorySink } from "./history-sink";
|
|
|
55
55
|
import { BudgetExhaustedError } from "./admission";
|
|
56
56
|
import {
|
|
57
57
|
providerRecoveryExhaustedFailure,
|
|
58
|
+
postClaimDatabaseRecoveryFailure,
|
|
58
59
|
providerRecoveryResult,
|
|
59
60
|
providerRetryAfterMs,
|
|
60
61
|
escapedMcpTimeoutRecoveryFailure,
|
|
@@ -1387,9 +1388,48 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1387
1388
|
control.activityError = error;
|
|
1388
1389
|
throw escaped;
|
|
1389
1390
|
}
|
|
1391
|
+
const postClaimRecovery =
|
|
1392
|
+
recoveryResult.status === "recovering"
|
|
1393
|
+
? postClaimDatabaseRecoveryFailure({
|
|
1394
|
+
error: recoveryError,
|
|
1395
|
+
turnId: attempt.turnId,
|
|
1396
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1397
|
+
executionGeneration: attempt.executionGeneration,
|
|
1398
|
+
providerRecovery: {
|
|
1399
|
+
failureCode: failure.code ?? "provider_unavailable",
|
|
1400
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1401
|
+
},
|
|
1402
|
+
})
|
|
1403
|
+
: null;
|
|
1404
|
+
if (postClaimRecovery) {
|
|
1405
|
+
control.activityStatus = "recovering";
|
|
1406
|
+
control.turnMetricOutcome = "recovering";
|
|
1407
|
+
control.activityError = error;
|
|
1408
|
+
throw postClaimRecovery;
|
|
1409
|
+
}
|
|
1390
1410
|
throw recoveryError;
|
|
1391
1411
|
}
|
|
1392
1412
|
}
|
|
1413
|
+
if (
|
|
1414
|
+
attempt.turnId &&
|
|
1415
|
+
attempt.triggerEventId &&
|
|
1416
|
+
attempt.executionGeneration > 0 &&
|
|
1417
|
+
!eventing.turnStartedPublished &&
|
|
1418
|
+
!attempt.modelRequestStarted
|
|
1419
|
+
) {
|
|
1420
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
1421
|
+
error,
|
|
1422
|
+
turnId: attempt.turnId,
|
|
1423
|
+
triggerEventId: attempt.triggerEventId,
|
|
1424
|
+
executionGeneration: attempt.executionGeneration,
|
|
1425
|
+
});
|
|
1426
|
+
if (recoveryFailure) {
|
|
1427
|
+
control.activityStatus = "recovering";
|
|
1428
|
+
control.turnMetricOutcome = "recovering";
|
|
1429
|
+
control.activityError = error;
|
|
1430
|
+
throw recoveryFailure;
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1393
1433
|
control.activityStatus = "failed";
|
|
1394
1434
|
control.activityError = error;
|
|
1395
1435
|
if (!attempt.turnId) {
|
|
@@ -128,6 +128,33 @@ export type GovernanceModelOk = {
|
|
|
128
128
|
|
|
129
129
|
export type GovernanceModelOutcome = { exit: RunAgentTurnResult } | { ok: GovernanceModelOk };
|
|
130
130
|
|
|
131
|
+
type PreferenceSnapshotTurn = Pick<ClaimTurnOk["turn"], "initiatingHumanSubjectId" | "initiator">;
|
|
132
|
+
|
|
133
|
+
export function preferenceSnapshotHumanSubjectId(turn: PreferenceSnapshotTurn): string | null {
|
|
134
|
+
return (
|
|
135
|
+
turn.initiatingHumanSubjectId ??
|
|
136
|
+
(turn.initiator.kind === "subject" ? turn.initiator.subjectId : null)
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Preferences are scoped partly to the frozen initiating human. A service-only
|
|
142
|
+
* turn has no such scope, so avoid calling the human-bound snapshot capability
|
|
143
|
+
* merely to catch its expected authority rejection.
|
|
144
|
+
*/
|
|
145
|
+
export async function preferenceSnapshotForTurn<T>(
|
|
146
|
+
turn: PreferenceSnapshotTurn,
|
|
147
|
+
load: () => Promise<T>,
|
|
148
|
+
): Promise<T | null> {
|
|
149
|
+
if (!preferenceSnapshotHumanSubjectId(turn)) return null;
|
|
150
|
+
try {
|
|
151
|
+
return await load();
|
|
152
|
+
} catch (error) {
|
|
153
|
+
if (error instanceof PreferenceRegistryInitiatorError) return null;
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
131
158
|
export async function prepareGovernanceAndModel(
|
|
132
159
|
deps: GovernanceModelDeps,
|
|
133
160
|
): Promise<GovernanceModelOutcome> {
|
|
@@ -197,10 +224,10 @@ export async function prepareGovernanceAndModel(
|
|
|
197
224
|
getWorkspace(db, input.workspaceId),
|
|
198
225
|
getOrCreateCompanyProfileSnapshot(db, governanceClaims),
|
|
199
226
|
getOrCreateWorkspaceInstructionPolicySnapshot(db, governanceClaims),
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
227
|
+
preferenceSnapshotForTurn(
|
|
228
|
+
turn,
|
|
229
|
+
async () => await getOrCreatePreferenceRegistrySnapshot(db, governanceClaims),
|
|
230
|
+
),
|
|
204
231
|
]),
|
|
205
232
|
getWorkspaceModelPolicy(db, input.workspaceId),
|
|
206
233
|
]);
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
normalizeProtocolJsonValue,
|
|
3
|
+
sanitizeHistoryItemsForModel,
|
|
4
|
+
toolCallIdFromSdkItem,
|
|
5
|
+
} from "@opengeni/runtime";
|
|
2
6
|
import { opaqueProviderArtifactFingerprint } from "@opengeni/codex";
|
|
3
7
|
|
|
4
8
|
export function historyRowsToAppend(
|
|
@@ -149,7 +153,7 @@ export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
|
149
153
|
// never receives a separate function result and therefore must not enter the
|
|
150
154
|
// pending function-call ledger.
|
|
151
155
|
if (raw.type === "hosted_tool_call" && raw.name === "image_generation_call") return null;
|
|
152
|
-
const callId = raw
|
|
156
|
+
const callId = toolCallIdFromSdkItem(raw) ?? raw.id;
|
|
153
157
|
const callType = raw.type;
|
|
154
158
|
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
155
159
|
return null;
|
|
@@ -182,7 +186,7 @@ export function completedToolCallFromSdkEvent(event: unknown): {
|
|
|
182
186
|
item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
|
|
183
187
|
? (item.rawItem as Record<string, unknown>)
|
|
184
188
|
: {};
|
|
185
|
-
const callId = raw
|
|
189
|
+
const callId = toolCallIdFromSdkItem(raw) ?? item.id;
|
|
186
190
|
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
187
191
|
return {
|
|
188
192
|
callId,
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { ChannelASession } from "@opengeni/runtime/sandbox";
|
|
2
|
+
import type { SandboxRuntimeState } from "./turn-context";
|
|
3
|
+
|
|
4
|
+
type ImageReferenceSandboxState = Pick<
|
|
5
|
+
SandboxRuntimeState,
|
|
6
|
+
"resolvedSandbox" | "lazyOwnedSandbox" | "turnSandboxProvisioner"
|
|
7
|
+
>;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Join the turn's canonical lazy sandbox boundary before reading a
|
|
11
|
+
* model-supplied /workspace image reference. Durable File and artifact
|
|
12
|
+
* references never call this helper because their bytes come from object
|
|
13
|
+
* storage instead.
|
|
14
|
+
*/
|
|
15
|
+
export async function resolveImageReferenceSandboxSession(
|
|
16
|
+
sandboxState: ImageReferenceSandboxState,
|
|
17
|
+
sdkOwnedSandboxSession: unknown,
|
|
18
|
+
): Promise<{ session: ChannelASession; leaseEpoch: number }> {
|
|
19
|
+
let sandbox = sandboxState.resolvedSandbox;
|
|
20
|
+
if (!sandbox && sandboxState.turnSandboxProvisioner) {
|
|
21
|
+
sandbox = await sandboxState.turnSandboxProvisioner.get();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const session = (sandboxState.lazyOwnedSandbox?.session ??
|
|
25
|
+
sandbox?.established.session ??
|
|
26
|
+
sdkOwnedSandboxSession) as ChannelASession | null;
|
|
27
|
+
if (!session) {
|
|
28
|
+
throw new Error("Sandbox image reference is unavailable");
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
return {
|
|
32
|
+
session,
|
|
33
|
+
leaseEpoch: sandbox?.leaseEpoch ?? 0,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { TurnHistorySink } from "./history-sink";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Persist the SDK's complete prior model/tool history before another provider
|
|
5
|
+
* request can start. The first request has no prior model/tool rows to append.
|
|
6
|
+
* Follow-up requests run only after the preceding complete call/result batch is
|
|
7
|
+
* replay-safe.
|
|
8
|
+
*/
|
|
9
|
+
export async function checkpointHistoryBeforeProviderDispatch(
|
|
10
|
+
historySink: Pick<TurnHistorySink, "reconcileConversationTruth">,
|
|
11
|
+
): Promise<void> {
|
|
12
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
13
|
+
}
|
|
@@ -44,6 +44,7 @@ import { ToolResultSpill } from "./tool-result-spill";
|
|
|
44
44
|
import { createTurnCredentialLeases } from "./credential-leases";
|
|
45
45
|
import { createTurnMediaArtifacts } from "./media-artifacts";
|
|
46
46
|
import { createTurnHistorySink } from "./history-sink";
|
|
47
|
+
import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
|
|
47
48
|
import { sandboxRunAs } from "@opengeni/runtime";
|
|
48
49
|
import { randomUUID } from "node:crypto";
|
|
49
50
|
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
@@ -562,6 +563,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
562
563
|
if (!eventing.publish || !attempt.turnId) {
|
|
563
564
|
throw new Error("Codex model request started before the turn event producer");
|
|
564
565
|
}
|
|
566
|
+
if (event.phase === "started") {
|
|
567
|
+
await checkpointHistoryBeforeProviderDispatch(historySink);
|
|
568
|
+
}
|
|
565
569
|
const shouldRecordStartedAudit =
|
|
566
570
|
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
567
571
|
if (shouldRecordStartedAudit) {
|
|
@@ -707,6 +711,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
707
711
|
if (!eventing.publish || !attempt.turnId) {
|
|
708
712
|
throw new Error("SuperGrok model request started before the turn event producer");
|
|
709
713
|
}
|
|
714
|
+
if (event.phase === "started") {
|
|
715
|
+
await checkpointHistoryBeforeProviderDispatch(historySink);
|
|
716
|
+
}
|
|
710
717
|
const shouldRecordStartedAudit =
|
|
711
718
|
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
712
719
|
if (shouldRecordStartedAudit) firstModelRequestAuditRecorded = true;
|