@opengeni/worker-bundle 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +22 -6
- package/dist/index.js +970 -325
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +15 -12
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +444 -146
- package/src/activities/capabilities.ts +19 -37
- package/src/activities/codex-capacity.ts +1 -1
- package/src/activities/redaction.ts +9 -56
- package/src/activities/sandbox-lease.ts +411 -0
- package/src/activities/scheduled-tasks.ts +53 -1
- package/src/activities/types.ts +6 -6
- package/src/concurrency.ts +58 -6
- package/src/http.ts +14 -2
- package/src/index.ts +36 -12
- package/src/observability-metrics.ts +137 -0
- package/src/sandbox-resume.ts +33 -114
- package/src/sandbox-routing.ts +95 -30
- package/src/workflows/session.ts +14 -11
|
@@ -104,6 +104,7 @@ import {
|
|
|
104
104
|
SUMMARY_BUFFER_TOKENS,
|
|
105
105
|
runOwnedSandboxSetup,
|
|
106
106
|
RoutingMutationOutcomeUnknownError,
|
|
107
|
+
WorkspaceArchiveIntegrityError,
|
|
107
108
|
swapTargetEstablishability,
|
|
108
109
|
type SandboxFileDownload,
|
|
109
110
|
type SandboxFileDownloadFailure,
|
|
@@ -113,6 +114,7 @@ import {
|
|
|
113
114
|
type ModelCallUsageInput,
|
|
114
115
|
type ModelCallUsageNormalization,
|
|
115
116
|
type BuildAgentOptions,
|
|
117
|
+
type RunAgentStreamOptions,
|
|
116
118
|
type TurnToolCancellationFence,
|
|
117
119
|
type BackendUnresolvableCode,
|
|
118
120
|
type EstablishedSandboxSession,
|
|
@@ -182,7 +184,11 @@ import {
|
|
|
182
184
|
type CodexUsageHeaderSnapshot,
|
|
183
185
|
} from "@opengeni/codex";
|
|
184
186
|
import { mergeResourceRefs } from "./common";
|
|
185
|
-
import {
|
|
187
|
+
import {
|
|
188
|
+
enabledCapabilityMcpToolRefs,
|
|
189
|
+
resolveSessionToolPolicy,
|
|
190
|
+
sessionToolPolicyAllowsDefaultNativeTools,
|
|
191
|
+
} from "@opengeni/core";
|
|
186
192
|
import { maybeCompactContext } from "./context-compaction";
|
|
187
193
|
import { TurnAttemptFencedError } from "./turn-attempt-fenced";
|
|
188
194
|
import {
|
|
@@ -226,7 +232,12 @@ import {
|
|
|
226
232
|
settingsWithRigImage,
|
|
227
233
|
} from "./packs";
|
|
228
234
|
import { deliverFailedChildTurnToParent } from "./parent-wake";
|
|
229
|
-
import {
|
|
235
|
+
import {
|
|
236
|
+
createSecretRedactor,
|
|
237
|
+
isCredentialHeaderName,
|
|
238
|
+
redactSensitiveText,
|
|
239
|
+
type SecretForRedaction,
|
|
240
|
+
} from "./redaction";
|
|
230
241
|
import { applyCodexHistoryStrip, turnInput, type TurnCodexAccount } from "./run-input";
|
|
231
242
|
import {
|
|
232
243
|
createRuntimeBatcher,
|
|
@@ -242,7 +253,6 @@ import type {
|
|
|
242
253
|
} from "./types";
|
|
243
254
|
import {
|
|
244
255
|
resumeBoxForTurn,
|
|
245
|
-
acquireSelfhostedLeaseForTurn,
|
|
246
256
|
sandboxLeaseHolderIdForAttempt,
|
|
247
257
|
maybePersistWarmWorkspaceSnapshot,
|
|
248
258
|
waitForWarmSnapshot,
|
|
@@ -613,6 +623,84 @@ function compactionFailureReason(reason: string): string {
|
|
|
613
623
|
: `compaction summarization failed: ${reason}`;
|
|
614
624
|
}
|
|
615
625
|
|
|
626
|
+
export type SafeErrorDiagnostic = {
|
|
627
|
+
name: string;
|
|
628
|
+
message: string;
|
|
629
|
+
status?: number;
|
|
630
|
+
code?: string;
|
|
631
|
+
};
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Produce the only exception shape allowed in worker logs. It deliberately
|
|
635
|
+
* excludes stack, cause, response/request bodies, and arbitrary enumerable
|
|
636
|
+
* properties while retaining bounded, redacted diagnostics.
|
|
637
|
+
*/
|
|
638
|
+
export function safeErrorDiagnostic(
|
|
639
|
+
error: unknown,
|
|
640
|
+
redactText: (value: string) => string = redactSensitiveText,
|
|
641
|
+
): SafeErrorDiagnostic {
|
|
642
|
+
const rawName = error instanceof Error ? error.name : "Error";
|
|
643
|
+
const name = /^[A-Za-z][A-Za-z0-9_.-]{0,79}$/.test(rawName) ? rawName : "Error";
|
|
644
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
645
|
+
const diagnostic: SafeErrorDiagnostic = {
|
|
646
|
+
name,
|
|
647
|
+
message: redactText(rawMessage).slice(0, 2_048),
|
|
648
|
+
};
|
|
649
|
+
if (error && typeof error === "object") {
|
|
650
|
+
const status = Number(
|
|
651
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
652
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
653
|
+
);
|
|
654
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
655
|
+
diagnostic.status = status;
|
|
656
|
+
}
|
|
657
|
+
const rawCode = (error as { code?: unknown }).code;
|
|
658
|
+
if (typeof rawCode === "string" || typeof rawCode === "number") {
|
|
659
|
+
const code = redactText(String(rawCode)).slice(0, 80);
|
|
660
|
+
if (/^[A-Za-z0-9_.:-]+$/.test(code)) diagnostic.code = code;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
return diagnostic;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function safeErrorForTelemetry(error: unknown, redactText: (value: string) => string): Error {
|
|
667
|
+
const diagnostic = safeErrorDiagnostic(error, redactText);
|
|
668
|
+
const safe = new Error(diagnostic.message);
|
|
669
|
+
safe.name = diagnostic.name;
|
|
670
|
+
return safe;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
export function headerSecretRedactions(
|
|
674
|
+
prefix: string,
|
|
675
|
+
headers: Readonly<Record<string, string>> | undefined,
|
|
676
|
+
credentialHeaderNames?: Iterable<string>,
|
|
677
|
+
): SecretForRedaction[] {
|
|
678
|
+
const discovered: SecretForRedaction[] = [];
|
|
679
|
+
const explicitCredentialHeaders = credentialHeaderNames
|
|
680
|
+
? new Set([...credentialHeaderNames].map((name) => name.toLowerCase()))
|
|
681
|
+
: null;
|
|
682
|
+
for (const [headerName, value] of Object.entries(headers ?? {})) {
|
|
683
|
+
if (
|
|
684
|
+
!explicitCredentialHeaders?.has(headerName.toLowerCase()) &&
|
|
685
|
+
!isCredentialHeaderName(headerName)
|
|
686
|
+
) {
|
|
687
|
+
continue;
|
|
688
|
+
}
|
|
689
|
+
const safeHeaderName = headerName.toUpperCase().replace(/[^A-Z0-9]+/g, "_");
|
|
690
|
+
discovered.push({ name: `${prefix}_${safeHeaderName || "HEADER"}`, value });
|
|
691
|
+
if (/^(?:proxy-)?authorization$/i.test(headerName)) {
|
|
692
|
+
const credential = value.match(/^\s*[A-Za-z][A-Za-z0-9_-]*\s+(.+)\s*$/)?.[1];
|
|
693
|
+
if (credential) {
|
|
694
|
+
discovered.push({
|
|
695
|
+
name: `${prefix}_${safeHeaderName || "AUTHORIZATION"}_CREDENTIAL`,
|
|
696
|
+
value: credential,
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
return discovered;
|
|
702
|
+
}
|
|
703
|
+
|
|
616
704
|
function compactionFailureReasonFromError(error: unknown): string {
|
|
617
705
|
if (
|
|
618
706
|
error instanceof CompactionProviderResponseError ||
|
|
@@ -1198,14 +1286,15 @@ export function historyRowsToAppend(
|
|
|
1198
1286
|
// positions from 0) when callers do not pass an explicit next position.
|
|
1199
1287
|
nextPosition: number = persistedHistoryCount,
|
|
1200
1288
|
toolOutputTruncationTokens?: number,
|
|
1289
|
+
redactValue: (value: unknown) => unknown = (value) => value,
|
|
1201
1290
|
): {
|
|
1202
1291
|
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
1203
1292
|
nextWatermark: number;
|
|
1204
1293
|
nextPosition: number;
|
|
1205
1294
|
} {
|
|
1206
|
-
const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens)
|
|
1207
|
-
(item) => !isEphemeralInternalContext(item)
|
|
1208
|
-
|
|
1295
|
+
const sanitized = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens)
|
|
1296
|
+
.filter((item) => !isEphemeralInternalContext(item))
|
|
1297
|
+
.map((item) => redactValue(item) as Record<string, unknown>);
|
|
1209
1298
|
if (sanitized.length <= persistedHistoryCount) {
|
|
1210
1299
|
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
1211
1300
|
}
|
|
@@ -1220,6 +1309,16 @@ export function historyRowsToAppend(
|
|
|
1220
1309
|
};
|
|
1221
1310
|
}
|
|
1222
1311
|
|
|
1312
|
+
/** Literal final per-call boundary for replayed and current-turn model input. */
|
|
1313
|
+
export function secretRedactionModelInputFilter(
|
|
1314
|
+
redactValue: (value: unknown) => unknown,
|
|
1315
|
+
): NonNullable<RunAgentStreamOptions["callModelInputFilter"]> {
|
|
1316
|
+
return ({ modelData }) => ({
|
|
1317
|
+
...modelData,
|
|
1318
|
+
input: redactValue(modelData.input) as typeof modelData.input,
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1223
1322
|
function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
1224
1323
|
if (item.type === "message") {
|
|
1225
1324
|
return item.role === "assistant";
|
|
@@ -1322,12 +1421,29 @@ export async function resolveActiveSandboxBackend(
|
|
|
1322
1421
|
} catch (error) {
|
|
1323
1422
|
console.error(
|
|
1324
1423
|
"active sandbox backend resolution failed (turn proceeds on home backend)",
|
|
1325
|
-
error,
|
|
1424
|
+
safeErrorDiagnostic(error),
|
|
1326
1425
|
);
|
|
1327
1426
|
return undefined;
|
|
1328
1427
|
}
|
|
1329
1428
|
}
|
|
1330
1429
|
|
|
1430
|
+
/**
|
|
1431
|
+
* Managed-sandbox ownership exists only when this turn actually uses the
|
|
1432
|
+
* managed home. A Connected Machine is fenced by its active-route epoch and
|
|
1433
|
+
* must remain independent from cloud recovery state for the same session.
|
|
1434
|
+
*/
|
|
1435
|
+
export function managedSandboxOwnershipForTurn(
|
|
1436
|
+
machinePrimary: boolean,
|
|
1437
|
+
attemptId: string,
|
|
1438
|
+
sandboxGroupId: string,
|
|
1439
|
+
): { holderId: TurnSandboxLeaseHolderId; sandboxGroupId: string } | null {
|
|
1440
|
+
if (machinePrimary) return null;
|
|
1441
|
+
return {
|
|
1442
|
+
holderId: sandboxLeaseHolderIdForAttempt(attemptId),
|
|
1443
|
+
sandboxGroupId,
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1331
1447
|
/**
|
|
1332
1448
|
* Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
|
|
1333
1449
|
* invariant B). Returns the typed reason to RESET the pointer to the session HOME,
|
|
@@ -1543,6 +1659,22 @@ export function computerToolModeForTurn(
|
|
|
1543
1659
|
return "hosted";
|
|
1544
1660
|
}
|
|
1545
1661
|
|
|
1662
|
+
/**
|
|
1663
|
+
* Native web search requires two independent grants: the resolved provider
|
|
1664
|
+
* must advertise runnable support, and the immutable session/turn policy must
|
|
1665
|
+
* still track workspace defaults. The null model is the legacy built-in
|
|
1666
|
+
* Responses path, whose deployment flag is its provider capability gate.
|
|
1667
|
+
* There is deliberately no cross-provider or sandbox/curl fallback.
|
|
1668
|
+
*/
|
|
1669
|
+
export function hostedWebSearchForTurn(
|
|
1670
|
+
resolvedModel: { configured: { hostedWebSearch: boolean } } | null,
|
|
1671
|
+
deploymentWebSearchEnabled: boolean,
|
|
1672
|
+
defaultNativeToolsAllowed: boolean,
|
|
1673
|
+
): boolean {
|
|
1674
|
+
if (!defaultNativeToolsAllowed) return false;
|
|
1675
|
+
return resolvedModel?.configured.hostedWebSearch ?? deploymentWebSearchEnabled;
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1546
1678
|
/**
|
|
1547
1679
|
* Decide whether this turn's provider accepts typed file/image content in a
|
|
1548
1680
|
* user message. The legacy built-in client and registry Responses clients use
|
|
@@ -1650,6 +1782,9 @@ export function isLazySandboxProvisionRetryable(error: unknown): boolean {
|
|
|
1650
1782
|
if (error instanceof SandboxImageConflictError) {
|
|
1651
1783
|
return false;
|
|
1652
1784
|
}
|
|
1785
|
+
if (error instanceof WorkspaceArchiveIntegrityError) {
|
|
1786
|
+
return error.retryable;
|
|
1787
|
+
}
|
|
1653
1788
|
if (error instanceof SandboxLeaseSupersededError || error instanceof SandboxWarmingTimeoutError) {
|
|
1654
1789
|
return true;
|
|
1655
1790
|
}
|
|
@@ -2225,6 +2360,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2225
2360
|
await current?.flush().catch(() => undefined);
|
|
2226
2361
|
};
|
|
2227
2362
|
let preparedTools: Awaited<ReturnType<OpenGeniRuntime["prepareTools"]>> | null = null;
|
|
2363
|
+
// Resolved with the MCP policy at the immutable turn boundary. Provider
|
|
2364
|
+
// capability is checked separately when buildAgent is called; this flag is
|
|
2365
|
+
// only the durable omission/narrowing entitlement.
|
|
2366
|
+
let defaultNativeToolsAllowed = false;
|
|
2228
2367
|
const toolCancellationFenceRef: {
|
|
2229
2368
|
current: TurnToolCancellationFence | null;
|
|
2230
2369
|
} = {
|
|
@@ -2286,7 +2425,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2286
2425
|
]).catch((publishError) => {
|
|
2287
2426
|
// The lease transition is already authoritative. A fenced/failed audit
|
|
2288
2427
|
// append must not prevent the same logical turn from recovering.
|
|
2289
|
-
console.error("sandbox box lost event publish failed", publishError);
|
|
2428
|
+
console.error("sandbox box lost event publish failed", safeErrorDiagnostic(publishError));
|
|
2290
2429
|
});
|
|
2291
2430
|
};
|
|
2292
2431
|
const startLeaseHeartbeat = (
|
|
@@ -2447,7 +2586,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2447
2586
|
}
|
|
2448
2587
|
console.error(
|
|
2449
2588
|
"computer-use recording start failed (action outcome unaffected)",
|
|
2450
|
-
recordingError,
|
|
2589
|
+
safeErrorDiagnostic(recordingError),
|
|
2451
2590
|
);
|
|
2452
2591
|
}
|
|
2453
2592
|
})();
|
|
@@ -2495,6 +2634,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2495
2634
|
persistedHistoryCount,
|
|
2496
2635
|
nextHistoryPosition,
|
|
2497
2636
|
modelRunSettings.modelToolOutputTruncationTokens,
|
|
2637
|
+
(value) => redact(value),
|
|
2498
2638
|
);
|
|
2499
2639
|
const hasModelOrToolProgress = rows.some((row) =>
|
|
2500
2640
|
isModelOrToolProgressHistoryItem(row.item),
|
|
@@ -2538,13 +2678,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2538
2678
|
});
|
|
2539
2679
|
}
|
|
2540
2680
|
} catch (persistError) {
|
|
2541
|
-
console.error(
|
|
2681
|
+
console.error(
|
|
2682
|
+
"session history dual-write failed (run unaffected)",
|
|
2683
|
+
safeErrorDiagnostic(persistError, (value) => String(redact(value))),
|
|
2684
|
+
);
|
|
2542
2685
|
if (options.requireDurable) throw persistError;
|
|
2543
2686
|
}
|
|
2544
2687
|
};
|
|
2545
2688
|
// Reassigned after the variable set loads; the publish closure is
|
|
2546
2689
|
// created (and used for turn.started) before the variableSet is available.
|
|
2547
|
-
|
|
2690
|
+
// Generic auth/cookie/signed-URL/token classification is active before any
|
|
2691
|
+
// exact run-secret provenance is available. Registered values strengthen
|
|
2692
|
+
// this boundary as credentials are resolved and renewed during the turn.
|
|
2693
|
+
let redact: (payload: unknown) => unknown = createSecretRedactor([]);
|
|
2548
2694
|
const secretRedactions = new Map<string, string>();
|
|
2549
2695
|
const publishedRunCredentialNotices = new Set<string>();
|
|
2550
2696
|
const registerSecretRedactions = (secrets: SecretForRedaction[]): void => {
|
|
@@ -2805,6 +2951,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2805
2951
|
producerId,
|
|
2806
2952
|
producerSeq: ++producerSeq,
|
|
2807
2953
|
}));
|
|
2954
|
+
const redactedRunState = inputSettlement.runState
|
|
2955
|
+
? (redact(inputSettlement.runState) as typeof inputSettlement.runState)
|
|
2956
|
+
: undefined;
|
|
2808
2957
|
const result = await applySessionTurnSettlement(db, input.workspaceId, {
|
|
2809
2958
|
sessionId: input.sessionId,
|
|
2810
2959
|
turnId: turnId!,
|
|
@@ -2814,7 +2963,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2814
2963
|
sessionStatus: inputSettlement.sessionStatus,
|
|
2815
2964
|
activeTurnId: inputSettlement.activeTurnId,
|
|
2816
2965
|
events: inputs,
|
|
2817
|
-
...(
|
|
2966
|
+
...(redactedRunState ? { runState: redactedRunState } : {}),
|
|
2818
2967
|
...(recordingMutation ? { recording: recordingMutation } : {}),
|
|
2819
2968
|
...(compactionRequestFailure ? { compactionRequestFailure } : {}),
|
|
2820
2969
|
});
|
|
@@ -3246,43 +3395,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3246
3395
|
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
3247
3396
|
() => null,
|
|
3248
3397
|
);
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3398
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
3399
|
+
const armed = await armCodexCapacityWait(db, {
|
|
3400
|
+
accountId: input.accountId,
|
|
3401
|
+
workspaceId: input.workspaceId,
|
|
3402
|
+
sessionId: input.sessionId,
|
|
3403
|
+
turnId,
|
|
3404
|
+
attemptId: input.attemptId,
|
|
3405
|
+
workflowId: input.workflowId,
|
|
3406
|
+
goalId: activeGoal?.id ?? null,
|
|
3407
|
+
goalVersion: activeGoal?.version ?? null,
|
|
3408
|
+
earliestResetAt: null,
|
|
3409
|
+
resetKind: "bounded_refresh",
|
|
3410
|
+
failurePayload: {
|
|
3411
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
3412
|
+
code: "codex_allocator_disabled",
|
|
3413
|
+
detail: "waiting for a credential to be re-enabled, reconnected, or added",
|
|
3414
|
+
},
|
|
3415
|
+
});
|
|
3416
|
+
if (armed.action === "waiting") {
|
|
3417
|
+
await publishDurableSessionEvents(
|
|
3418
|
+
bus,
|
|
3419
|
+
input.workspaceId,
|
|
3420
|
+
input.sessionId,
|
|
3421
|
+
armed.events,
|
|
3422
|
+
);
|
|
3423
|
+
turnMetricOutcome = "recovering";
|
|
3424
|
+
activityStatus = "waiting_capacity";
|
|
3425
|
+
return claimedResult({
|
|
3426
|
+
status: "waiting_capacity",
|
|
3427
|
+
capacityWait: {
|
|
3428
|
+
waiterId: armed.waiter.id,
|
|
3429
|
+
generation: armed.waiter.generation,
|
|
3430
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
3431
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
3265
3432
|
},
|
|
3266
3433
|
});
|
|
3267
|
-
if (armed.action === "waiting") {
|
|
3268
|
-
await publishDurableSessionEvents(
|
|
3269
|
-
bus,
|
|
3270
|
-
input.workspaceId,
|
|
3271
|
-
input.sessionId,
|
|
3272
|
-
armed.events,
|
|
3273
|
-
);
|
|
3274
|
-
turnMetricOutcome = "failed";
|
|
3275
|
-
activityStatus = "idle";
|
|
3276
|
-
return claimedResult({
|
|
3277
|
-
status: "idle",
|
|
3278
|
-
capacityWait: {
|
|
3279
|
-
waiterId: armed.waiter.id,
|
|
3280
|
-
generation: armed.waiter.generation,
|
|
3281
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
3282
|
-
wakeRevision: armed.waiter.wakeRevision,
|
|
3283
|
-
},
|
|
3284
|
-
});
|
|
3285
|
-
}
|
|
3286
3434
|
}
|
|
3287
3435
|
if (
|
|
3288
3436
|
!(await settle!({
|
|
@@ -3361,43 +3509,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3361
3509
|
"all connected Codex subscriptions are rate-limited",
|
|
3362
3510
|
{ allAccounts: true },
|
|
3363
3511
|
);
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3512
|
+
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
3513
|
+
leased.accounts,
|
|
3514
|
+
new Date(),
|
|
3515
|
+
);
|
|
3516
|
+
const armed = await armCodexCapacityWait(db, {
|
|
3517
|
+
accountId: input.accountId,
|
|
3518
|
+
workspaceId: input.workspaceId,
|
|
3519
|
+
sessionId: input.sessionId,
|
|
3520
|
+
turnId,
|
|
3521
|
+
attemptId: input.attemptId,
|
|
3522
|
+
workflowId: input.workflowId,
|
|
3523
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
3524
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
3525
|
+
earliestResetAt: authoritativeResetAt,
|
|
3526
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
3527
|
+
failurePayload,
|
|
3528
|
+
});
|
|
3529
|
+
if (armed.action === "waiting") {
|
|
3530
|
+
await publishDurableSessionEvents(
|
|
3531
|
+
bus,
|
|
3532
|
+
input.workspaceId,
|
|
3533
|
+
input.sessionId,
|
|
3534
|
+
armed.events,
|
|
3368
3535
|
);
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
3380
|
-
failurePayload,
|
|
3536
|
+
turnMetricOutcome = "recovering";
|
|
3537
|
+
activityStatus = "waiting_capacity";
|
|
3538
|
+
return claimedResult({
|
|
3539
|
+
status: "waiting_capacity",
|
|
3540
|
+
capacityWait: {
|
|
3541
|
+
waiterId: armed.waiter.id,
|
|
3542
|
+
generation: armed.waiter.generation,
|
|
3543
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
3544
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
3545
|
+
},
|
|
3381
3546
|
});
|
|
3382
|
-
if (armed.action === "waiting") {
|
|
3383
|
-
await publishDurableSessionEvents(
|
|
3384
|
-
bus,
|
|
3385
|
-
input.workspaceId,
|
|
3386
|
-
input.sessionId,
|
|
3387
|
-
armed.events,
|
|
3388
|
-
);
|
|
3389
|
-
turnMetricOutcome = "failed";
|
|
3390
|
-
activityStatus = "idle";
|
|
3391
|
-
return claimedResult({
|
|
3392
|
-
status: "idle",
|
|
3393
|
-
capacityWait: {
|
|
3394
|
-
waiterId: armed.waiter.id,
|
|
3395
|
-
generation: armed.waiter.generation,
|
|
3396
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
3397
|
-
wakeRevision: armed.waiter.wakeRevision,
|
|
3398
|
-
},
|
|
3399
|
-
});
|
|
3400
|
-
}
|
|
3401
3547
|
}
|
|
3402
3548
|
if (
|
|
3403
3549
|
!(await settle!({
|
|
@@ -3552,13 +3698,33 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3552
3698
|
openaiReasoningEffort: turn.reasoningEffort,
|
|
3553
3699
|
sandboxBackend: turn.sandboxBackend,
|
|
3554
3700
|
};
|
|
3701
|
+
const sessionMcpHeaderNames = new Map<string, readonly string[]>();
|
|
3555
3702
|
const runSettings = await settingsWithSessionMcpServersForRun(
|
|
3556
3703
|
db,
|
|
3557
3704
|
input.workspaceId,
|
|
3558
3705
|
input.sessionId,
|
|
3559
3706
|
input.attemptId,
|
|
3560
3707
|
baseRunSettings,
|
|
3708
|
+
{
|
|
3709
|
+
onResolvedServers: (servers) => {
|
|
3710
|
+
for (const server of servers) {
|
|
3711
|
+
sessionMcpHeaderNames.set(server.id, server.headerNames);
|
|
3712
|
+
}
|
|
3713
|
+
},
|
|
3714
|
+
},
|
|
3561
3715
|
);
|
|
3716
|
+
registerSecretRedactions([
|
|
3717
|
+
...(runSettings.accessKey
|
|
3718
|
+
? [{ name: "OPENGENI_ACCESS_KEY", value: runSettings.accessKey }]
|
|
3719
|
+
: []),
|
|
3720
|
+
...runSettings.mcpServers.flatMap((server) => {
|
|
3721
|
+
return headerSecretRedactions(
|
|
3722
|
+
`MCP_${server.id.toUpperCase()}_STATIC`,
|
|
3723
|
+
server.headers,
|
|
3724
|
+
sessionMcpHeaderNames.get(server.id),
|
|
3725
|
+
);
|
|
3726
|
+
}),
|
|
3727
|
+
]);
|
|
3562
3728
|
// Multi-provider per-turn routing → the provider gating (compaction mode,
|
|
3563
3729
|
// hosted web search, encrypted reasoning, context window) the agent and
|
|
3564
3730
|
// compaction summarizer must use; null falls back to the legacy global
|
|
@@ -3576,6 +3742,39 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3576
3742
|
capabilitySettings,
|
|
3577
3743
|
turnExecutionPolicy.productModelId,
|
|
3578
3744
|
);
|
|
3745
|
+
const selectedProvider = resolvedModel?.provider;
|
|
3746
|
+
const publicProviderHeaders = new Set(
|
|
3747
|
+
selectedProvider?.publicDefaultHeaderNames?.map((name) => name.toLowerCase()) ?? [],
|
|
3748
|
+
);
|
|
3749
|
+
const publicProviderQueries = new Set(
|
|
3750
|
+
selectedProvider?.publicDefaultQueryNames?.map((name) => name.toLowerCase()) ?? [],
|
|
3751
|
+
);
|
|
3752
|
+
const fallbackProviderApiKey =
|
|
3753
|
+
capabilitySettings.openaiProvider === "azure"
|
|
3754
|
+
? (capabilitySettings.azureOpenaiApiKey ?? capabilitySettings.azureOpenaiAdToken)
|
|
3755
|
+
: capabilitySettings.openaiApiKey;
|
|
3756
|
+
registerSecretRedactions([
|
|
3757
|
+
...(selectedProvider?.apiKey
|
|
3758
|
+
? [
|
|
3759
|
+
{
|
|
3760
|
+
name: `MODEL_${selectedProvider.id.toUpperCase()}_API_KEY`,
|
|
3761
|
+
value: selectedProvider.apiKey,
|
|
3762
|
+
},
|
|
3763
|
+
]
|
|
3764
|
+
: !resolvedModel && fallbackProviderApiKey
|
|
3765
|
+
? [{ name: "MODEL_PROVIDER_API_KEY", value: fallbackProviderApiKey }]
|
|
3766
|
+
: []),
|
|
3767
|
+
...Object.entries(selectedProvider?.defaultHeaders ?? {}).flatMap(([name, value]) =>
|
|
3768
|
+
publicProviderHeaders.has(name.toLowerCase())
|
|
3769
|
+
? []
|
|
3770
|
+
: [{ name: `MODEL_PROVIDER_HEADER_${name.toUpperCase()}`, value }],
|
|
3771
|
+
),
|
|
3772
|
+
...Object.entries(selectedProvider?.defaultQuery ?? {}).flatMap(([name, value]) =>
|
|
3773
|
+
publicProviderQueries.has(name.toLowerCase())
|
|
3774
|
+
? []
|
|
3775
|
+
: [{ name: `MODEL_PROVIDER_QUERY_${name.toUpperCase()}`, value }],
|
|
3776
|
+
),
|
|
3777
|
+
]);
|
|
3579
3778
|
// Bind the provider/model catalog's context policy to every model-facing
|
|
3580
3779
|
// path for this turn. In particular, Codex subscription turns must not
|
|
3581
3780
|
// inherit the deployment's OpenAI/Azure mode or 1.05M context defaults:
|
|
@@ -3630,6 +3829,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3630
3829
|
input.workspaceId,
|
|
3631
3830
|
effectiveCodexCredentialId ?? "",
|
|
3632
3831
|
);
|
|
3832
|
+
const registerSnapshot = async (
|
|
3833
|
+
snapshotPromise: ReturnType<typeof resolver.getToken>,
|
|
3834
|
+
) => {
|
|
3835
|
+
const snapshot = await snapshotPromise;
|
|
3836
|
+
registerSecretRedactions([
|
|
3837
|
+
{ name: "CODEX_ACCESS_TOKEN", value: snapshot.accessToken },
|
|
3838
|
+
]);
|
|
3839
|
+
return snapshot;
|
|
3840
|
+
};
|
|
3633
3841
|
return {
|
|
3634
3842
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
3635
3843
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -3639,8 +3847,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3639
3847
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
3640
3848
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
3641
3849
|
sessionId: input.sessionId,
|
|
3642
|
-
getToken: resolver.getToken,
|
|
3643
|
-
refresh: resolver.refresh,
|
|
3850
|
+
getToken: () => registerSnapshot(resolver.getToken()),
|
|
3851
|
+
refresh: () => registerSnapshot(resolver.refresh()),
|
|
3644
3852
|
resolveModel: buildModelResolver(
|
|
3645
3853
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
3646
3854
|
CODEX_FALLBACK_MODEL_SLUGS[0],
|
|
@@ -3783,7 +3991,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3783
3991
|
// recovery path re-dispatch this exact maintenance execution.
|
|
3784
3992
|
if (shouldRecoverCompactionProviderFailure(error)) throw error;
|
|
3785
3993
|
if (!isCompactionSummaryFailure(error)) throw error;
|
|
3786
|
-
const errorMessage = compactionFailureReasonFromError(error);
|
|
3994
|
+
const errorMessage = String(redact(compactionFailureReasonFromError(error)));
|
|
3787
3995
|
if (
|
|
3788
3996
|
!(await settle!({
|
|
3789
3997
|
events: [
|
|
@@ -3867,7 +4075,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3867
4075
|
// materialized allow-list. `withCodexAppsTool` below keeps the existing
|
|
3868
4076
|
// single Codex lazy router/connector overlay; it is not a second policy
|
|
3869
4077
|
// or discovery path.
|
|
3870
|
-
const
|
|
4078
|
+
const resolvedToolPolicy = resolveSessionToolPolicy({
|
|
3871
4079
|
...(session.toolPolicy ? { toolPolicy: session.toolPolicy } : {}),
|
|
3872
4080
|
sessionTools: session.tools,
|
|
3873
4081
|
turnTools: turn.tools,
|
|
@@ -3876,7 +4084,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3876
4084
|
defaultMcpServerIds: enabledCapabilityMcpToolRefs(settings, mcpSettings).map(
|
|
3877
4085
|
(tool) => tool.id,
|
|
3878
4086
|
),
|
|
3879
|
-
})
|
|
4087
|
+
});
|
|
4088
|
+
const effectivePolicyTools = resolvedToolPolicy.toolRefs;
|
|
4089
|
+
defaultNativeToolsAllowed = sessionToolPolicyAllowsDefaultNativeTools(
|
|
4090
|
+
resolvedToolPolicy.effectivePolicy,
|
|
4091
|
+
);
|
|
3880
4092
|
const turnTools = withCodexAppsTool(
|
|
3881
4093
|
runSettings,
|
|
3882
4094
|
withFirstPartyTools(runSettings, effectivePolicyTools),
|
|
@@ -4141,6 +4353,19 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4141
4353
|
cancellationSignal,
|
|
4142
4354
|
undefined,
|
|
4143
4355
|
);
|
|
4356
|
+
registerSecretRedactions([
|
|
4357
|
+
...Object.entries(sandboxGitTokens ?? {}).flatMap(([provider, value]) =>
|
|
4358
|
+
value ? [{ name: `GIT_${provider.toUpperCase()}_TOKEN`, value }] : [],
|
|
4359
|
+
),
|
|
4360
|
+
...(sandboxGitToken ? [{ name: "GIT_TOKEN", value: sandboxGitToken }] : []),
|
|
4361
|
+
...((sandboxGitCredentialBindings ?? []).map((binding) => ({
|
|
4362
|
+
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
4363
|
+
value: binding.token,
|
|
4364
|
+
})) satisfies SecretForRedaction[]),
|
|
4365
|
+
...(sandboxToolspaceToken
|
|
4366
|
+
? [{ name: "TOOLSPACE_TOKEN", value: sandboxToolspaceToken }]
|
|
4367
|
+
: []),
|
|
4368
|
+
]);
|
|
4144
4369
|
const sandboxToolspaceTokenFile = sandboxToolspaceToken
|
|
4145
4370
|
? toolspaceTokenFileFromEnvironment(sandboxEnvironment, input.sessionId)
|
|
4146
4371
|
: undefined;
|
|
@@ -4186,6 +4411,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4186
4411
|
);
|
|
4187
4412
|
if (binding) {
|
|
4188
4413
|
assertGitCredentialRenewalTransportUnchanged(initialBinding, binding);
|
|
4414
|
+
registerSecretRedactions([
|
|
4415
|
+
{
|
|
4416
|
+
name: `GIT_${binding.provider.toUpperCase()}_BINDING_TOKEN`,
|
|
4417
|
+
value: binding.token,
|
|
4418
|
+
},
|
|
4419
|
+
]);
|
|
4189
4420
|
}
|
|
4190
4421
|
pendingBinding = binding;
|
|
4191
4422
|
return binding
|
|
@@ -4266,8 +4497,17 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4266
4497
|
await previous?.stop();
|
|
4267
4498
|
if (toolspaceTokenRenewalClosed) return;
|
|
4268
4499
|
|
|
4269
|
-
const mint = async () =>
|
|
4270
|
-
await mintSandboxToolspaceToken(
|
|
4500
|
+
const mint = async () => {
|
|
4501
|
+
const material = await mintSandboxToolspaceToken(
|
|
4502
|
+
runSettings,
|
|
4503
|
+
connectionScope,
|
|
4504
|
+
toolspaceAuthority,
|
|
4505
|
+
);
|
|
4506
|
+
if (material) {
|
|
4507
|
+
registerSecretRedactions([{ name: "TOOLSPACE_TOKEN", value: material.token }]);
|
|
4508
|
+
}
|
|
4509
|
+
return material;
|
|
4510
|
+
};
|
|
4271
4511
|
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
4272
4512
|
const runAs = sandboxRunAs(runSettings);
|
|
4273
4513
|
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
@@ -4483,8 +4723,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4483
4723
|
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
4484
4724
|
// here makes current==target so the delta is empty.
|
|
4485
4725
|
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
4486
|
-
|
|
4487
|
-
|
|
4726
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
4727
|
+
machinePrimary,
|
|
4728
|
+
input.attemptId,
|
|
4729
|
+
session.sandboxGroupId,
|
|
4730
|
+
);
|
|
4731
|
+
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
4732
|
+
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
4488
4733
|
// STAGE D honest-label guard: a machine-home session carries
|
|
4489
4734
|
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
4490
4735
|
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
@@ -4497,13 +4742,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4497
4742
|
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
4498
4743
|
if (machinePrimary) {
|
|
4499
4744
|
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
4500
|
-
// establish
|
|
4501
|
-
//
|
|
4502
|
-
//
|
|
4503
|
-
//
|
|
4504
|
-
//
|
|
4505
|
-
// bind (no NATS round-trip), so build it FIRST; if the lease then fences,
|
|
4506
|
-
// there is nothing to clean up.
|
|
4745
|
+
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
4746
|
+
// the machine route's own epoch fence; the managed-home lease separately
|
|
4747
|
+
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
4748
|
+
// ownership domains lets an unrelated degraded cloud archive block a
|
|
4749
|
+
// healthy explicitly selected machine.
|
|
4507
4750
|
// Whether the machine's latest Hello advertised the op-stream engine
|
|
4508
4751
|
// (refreshed on every connect). Read only when the server flag is on —
|
|
4509
4752
|
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
@@ -4533,21 +4776,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4533
4776
|
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
4534
4777
|
machinePrimarySession =
|
|
4535
4778
|
established.session as import("@opengeni/runtime").SelfhostedSession;
|
|
4536
|
-
const lease = await waitForTurnOperation(
|
|
4537
|
-
acquireSelfhostedLeaseForTurn(
|
|
4538
|
-
{ db, settings },
|
|
4539
|
-
{
|
|
4540
|
-
accountId: input.accountId,
|
|
4541
|
-
workspaceId: input.workspaceId,
|
|
4542
|
-
sandboxGroupId: session.sandboxGroupId,
|
|
4543
|
-
sessionId: input.sessionId,
|
|
4544
|
-
},
|
|
4545
|
-
"turn",
|
|
4546
|
-
sandboxHolderId,
|
|
4547
|
-
),
|
|
4548
|
-
cancellationSignal,
|
|
4549
|
-
async (lateLease) => await lateLease.release(),
|
|
4550
|
-
);
|
|
4551
4779
|
setupBoxSession = established.session;
|
|
4552
4780
|
resolvedSandbox = {
|
|
4553
4781
|
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
@@ -4589,8 +4817,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4589
4817
|
},
|
|
4590
4818
|
established,
|
|
4591
4819
|
),
|
|
4592
|
-
|
|
4593
|
-
|
|
4820
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
4821
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
4822
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
4823
|
+
leaseEpoch: activeSandboxPointer!.activeEpoch,
|
|
4824
|
+
release: async () => undefined,
|
|
4594
4825
|
};
|
|
4595
4826
|
} else if (establishPolicy === "on-demand") {
|
|
4596
4827
|
// Lazy sandbox provisioning: holder/group ids are fixed at turn start,
|
|
@@ -4623,7 +4854,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4623
4854
|
// running a DIFFERENT image (solo → recreate on the new image; N-holders →
|
|
4624
4855
|
// SandboxImageConflictError surfaced as an actionable turn error). Prefer the
|
|
4625
4856
|
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
4626
|
-
// (establishSelfhostedTurnSession
|
|
4857
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
4627
4858
|
// an image — B3 lives only on this Modal else-branch.
|
|
4628
4859
|
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
4629
4860
|
? {
|
|
@@ -4637,7 +4868,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4637
4868
|
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
4638
4869
|
},
|
|
4639
4870
|
"turn",
|
|
4640
|
-
|
|
4871
|
+
managedOwnership!.holderId,
|
|
4641
4872
|
),
|
|
4642
4873
|
cancellationSignal,
|
|
4643
4874
|
async (lateSandbox) => await lateSandbox.release(),
|
|
@@ -4719,7 +4950,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4719
4950
|
// (initialize + tools/list) can resolve the per-workspace bearer from
|
|
4720
4951
|
// codexRequestStorage (runtime/codexAppsMcpRequestInit). withCodex is the
|
|
4721
4952
|
// identity on every non-codex turn, so this is a no-op for existing paths.
|
|
4722
|
-
const
|
|
4953
|
+
const rawResolveCredential = connectionTokenResolverForTurn({
|
|
4723
4954
|
db,
|
|
4724
4955
|
settings: runSettings,
|
|
4725
4956
|
connectionCredentials: connectionCredentials ?? null,
|
|
@@ -4730,6 +4961,15 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4730
4961
|
attemptId: input.attemptId,
|
|
4731
4962
|
turn,
|
|
4732
4963
|
});
|
|
4964
|
+
const resolveCredential: typeof rawResolveCredential = async (request) => {
|
|
4965
|
+
const result = await rawResolveCredential(request);
|
|
4966
|
+
if (result.status === "ok") {
|
|
4967
|
+
registerSecretRedactions(
|
|
4968
|
+
headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
|
|
4969
|
+
);
|
|
4970
|
+
}
|
|
4971
|
+
return result;
|
|
4972
|
+
};
|
|
4733
4973
|
preparedTools = await waitForTurnOperation(
|
|
4734
4974
|
withCodex(() =>
|
|
4735
4975
|
runtime.prepareTools(runSettings, turnTools, {
|
|
@@ -4795,6 +5035,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4795
5035
|
},
|
|
4796
5036
|
}
|
|
4797
5037
|
: {};
|
|
5038
|
+
const hostedWebSearch = hostedWebSearchForTurn(
|
|
5039
|
+
resolvedModel,
|
|
5040
|
+
runSettings.webSearchEnabled,
|
|
5041
|
+
defaultNativeToolsAllowed,
|
|
5042
|
+
);
|
|
4798
5043
|
const agent = runtime.buildAgent(modelRunSettings, turnResources, {
|
|
4799
5044
|
reasoningEffort: turn.reasoningEffort,
|
|
4800
5045
|
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
@@ -4854,11 +5099,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4854
5099
|
// store/compaction follow the provider's compaction mode (registry
|
|
4855
5100
|
// providers resolve to "client"); encrypted reasoning is only
|
|
4856
5101
|
// round-tripped on the Responses wire API; hosted web search is attached
|
|
4857
|
-
// only when the model opts in
|
|
4858
|
-
//
|
|
5102
|
+
// only when BOTH the model opts in and this immutable turn still tracks
|
|
5103
|
+
// workspace defaults (an explicit session/turn remains narrowed); the
|
|
5104
|
+
// effective context window drives the compaction threshold.
|
|
5105
|
+
hostedWebSearch,
|
|
4859
5106
|
...(resolvedModel
|
|
4860
5107
|
? {
|
|
4861
|
-
hostedWebSearch: resolvedModel.configured.hostedWebSearch,
|
|
4862
5108
|
encryptedReasoning:
|
|
4863
5109
|
resolvedModel.provider.api === "responses" &&
|
|
4864
5110
|
runSettings.openaiReasoningEncryptedContent,
|
|
@@ -5206,7 +5452,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5206
5452
|
} catch (compactError) {
|
|
5207
5453
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
5208
5454
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
5209
|
-
const errorMessage = compactionFailureReasonFromError(compactError);
|
|
5455
|
+
const errorMessage = String(redact(compactionFailureReasonFromError(compactError)));
|
|
5210
5456
|
observability.error("context compaction failed", {
|
|
5211
5457
|
sessionId: input.sessionId,
|
|
5212
5458
|
turnId,
|
|
@@ -5559,6 +5805,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5559
5805
|
contextCompactionSignal: () => modelResponseUsageContextSignal(modelResponseUsageState),
|
|
5560
5806
|
contextCompactionRequested: () =>
|
|
5561
5807
|
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
5808
|
+
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
5562
5809
|
...(toolCancellationFenceRef.current
|
|
5563
5810
|
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
5564
5811
|
: {}),
|
|
@@ -5641,7 +5888,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5641
5888
|
// conversation context preserved for the post-top-up resume.
|
|
5642
5889
|
let serializedRunState: string | null = null;
|
|
5643
5890
|
try {
|
|
5644
|
-
serializedRunState = stream.state.toString();
|
|
5891
|
+
serializedRunState = String(redact(stream.state.toString()));
|
|
5645
5892
|
} catch {
|
|
5646
5893
|
serializedRunState = null;
|
|
5647
5894
|
}
|
|
@@ -5661,7 +5908,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5661
5908
|
executionGeneration,
|
|
5662
5909
|
attemptId: input.attemptId,
|
|
5663
5910
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
5664
|
-
|
|
5911
|
+
callId: pendingToolCall.callId,
|
|
5912
|
+
callType: pendingToolCall.callType,
|
|
5913
|
+
callItem: redact(pendingToolCall.callItem) as Record<string, unknown>,
|
|
5665
5914
|
});
|
|
5666
5915
|
if (!registered.accepted) {
|
|
5667
5916
|
throw new TurnAttemptFencedError(
|
|
@@ -5686,7 +5935,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5686
5935
|
attemptId: input.attemptId,
|
|
5687
5936
|
callId: completedToolCall.callId,
|
|
5688
5937
|
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
5689
|
-
resultItem: completedToolCall.resultItem,
|
|
5938
|
+
resultItem: redact(completedToolCall.resultItem) as Record<string, unknown>,
|
|
5690
5939
|
});
|
|
5691
5940
|
if (!recorded.accepted) {
|
|
5692
5941
|
throw new TurnAttemptFencedError(
|
|
@@ -5911,7 +6160,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5911
6160
|
return claimedResult({ status: "requires_action" });
|
|
5912
6161
|
}
|
|
5913
6162
|
|
|
5914
|
-
const finalOutput = String(stream.finalOutput ?? "");
|
|
6163
|
+
const finalOutput = String(redact(String(stream.finalOutput ?? "")));
|
|
5915
6164
|
await reconcileConversationTruth();
|
|
5916
6165
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
5917
6166
|
// history store (a redispatch would NOT re-execute them), so this
|
|
@@ -5992,8 +6241,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5992
6241
|
sessionId: input.sessionId,
|
|
5993
6242
|
turnId: activeTurnId,
|
|
5994
6243
|
reason: recoveryKind,
|
|
5995
|
-
code: overflow?.code,
|
|
5996
|
-
error: overflow?.message ?? compactionNeeded?.message,
|
|
6244
|
+
code: overflow?.code ? String(redact(overflow.code)) : undefined,
|
|
6245
|
+
error: String(redact(overflow?.message ?? compactionNeeded?.message ?? "")),
|
|
5997
6246
|
signalTokens: compactionNeeded?.signalTokens,
|
|
5998
6247
|
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
5999
6248
|
});
|
|
@@ -6020,7 +6269,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6020
6269
|
// empty summary and must not create a new goal continuation.
|
|
6021
6270
|
if (shouldRecoverCompactionProviderFailure(compactError)) throw compactError;
|
|
6022
6271
|
if (!isCompactionSummaryFailure(compactError)) throw compactError;
|
|
6023
|
-
compactionFailureMessage =
|
|
6272
|
+
compactionFailureMessage = String(
|
|
6273
|
+
redact(compactionFailureReasonFromError(compactError)),
|
|
6274
|
+
);
|
|
6024
6275
|
observability.warn("context compaction recovery compaction failed", {
|
|
6025
6276
|
sessionId: input.sessionId,
|
|
6026
6277
|
turnId: activeTurnId,
|
|
@@ -6126,7 +6377,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6126
6377
|
turnMetricOutcome = "recovering";
|
|
6127
6378
|
return claimedResult({ status: "recovering" });
|
|
6128
6379
|
} catch (recoveryError) {
|
|
6129
|
-
console.error(
|
|
6380
|
+
console.error(
|
|
6381
|
+
"sandbox lease supersession recovery failed",
|
|
6382
|
+
safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
|
|
6383
|
+
);
|
|
6130
6384
|
throw recoveryError;
|
|
6131
6385
|
}
|
|
6132
6386
|
}
|
|
@@ -6172,7 +6426,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6172
6426
|
// The database transition is atomic. If it could not commit, surface
|
|
6173
6427
|
// the failure so Temporal can retry on a healthy worker; never mutate
|
|
6174
6428
|
// the turn through a second cancellation path.
|
|
6175
|
-
console.error(
|
|
6429
|
+
console.error(
|
|
6430
|
+
"worker-shutdown recovery checkpoint failed",
|
|
6431
|
+
safeErrorDiagnostic(recoveryError, (value) => String(redact(value))),
|
|
6432
|
+
);
|
|
6176
6433
|
throw recoveryError;
|
|
6177
6434
|
}
|
|
6178
6435
|
}
|
|
@@ -6729,7 +6986,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6729
6986
|
// same-policy continuation path. When no alternate exists (all capped,
|
|
6730
6987
|
// or a single non-rotating account), persist the native capacity wait
|
|
6731
6988
|
// instead of an in-memory delay/user-message recovery.
|
|
6732
|
-
if (
|
|
6989
|
+
if (rotationResumeMs === null) {
|
|
6733
6990
|
const providerResetAt =
|
|
6734
6991
|
capacityAuthoritativeResetAt ??
|
|
6735
6992
|
(usageLimit.resetsInSeconds !== null &&
|
|
@@ -6744,8 +7001,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6744
7001
|
turnId,
|
|
6745
7002
|
attemptId: input.attemptId,
|
|
6746
7003
|
workflowId: input.workflowId,
|
|
6747
|
-
goalId: goal.id,
|
|
6748
|
-
goalVersion: goal.version,
|
|
7004
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
7005
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
6749
7006
|
earliestResetAt: providerResetAt,
|
|
6750
7007
|
resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
|
|
6751
7008
|
failurePayload,
|
|
@@ -6766,11 +7023,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6766
7023
|
input.sessionId,
|
|
6767
7024
|
armed.events,
|
|
6768
7025
|
);
|
|
6769
|
-
turnMetricOutcome = "
|
|
6770
|
-
activityStatus = "
|
|
7026
|
+
turnMetricOutcome = "recovering";
|
|
7027
|
+
activityStatus = "waiting_capacity";
|
|
6771
7028
|
activityError = error;
|
|
6772
7029
|
return claimedResult({
|
|
6773
|
-
status: "
|
|
7030
|
+
status: "waiting_capacity",
|
|
6774
7031
|
capacityWait: {
|
|
6775
7032
|
waiterId: armed.waiter.id,
|
|
6776
7033
|
generation: armed.waiter.generation,
|
|
@@ -6893,7 +7150,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6893
7150
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
6894
7151
|
// it after a pacing delay. This is independent of goal state and never
|
|
6895
7152
|
// relies on a synthetic continuation prompt.
|
|
6896
|
-
const failure = agentRunFailurePayload(error, { isCodexTurn })
|
|
7153
|
+
const failure = redact(agentRunFailurePayload(error, { isCodexTurn })) as ReturnType<
|
|
7154
|
+
typeof agentRunFailurePayload
|
|
7155
|
+
>;
|
|
6897
7156
|
if (isSessionEventPersistenceError(error)) {
|
|
6898
7157
|
// Never pass the original Drizzle/postgres-js error to telemetry: its
|
|
6899
7158
|
// nested cause may contain raw SQL and bound parameters. The typed DB
|
|
@@ -7031,8 +7290,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7031
7290
|
);
|
|
7032
7291
|
}
|
|
7033
7292
|
await drainAttemptOwnedSandboxWriters({
|
|
7034
|
-
|
|
7035
|
-
|
|
7293
|
+
// Normal turn completion owns the same process boundary as
|
|
7294
|
+
// Pause/Steer: yielded provider shells must be terminated, polled,
|
|
7295
|
+
// and durably settled before workspace capture. Only receipt
|
|
7296
|
+
// publication remains conditional on acknowledgeQuiescence.
|
|
7297
|
+
toolCancellationFence,
|
|
7298
|
+
cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
|
|
7036
7299
|
gitCredentialRenewals: gitRenewalsToStop,
|
|
7037
7300
|
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
7038
7301
|
runCredentialRenewal: runRenewalToStop,
|
|
@@ -7090,14 +7353,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7090
7353
|
});
|
|
7091
7354
|
},
|
|
7092
7355
|
onReceiptFailure: (error) => {
|
|
7093
|
-
console.error(
|
|
7356
|
+
console.error(
|
|
7357
|
+
"agent turn quiescence receipt exhausted; signalling proof",
|
|
7358
|
+
safeErrorDiagnostic(error, (value) => String(redact(value))),
|
|
7359
|
+
);
|
|
7094
7360
|
},
|
|
7095
7361
|
onPublishFailure: (error) => {
|
|
7096
|
-
console.error(
|
|
7362
|
+
console.error(
|
|
7363
|
+
"agent turn quiescence event fanout failed",
|
|
7364
|
+
safeErrorDiagnostic(error, (value) => String(redact(value))),
|
|
7365
|
+
);
|
|
7097
7366
|
},
|
|
7098
7367
|
onSignalFailure: (error, attempt, retryMs) => {
|
|
7099
7368
|
console.error("agent turn quiescence proof signal failed; retrying", {
|
|
7100
|
-
error,
|
|
7369
|
+
error: safeErrorDiagnostic(error, (value) => String(redact(value))),
|
|
7101
7370
|
attempt,
|
|
7102
7371
|
retryMs,
|
|
7103
7372
|
});
|
|
@@ -7348,14 +7617,20 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7348
7617
|
resolvedSandbox = null; // drop ownership now; the exact-holder release may finish later
|
|
7349
7618
|
await waitForTurnFinalizerStep(
|
|
7350
7619
|
sandboxToRelease.release().catch((releaseError) => {
|
|
7351
|
-
console.error(
|
|
7620
|
+
console.error(
|
|
7621
|
+
"sandbox lease release failed (turn outcome unaffected)",
|
|
7622
|
+
safeErrorDiagnostic(releaseError, (value) => String(redact(value))),
|
|
7623
|
+
);
|
|
7352
7624
|
}),
|
|
7353
7625
|
finalizerSignal,
|
|
7354
7626
|
);
|
|
7355
7627
|
}
|
|
7356
7628
|
} catch (error) {
|
|
7357
7629
|
finalizationError ??= error;
|
|
7358
|
-
console.error(
|
|
7630
|
+
console.error(
|
|
7631
|
+
"agent turn finalization failed (turn outcome unaffected)",
|
|
7632
|
+
safeErrorDiagnostic(error, (value) => String(redact(value))),
|
|
7633
|
+
);
|
|
7359
7634
|
} finally {
|
|
7360
7635
|
cancellationSignal?.removeEventListener("abort", noteCancellationRequested);
|
|
7361
7636
|
const completedAt = performance.now();
|
|
@@ -7405,7 +7680,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7405
7680
|
"opengeni.duration_ms": Math.round(durationSeconds * 1000),
|
|
7406
7681
|
"opengeni.finalization_duration_ms": Math.round(finalizationDurationSeconds * 1000),
|
|
7407
7682
|
},
|
|
7408
|
-
error:
|
|
7683
|
+
error:
|
|
7684
|
+
finalizationError || activityError
|
|
7685
|
+
? safeErrorForTelemetry(finalizationError ?? activityError, (value) =>
|
|
7686
|
+
String(redact(value)),
|
|
7687
|
+
)
|
|
7688
|
+
: undefined,
|
|
7409
7689
|
});
|
|
7410
7690
|
assertPhysicalToolQuiescenceForCancellation({
|
|
7411
7691
|
acknowledgeQuiescence,
|
|
@@ -7480,6 +7760,13 @@ async function assertGitHubTokenMintSelectionAuthorized(
|
|
|
7480
7760
|
* that will NOT clear on retry) is classified and returned BEFORE this in
|
|
7481
7761
|
* agentRunFailurePayload, so it never reaches here.
|
|
7482
7762
|
*/
|
|
7763
|
+
const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
|
|
7764
|
+
"unable to connect. is the computer able to access the url?";
|
|
7765
|
+
|
|
7766
|
+
function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
|
|
7767
|
+
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
7768
|
+
}
|
|
7769
|
+
|
|
7483
7770
|
export function isTransientProviderError(error: unknown): boolean {
|
|
7484
7771
|
const status =
|
|
7485
7772
|
typeof error === "object" && error !== null && "status" in error
|
|
@@ -7501,6 +7788,9 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
7501
7788
|
return true;
|
|
7502
7789
|
}
|
|
7503
7790
|
const message = error instanceof Error ? error.message : String(error);
|
|
7791
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
7792
|
+
return true;
|
|
7793
|
+
}
|
|
7504
7794
|
return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
|
|
7505
7795
|
message,
|
|
7506
7796
|
);
|
|
@@ -7627,6 +7917,14 @@ export function agentRunFailurePayload(
|
|
|
7627
7917
|
// retryable so a goal-bearing session idles and auto-continues instead of going
|
|
7628
7918
|
// terminal on a provider's bad minute. See isTransientProviderError.
|
|
7629
7919
|
if (isTransientProviderError(error)) {
|
|
7920
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
7921
|
+
return {
|
|
7922
|
+
error:
|
|
7923
|
+
"OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
7924
|
+
code: "upstream_connectivity_unavailable",
|
|
7925
|
+
retryable: true,
|
|
7926
|
+
};
|
|
7927
|
+
}
|
|
7630
7928
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
7631
7929
|
}
|
|
7632
7930
|
return { error: message };
|