@opengeni/worker-bundle 0.10.8 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1484 -481
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +5 -4
- package/package.json +10 -10
- package/src/activities/agent-turn.ts +484 -169
- package/src/activities/codex-fleet-shadow.ts +296 -0
- package/src/activities/goals.ts +13 -79
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +186 -13
- package/src/activities/sandbox-lease.ts +130 -83
- package/src/activities/scheduled-tasks.ts +42 -27
- package/src/sandbox-resume.ts +249 -94
- package/src/sandbox-routing.ts +597 -2
- package/src/workflows/activities.ts +4 -3
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
+
advanceWorkspaceGeneration,
|
|
3
|
+
verifyWorkspaceMutationSettlement,
|
|
2
4
|
applySessionTurnSettlement,
|
|
3
5
|
requestSessionTurnRecovery,
|
|
4
6
|
claimSessionWorkForAttempt,
|
|
@@ -101,6 +103,7 @@ import {
|
|
|
101
103
|
sandboxFileDownloadFailureNote,
|
|
102
104
|
SUMMARY_BUFFER_TOKENS,
|
|
103
105
|
runOwnedSandboxSetup,
|
|
106
|
+
RoutingMutationOutcomeUnknownError,
|
|
104
107
|
swapTargetEstablishability,
|
|
105
108
|
type SandboxFileDownload,
|
|
106
109
|
type SandboxFileDownloadFailure,
|
|
@@ -154,6 +157,11 @@ import {
|
|
|
154
157
|
type CodexRotationStrategy,
|
|
155
158
|
type RotationDecision,
|
|
156
159
|
} from "./codex-rotation";
|
|
160
|
+
import {
|
|
161
|
+
codexFleetShadowDecisionMetricLabelsV1,
|
|
162
|
+
codexFleetShadowErrorMetricLabelsV1,
|
|
163
|
+
publishCodexFleetShadowDecisionV1,
|
|
164
|
+
} from "./codex-fleet-shadow";
|
|
157
165
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
158
166
|
import { buildCodexTokenResolver } from "./codex-auth";
|
|
159
167
|
import {
|
|
@@ -526,6 +534,7 @@ export async function persistOrSignalSessionAttemptQuiescence(input: {
|
|
|
526
534
|
export async function drainAttemptOwnedSandboxWriters(input: {
|
|
527
535
|
toolCancellationFence: Pick<TurnToolCancellationFence, "cancel" | "waitForQuiescence"> | null;
|
|
528
536
|
cancellationReason?: unknown;
|
|
537
|
+
gitCredentialRenewals: readonly Pick<GitCredentialRenewalController, "stop">[];
|
|
529
538
|
toolspaceTokenRenewal: Pick<ToolspaceTokenRenewalController, "stop"> | null;
|
|
530
539
|
runCredentialRenewal: Pick<RunCredentialRenewalController, "stop"> | null;
|
|
531
540
|
}): Promise<void> {
|
|
@@ -535,6 +544,7 @@ export async function drainAttemptOwnedSandboxWriters(input: {
|
|
|
535
544
|
);
|
|
536
545
|
await input.toolCancellationFence.waitForQuiescence();
|
|
537
546
|
}
|
|
547
|
+
await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
|
|
538
548
|
await input.toolspaceTokenRenewal?.stop();
|
|
539
549
|
await input.runCredentialRenewal?.stop();
|
|
540
550
|
}
|
|
@@ -1533,6 +1543,19 @@ export function computerToolModeForTurn(
|
|
|
1533
1543
|
return "hosted";
|
|
1534
1544
|
}
|
|
1535
1545
|
|
|
1546
|
+
/**
|
|
1547
|
+
* Decide whether this turn's provider accepts typed file/image content in a
|
|
1548
|
+
* user message. The legacy built-in client and registry Responses clients use
|
|
1549
|
+
* the Agents SDK Responses converter, which supports `input_image` and
|
|
1550
|
+
* `input_file`. Chat-completions providers retain the sandbox-path projection
|
|
1551
|
+
* until that transport declares and proves an equivalent capability.
|
|
1552
|
+
*/
|
|
1553
|
+
export function modelAcceptsTypedAttachmentContentForTurn(
|
|
1554
|
+
resolvedModel: { provider: { api: ModelProviderApi } } | null,
|
|
1555
|
+
): boolean {
|
|
1556
|
+
return resolvedModel === null || resolvedModel.provider.api === "responses";
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1536
1559
|
export type TurnSandboxProvisioner<T> = {
|
|
1537
1560
|
get(): Promise<T>;
|
|
1538
1561
|
hasStarted(): boolean;
|
|
@@ -2023,6 +2046,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2023
2046
|
// run. null when the flag is off (byte-for-byte the legacy build-and-discard
|
|
2024
2047
|
// path) OR when the backend is "none". Released + dropped in `finally`.
|
|
2025
2048
|
let resolvedSandbox: ResumedTurnSandbox | null = null;
|
|
2049
|
+
const requireResolvedSandboxForMutation = (message: string): ResumedTurnSandbox => {
|
|
2050
|
+
if (!resolvedSandbox) throw new Error(message);
|
|
2051
|
+
return resolvedSandbox;
|
|
2052
|
+
};
|
|
2026
2053
|
// The machine-primary SelfhostedSession (the UNWRAPPED backend, not the
|
|
2027
2054
|
// routing proxy): held so the turn's completion can final-ack this turn's
|
|
2028
2055
|
// settled op-stream ops AFTER the results are durably persisted.
|
|
@@ -2034,11 +2061,102 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2034
2061
|
// THIS handle so a mid-turn sandbox_swap can never re-route those execs onto a
|
|
2035
2062
|
// connected machine (the user's real computer).
|
|
2036
2063
|
let setupBoxSession: unknown = null;
|
|
2064
|
+
// A same-target API repair can replace the home provider while this turn is
|
|
2065
|
+
// alive. Keep setup/snapshot persistence on the rebound raw session while
|
|
2066
|
+
// preserving the SDK-owned routing proxy for eager turns. Lazy turns hold the
|
|
2067
|
+
// proxy separately, so their worker-side handle may replace its raw session.
|
|
2068
|
+
const onHomeSandboxRebound = (rebound: {
|
|
2069
|
+
established: EstablishedSandboxSession;
|
|
2070
|
+
leaseEpoch: number;
|
|
2071
|
+
}): void => {
|
|
2072
|
+
const current = resolvedSandbox;
|
|
2073
|
+
const previousSession = current?.established.session;
|
|
2074
|
+
const preserveRoutingProxy = current !== null && previousSession !== setupBoxSession;
|
|
2075
|
+
setupBoxSession = rebound.established.session;
|
|
2076
|
+
if (!current) return;
|
|
2077
|
+
current.leaseEpoch = rebound.leaseEpoch;
|
|
2078
|
+
current.established = preserveRoutingProxy
|
|
2079
|
+
? {
|
|
2080
|
+
...current.established,
|
|
2081
|
+
client: rebound.established.client,
|
|
2082
|
+
// Keep the stable SDK-facing proxy; only its resolver changes the
|
|
2083
|
+
// underlying backend. The worker's setupBoxSession above is raw.
|
|
2084
|
+
session: previousSession,
|
|
2085
|
+
sessionState: rebound.established.sessionState,
|
|
2086
|
+
instanceId: rebound.established.instanceId,
|
|
2087
|
+
backendId: rebound.established.backendId,
|
|
2088
|
+
...(rebound.established.origin ? { origin: rebound.established.origin } : {}),
|
|
2089
|
+
...(rebound.established.restoredArchive
|
|
2090
|
+
? { restoredArchive: rebound.established.restoredArchive }
|
|
2091
|
+
: {}),
|
|
2092
|
+
}
|
|
2093
|
+
: rebound.established;
|
|
2094
|
+
};
|
|
2037
2095
|
// The globally unique durable turn-attempt holder id + the group id,
|
|
2038
2096
|
// captured so the lease heartbeat can refresh the lease TTL epoch-fenced
|
|
2039
2097
|
// (a superseded owner self-evicts) and finally can release.
|
|
2040
2098
|
let sandboxHolderId: TurnSandboxLeaseHolderId | null = null;
|
|
2041
2099
|
let sandboxGroupId: string | null = null;
|
|
2100
|
+
const runWorkspaceMutationForSandbox = async <T>(
|
|
2101
|
+
sandbox: ResumedTurnSandbox,
|
|
2102
|
+
operation: string,
|
|
2103
|
+
mutation: () => Promise<T>,
|
|
2104
|
+
): Promise<T> => {
|
|
2105
|
+
// Connected machines are the user's own persistence and never dirty the
|
|
2106
|
+
// cloud home archive. Every persistable raw-session write batch is fenced
|
|
2107
|
+
// against the exact current lease/provider before the provider sees it.
|
|
2108
|
+
if (sandbox.established.backendId === "selfhosted") return await mutation();
|
|
2109
|
+
if (!sandboxGroupId || !sandboxHolderId || !turnId || executionGeneration <= 0) {
|
|
2110
|
+
throw new Error("Workspace mutation attempted before exact turn sandbox admission");
|
|
2111
|
+
}
|
|
2112
|
+
const identity = {
|
|
2113
|
+
accountId: input.accountId,
|
|
2114
|
+
workspaceId: input.workspaceId,
|
|
2115
|
+
sessionId: input.sessionId,
|
|
2116
|
+
turnId,
|
|
2117
|
+
executionGeneration,
|
|
2118
|
+
attemptId: input.attemptId,
|
|
2119
|
+
holderId: sandboxHolderId,
|
|
2120
|
+
sandboxGroupId,
|
|
2121
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
2122
|
+
expectedInstanceId: sandbox.established.instanceId,
|
|
2123
|
+
operation,
|
|
2124
|
+
};
|
|
2125
|
+
const admission = await advanceWorkspaceGeneration(db, identity);
|
|
2126
|
+
let result: T;
|
|
2127
|
+
try {
|
|
2128
|
+
result = await mutation();
|
|
2129
|
+
} catch (providerError) {
|
|
2130
|
+
try {
|
|
2131
|
+
await verifyWorkspaceMutationSettlement(db, {
|
|
2132
|
+
...identity,
|
|
2133
|
+
admission,
|
|
2134
|
+
outcome: "rejected",
|
|
2135
|
+
});
|
|
2136
|
+
} catch (settlementError) {
|
|
2137
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
2138
|
+
operation,
|
|
2139
|
+
`Platform workspace mutation "${operation}" rejected at the provider but lost its durable physical settlement; its outcome is unknown and it was not replayed`,
|
|
2140
|
+
{ cause: settlementError },
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
throw providerError;
|
|
2144
|
+
}
|
|
2145
|
+
try {
|
|
2146
|
+
await verifyWorkspaceMutationSettlement(db, {
|
|
2147
|
+
...identity,
|
|
2148
|
+
admission,
|
|
2149
|
+
outcome: "resolved",
|
|
2150
|
+
});
|
|
2151
|
+
} catch (settlementError) {
|
|
2152
|
+
throw new RoutingMutationOutcomeUnknownError(
|
|
2153
|
+
operation,
|
|
2154
|
+
`Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,
|
|
2155
|
+
{ cause: settlementError },
|
|
2156
|
+
);
|
|
2157
|
+
}
|
|
2158
|
+
return result;
|
|
2159
|
+
};
|
|
2042
2160
|
// Lease-TTL refresh timer (parallels the activity heartbeat): while the turn
|
|
2043
2161
|
// runs it refreshes expires_at epoch-fenced so a legit multi-day turn is
|
|
2044
2162
|
// never TTL-reaped. Cleared in finally. Only set when the flag resolved a box.
|
|
@@ -2994,6 +3112,80 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2994
3112
|
codexLeaseConfirmedUntilMs !== null;
|
|
2995
3113
|
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
2996
3114
|
|
|
3115
|
+
const actualOutcome = effectiveCodexCredentialId
|
|
3116
|
+
? "selected"
|
|
3117
|
+
: rotationDecision.kind === "allCapped"
|
|
3118
|
+
? "waiting"
|
|
3119
|
+
: "none";
|
|
3120
|
+
const actualReason = effectiveCodexCredentialId
|
|
3121
|
+
? leased.reused
|
|
3122
|
+
? "lease_reused"
|
|
3123
|
+
: sessionPin === effectiveCodexCredentialId
|
|
3124
|
+
? "pin"
|
|
3125
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
3126
|
+
? "rotation"
|
|
3127
|
+
: "active"
|
|
3128
|
+
: rotationDecision.kind === "allCapped"
|
|
3129
|
+
? "all_capped"
|
|
3130
|
+
: "none";
|
|
3131
|
+
const fencedInFlight =
|
|
3132
|
+
leased.reused ||
|
|
3133
|
+
(continuationCodexCredentialId !== null &&
|
|
3134
|
+
continuationCodexCredentialId === effectiveCodexCredentialId);
|
|
3135
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
3136
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
3137
|
+
decision: {
|
|
3138
|
+
accounts: leased.accounts,
|
|
3139
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
3140
|
+
actualOutcome,
|
|
3141
|
+
actualReason,
|
|
3142
|
+
affinityCredentialId: fencedInFlight
|
|
3143
|
+
? effectiveCodexCredentialId
|
|
3144
|
+
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
3145
|
+
fencedInFlight,
|
|
3146
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
3147
|
+
now: new Date(),
|
|
3148
|
+
aliasSeed: randomUUID(),
|
|
3149
|
+
},
|
|
3150
|
+
publish,
|
|
3151
|
+
});
|
|
3152
|
+
if (shadowResult.outcome === "published") {
|
|
3153
|
+
const shadowPayload = shadowResult.payload;
|
|
3154
|
+
observability.incrementCounter({
|
|
3155
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
3156
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
3157
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
3158
|
+
});
|
|
3159
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
3160
|
+
workspaceId: input.workspaceId,
|
|
3161
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
3162
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
3163
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
3164
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
3165
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
3166
|
+
comparison: shadowPayload.comparison,
|
|
3167
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
3168
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
3169
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
3170
|
+
});
|
|
3171
|
+
} else if (shadowResult.outcome === "failed") {
|
|
3172
|
+
// Shadow observability is explicitly non-authoritative. A malformed
|
|
3173
|
+
// snapshot or event-write fault must never change the authoritative lease,
|
|
3174
|
+
// capacity wait, failover, or the account serving this fenced turn.
|
|
3175
|
+
observability.incrementCounter({
|
|
3176
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
3177
|
+
help: "Shadow decision build/publication failures.",
|
|
3178
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
3179
|
+
});
|
|
3180
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
3181
|
+
workspaceId: input.workspaceId,
|
|
3182
|
+
stage: shadowResult.stage,
|
|
3183
|
+
reason: shadowResult.reason,
|
|
3184
|
+
errorName: shadowResult.errorName,
|
|
3185
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
3186
|
+
});
|
|
3187
|
+
}
|
|
3188
|
+
|
|
2997
3189
|
const eligibleCount = leased.accounts.filter((account) =>
|
|
2998
3190
|
isCodexCredentialEligible(account, new Date()),
|
|
2999
3191
|
).length;
|
|
@@ -3964,6 +4156,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
3964
4156
|
const attachGitCredentialRenewal = async (
|
|
3965
4157
|
tokenSession: GitCredentialTokenWriterSession,
|
|
3966
4158
|
initial: MintedRunGitCredentials | undefined,
|
|
4159
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
3967
4160
|
): Promise<void> => {
|
|
3968
4161
|
if (!initial || initial.bindings.length === 0) return;
|
|
3969
4162
|
const previous = gitCredentialRenewals;
|
|
@@ -4008,16 +4201,25 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4008
4201
|
throw new Error("credential renewal produced no binding token");
|
|
4009
4202
|
}
|
|
4010
4203
|
const runAs = sandboxRunAs(runSettings);
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
|
|
4204
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
4205
|
+
if (!targetSandbox) {
|
|
4206
|
+
throw new Error("Git credential renewal has no exact sandbox lease target");
|
|
4207
|
+
}
|
|
4208
|
+
await runWorkspaceMutationForSandbox(
|
|
4209
|
+
targetSandbox,
|
|
4210
|
+
"gitCredentialRenewal",
|
|
4211
|
+
async () =>
|
|
4212
|
+
await refreshGitCredentialBindingTokenFiles(tokenSession, [pendingBinding!], {
|
|
4213
|
+
...(runAs ? { runAs } : {}),
|
|
4214
|
+
...(toolCancellationFenceRef.current
|
|
4215
|
+
? {
|
|
4216
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4217
|
+
toolCancellationFenceRef.current,
|
|
4218
|
+
),
|
|
4219
|
+
}
|
|
4220
|
+
: {}),
|
|
4221
|
+
}),
|
|
4222
|
+
);
|
|
4021
4223
|
},
|
|
4022
4224
|
onSuccess: ({ providers: renewedProviders }) => {
|
|
4023
4225
|
for (const provider of renewedProviders) {
|
|
@@ -4056,6 +4258,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4056
4258
|
const attachToolspaceTokenRenewal = async (
|
|
4057
4259
|
tokenSession: ToolspaceTokenWriterSession,
|
|
4058
4260
|
initialExpiresAt = sandboxToolspaceTokenExpiresAt,
|
|
4261
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
4059
4262
|
): Promise<void> => {
|
|
4060
4263
|
if (!sandboxToolspaceToken || !initialExpiresAt) return;
|
|
4061
4264
|
const previous = toolspaceTokenRenewal;
|
|
@@ -4067,22 +4270,31 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4067
4270
|
await mintSandboxToolspaceToken(runSettings, connectionScope, toolspaceAuthority);
|
|
4068
4271
|
const write = async (material: NonNullable<Awaited<ReturnType<typeof mint>>>) => {
|
|
4069
4272
|
const runAs = sandboxRunAs(runSettings);
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4273
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
4274
|
+
if (!targetSandbox) {
|
|
4275
|
+
throw new Error("Toolspace token renewal has no exact sandbox lease target");
|
|
4276
|
+
}
|
|
4277
|
+
await runWorkspaceMutationForSandbox(
|
|
4278
|
+
targetSandbox,
|
|
4279
|
+
"toolspaceTokenRenewal",
|
|
4280
|
+
async () =>
|
|
4281
|
+
await refreshToolspaceTokenFile(tokenSession, material.token, {
|
|
4282
|
+
...(runAs ? { runAs } : {}),
|
|
4283
|
+
...(sandboxToolspaceTokenFile
|
|
4284
|
+
? {
|
|
4285
|
+
tokenFile: sandboxToolspaceTokenFile,
|
|
4286
|
+
legacyTokenFile: sandboxEnvironment.OPENGENI_TOOLSPACE_TOKEN_FILE!,
|
|
4287
|
+
}
|
|
4288
|
+
: {}),
|
|
4289
|
+
...(toolCancellationFenceRef.current
|
|
4290
|
+
? {
|
|
4291
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4292
|
+
toolCancellationFenceRef.current,
|
|
4293
|
+
),
|
|
4294
|
+
}
|
|
4295
|
+
: {}),
|
|
4296
|
+
}),
|
|
4297
|
+
);
|
|
4086
4298
|
};
|
|
4087
4299
|
let renewalExpiresAt = initialExpiresAt;
|
|
4088
4300
|
if (renewalExpiresAt.getTime() <= Date.now() + TOOLSPACE_TOKEN_EXPIRY_LEAD_MS) {
|
|
@@ -4127,6 +4339,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4127
4339
|
|
|
4128
4340
|
const attachRunCredentialRenewal = async (
|
|
4129
4341
|
credentialSession: RunCredentialCommandSession,
|
|
4342
|
+
initialSandbox?: ResumedTurnSandbox,
|
|
4130
4343
|
): Promise<void> => {
|
|
4131
4344
|
if (!runCredentialResolver) return;
|
|
4132
4345
|
const previous = runCredentialRenewal;
|
|
@@ -4135,15 +4348,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4135
4348
|
if (runCredentialRenewalClosed) return;
|
|
4136
4349
|
runCredentialSession = credentialSession;
|
|
4137
4350
|
|
|
4351
|
+
const requireTargetSandbox = (): ResumedTurnSandbox => {
|
|
4352
|
+
const targetSandbox = resolvedSandbox ?? initialSandbox;
|
|
4353
|
+
if (!targetSandbox) {
|
|
4354
|
+
throw new Error("Run credential mutation has no exact sandbox lease target");
|
|
4355
|
+
}
|
|
4356
|
+
return targetSandbox;
|
|
4357
|
+
};
|
|
4358
|
+
|
|
4138
4359
|
if (!initialRunCredentialMaterial) {
|
|
4139
|
-
await
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4360
|
+
await runWorkspaceMutationForSandbox(
|
|
4361
|
+
requireTargetSandbox(),
|
|
4362
|
+
"runCredentialClear",
|
|
4363
|
+
async () =>
|
|
4364
|
+
await clearRunCredentials(
|
|
4365
|
+
credentialSession,
|
|
4366
|
+
input.sessionId,
|
|
4367
|
+
toolCancellationFenceRef.current
|
|
4368
|
+
? toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4369
|
+
toolCancellationFenceRef.current,
|
|
4370
|
+
)
|
|
4371
|
+
: undefined,
|
|
4372
|
+
),
|
|
4147
4373
|
);
|
|
4148
4374
|
return;
|
|
4149
4375
|
}
|
|
@@ -4153,33 +4379,43 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4153
4379
|
pruneOtherAttempts = false,
|
|
4154
4380
|
): Promise<void> => {
|
|
4155
4381
|
if (!material) {
|
|
4156
|
-
await
|
|
4157
|
-
|
|
4158
|
-
|
|
4159
|
-
|
|
4160
|
-
|
|
4382
|
+
await runWorkspaceMutationForSandbox(
|
|
4383
|
+
requireTargetSandbox(),
|
|
4384
|
+
"runCredentialAttemptClear",
|
|
4385
|
+
async () =>
|
|
4386
|
+
await clearRunCredentialsForAttempt(credentialSession, {
|
|
4387
|
+
sessionId: input.sessionId,
|
|
4388
|
+
attemptId: input.attemptId,
|
|
4389
|
+
executionGeneration,
|
|
4390
|
+
}),
|
|
4391
|
+
);
|
|
4161
4392
|
return;
|
|
4162
4393
|
}
|
|
4163
4394
|
registerSecretRedactions(material.redactions);
|
|
4164
|
-
await
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4395
|
+
await runWorkspaceMutationForSandbox(
|
|
4396
|
+
requireTargetSandbox(),
|
|
4397
|
+
"runCredentialMaterialization",
|
|
4398
|
+
async () =>
|
|
4399
|
+
await materializeRunCredentials(credentialSession, material, {
|
|
4400
|
+
sessionId: input.sessionId,
|
|
4401
|
+
attemptId: input.attemptId,
|
|
4402
|
+
executionGeneration,
|
|
4403
|
+
...(pruneOtherAttempts ? { pruneOtherAttempts: true } : {}),
|
|
4404
|
+
...(!pruneOtherAttempts ? { pruneSupersededGenerations: true } : {}),
|
|
4405
|
+
...(material.authNeeded.length > 0 &&
|
|
4406
|
+
Object.keys(material.environment).length === 0 &&
|
|
4407
|
+
material.files.length === 0
|
|
4408
|
+
? { prunePreviousGenerations: true }
|
|
4409
|
+
: {}),
|
|
4410
|
+
...(toolCancellationFenceRef.current
|
|
4411
|
+
? {
|
|
4412
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
4413
|
+
toolCancellationFenceRef.current,
|
|
4414
|
+
),
|
|
4415
|
+
}
|
|
4416
|
+
: {}),
|
|
4417
|
+
}),
|
|
4418
|
+
);
|
|
4183
4419
|
for (const payload of runCredentialAuthNeededPayloads(material)) {
|
|
4184
4420
|
const key = JSON.stringify(payload);
|
|
4185
4421
|
if (publishedRunCredentialNotices.has(key)) continue;
|
|
@@ -4325,11 +4561,18 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4325
4561
|
settings,
|
|
4326
4562
|
bus,
|
|
4327
4563
|
onOp: machineOpObserver.observer,
|
|
4564
|
+
onHomeSandboxRebound,
|
|
4328
4565
|
},
|
|
4329
4566
|
{
|
|
4330
4567
|
workspaceId: input.workspaceId,
|
|
4331
4568
|
sessionId: input.sessionId,
|
|
4332
4569
|
environment: sandboxEnvironment,
|
|
4570
|
+
workspaceMutationFence: {
|
|
4571
|
+
accountId: input.accountId,
|
|
4572
|
+
turnId: turn.id,
|
|
4573
|
+
executionGeneration,
|
|
4574
|
+
attemptId: input.attemptId,
|
|
4575
|
+
},
|
|
4333
4576
|
pinnedSelfhosted: {
|
|
4334
4577
|
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
4335
4578
|
epoch: activeSandboxPointer!.activeEpoch,
|
|
@@ -4414,24 +4657,41 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4414
4657
|
// the flag off the established group box is injected unchanged (today's
|
|
4415
4658
|
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
4416
4659
|
// routing veneer, not an owner.
|
|
4417
|
-
|
|
4418
|
-
resolvedSandbox
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4660
|
+
resolvedSandbox = {
|
|
4661
|
+
...resolvedSandbox,
|
|
4662
|
+
established: wrapTurnBoxWithRouting(
|
|
4663
|
+
{
|
|
4664
|
+
db,
|
|
4665
|
+
settings,
|
|
4666
|
+
bus,
|
|
4667
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
4668
|
+
onHomeSandboxRebound,
|
|
4669
|
+
},
|
|
4670
|
+
// Thread the SAME declared environment the group box was created with
|
|
4671
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
4672
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
4673
|
+
// (no "cannot change manifest environment variables" throw).
|
|
4674
|
+
{
|
|
4675
|
+
workspaceId: input.workspaceId,
|
|
4676
|
+
sessionId: input.sessionId,
|
|
4677
|
+
environment: sandboxEnvironment,
|
|
4678
|
+
workspaceMutationFence: {
|
|
4679
|
+
accountId: input.accountId,
|
|
4680
|
+
turnId: turn.id,
|
|
4681
|
+
executionGeneration,
|
|
4682
|
+
attemptId: input.attemptId,
|
|
4430
4683
|
},
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4684
|
+
homeLease: {
|
|
4685
|
+
accountId: input.accountId,
|
|
4686
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
4687
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
4688
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
4689
|
+
backend: groupBoxBackend,
|
|
4690
|
+
},
|
|
4691
|
+
},
|
|
4692
|
+
resolvedSandbox.established,
|
|
4693
|
+
),
|
|
4694
|
+
};
|
|
4435
4695
|
}
|
|
4436
4696
|
if (resolvedSandbox) {
|
|
4437
4697
|
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
@@ -4741,6 +5001,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4741
5001
|
await publishSandboxLifecycleEvents(provisioned);
|
|
4742
5002
|
await attachRunCredentialRenewal(
|
|
4743
5003
|
provisioned.established.session as RunCredentialCommandSession,
|
|
5004
|
+
provisioned,
|
|
4744
5005
|
);
|
|
4745
5006
|
const provisionedSetupSession = initialRunCredentialMaterial
|
|
4746
5007
|
? withRunCredentialsSession(
|
|
@@ -4748,39 +5009,46 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4748
5009
|
input.sessionId,
|
|
4749
5010
|
)
|
|
4750
5011
|
: provisioned.established.session;
|
|
4751
|
-
await
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
|
|
4775
|
-
|
|
5012
|
+
await runWorkspaceMutationForSandbox(
|
|
5013
|
+
provisioned,
|
|
5014
|
+
"lazyOwnedSandboxSetup",
|
|
5015
|
+
async () =>
|
|
5016
|
+
await runOwnedSandboxSetup(
|
|
5017
|
+
agent,
|
|
5018
|
+
provisioned.established.session as never,
|
|
5019
|
+
provisionedSetupSession as never,
|
|
5020
|
+
{
|
|
5021
|
+
settings: runSettings,
|
|
5022
|
+
environment: sandboxEnvironment,
|
|
5023
|
+
onRuntimeEvent: async (event) => {
|
|
5024
|
+
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
5025
|
+
},
|
|
5026
|
+
...(lazyGitTokens ? { gitTokenSeedsOverride: lazyGitTokens } : {}),
|
|
5027
|
+
...(lazyGitCredentials?.bindings
|
|
5028
|
+
? { gitCredentialBindingsOverride: lazyGitCredentials.bindings }
|
|
5029
|
+
: {}),
|
|
5030
|
+
...(lazyToolspaceToken
|
|
5031
|
+
? { toolspaceTokenSeedOverride: lazyToolspaceToken.token }
|
|
5032
|
+
: {}),
|
|
5033
|
+
...(toolCancellationFenceRef.current
|
|
5034
|
+
? {
|
|
5035
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5036
|
+
toolCancellationFenceRef.current,
|
|
5037
|
+
),
|
|
5038
|
+
}
|
|
5039
|
+
: {}),
|
|
5040
|
+
},
|
|
5041
|
+
),
|
|
4776
5042
|
);
|
|
4777
5043
|
await attachToolspaceTokenRenewal(
|
|
4778
5044
|
provisioned.established.session as ToolspaceTokenWriterSession,
|
|
4779
5045
|
lazyToolspaceToken?.expiresAt,
|
|
5046
|
+
provisioned,
|
|
4780
5047
|
);
|
|
4781
5048
|
await attachGitCredentialRenewal(
|
|
4782
5049
|
provisioned.established.session as GitCredentialTokenWriterSession,
|
|
4783
5050
|
lazyGitCredentials,
|
|
5051
|
+
provisioned,
|
|
4784
5052
|
);
|
|
4785
5053
|
// Return the REAL established box (NOT a copy whose session is the routing
|
|
4786
5054
|
// proxy). resolveActiveBackend dispatches ops to `provisioned.established.session`;
|
|
@@ -4850,17 +5118,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4850
5118
|
settings,
|
|
4851
5119
|
bus,
|
|
4852
5120
|
onOp: machineOpObserver.observer,
|
|
5121
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
5122
|
+
onHomeSandboxRebound,
|
|
4853
5123
|
},
|
|
4854
5124
|
{
|
|
4855
5125
|
workspaceId: input.workspaceId,
|
|
4856
5126
|
sessionId: input.sessionId,
|
|
4857
5127
|
environment: sandboxEnvironment,
|
|
5128
|
+
workspaceMutationFence: {
|
|
5129
|
+
accountId: input.accountId,
|
|
5130
|
+
turnId: turn.id,
|
|
5131
|
+
executionGeneration,
|
|
5132
|
+
attemptId: input.attemptId,
|
|
5133
|
+
},
|
|
4858
5134
|
},
|
|
4859
5135
|
{
|
|
4860
5136
|
client: lazyClient,
|
|
4861
5137
|
backendId: sdkBackendIdForSandboxBackend(groupBoxBackend),
|
|
4862
5138
|
agentDefaultManifest,
|
|
4863
5139
|
provisioner: turnSandboxProvisioner,
|
|
5140
|
+
homeLeaseIdentity: {
|
|
5141
|
+
accountId: input.accountId,
|
|
5142
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
5143
|
+
backend: groupBoxBackend,
|
|
5144
|
+
},
|
|
4864
5145
|
},
|
|
4865
5146
|
);
|
|
4866
5147
|
}
|
|
@@ -4982,22 +5263,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4982
5263
|
);
|
|
4983
5264
|
const runAs = sandboxRunAs(runSettings);
|
|
4984
5265
|
if (downloadsToMaterialize.length > 0) {
|
|
4985
|
-
const materialized = await
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
4993
|
-
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5266
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
5267
|
+
resolvedSandbox,
|
|
5268
|
+
"fileMaterialization",
|
|
5269
|
+
async () =>
|
|
5270
|
+
await materializeSandboxFileDownloads(
|
|
5271
|
+
setupBoxSession as any,
|
|
5272
|
+
downloadsToMaterialize,
|
|
5273
|
+
{
|
|
5274
|
+
onRuntimeEvent: async (event) => {
|
|
5275
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
5276
|
+
},
|
|
5277
|
+
...(runAs ? { runAs } : {}),
|
|
5278
|
+
...(toolCancellationFenceRef.current
|
|
5279
|
+
? {
|
|
5280
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5281
|
+
toolCancellationFenceRef.current,
|
|
5282
|
+
),
|
|
5283
|
+
}
|
|
5284
|
+
: {}),
|
|
5285
|
+
},
|
|
5286
|
+
),
|
|
5001
5287
|
);
|
|
5002
5288
|
fileMaterializationFailures = materialized.failures;
|
|
5003
5289
|
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
@@ -5045,6 +5331,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5045
5331
|
recovering: turn.executionGeneration > 1,
|
|
5046
5332
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
5047
5333
|
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
5334
|
+
...(modelAcceptsTypedAttachmentContentForTurn(resolvedModel) && objectStorage
|
|
5335
|
+
? {
|
|
5336
|
+
readFileBytesForModel: (file) => objectStorage.getFileBytes(file),
|
|
5337
|
+
}
|
|
5338
|
+
: {}),
|
|
5048
5339
|
},
|
|
5049
5340
|
);
|
|
5050
5341
|
runInput = prepared.input;
|
|
@@ -5159,8 +5450,50 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5159
5450
|
throwIfWorkerShuttingDown();
|
|
5160
5451
|
throwIfTurnCancelled();
|
|
5161
5452
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
5162
|
-
const runStreamOnce = (): ReturnType<OpenGeniRuntime["runStream"]> =>
|
|
5163
|
-
|
|
5453
|
+
const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
|
|
5454
|
+
// Eager owned sessions must settle the exact platform-setup provider
|
|
5455
|
+
// promise before the long model stream starts; otherwise one admission
|
|
5456
|
+
// would remain in flight for the entire turn and suppress every
|
|
5457
|
+
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
5458
|
+
// its first-operation provisioner above.
|
|
5459
|
+
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
5460
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
5461
|
+
await runWorkspaceMutationForSandbox(
|
|
5462
|
+
resolvedSandbox,
|
|
5463
|
+
"eagerOwnedSandboxSetup",
|
|
5464
|
+
async () =>
|
|
5465
|
+
await runOwnedSandboxSetup(
|
|
5466
|
+
agent,
|
|
5467
|
+
ownedEstablished.session as never,
|
|
5468
|
+
eagerSetupSession as never,
|
|
5469
|
+
{
|
|
5470
|
+
settings: runSettings,
|
|
5471
|
+
environment: sandboxEnvironment,
|
|
5472
|
+
preparedInput: runInput!,
|
|
5473
|
+
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
5474
|
+
onRuntimeEvent: async (event) => {
|
|
5475
|
+
await renewCodexLease("runtime_event");
|
|
5476
|
+
if (codexLeaseLost) {
|
|
5477
|
+
throw new Error("Codex credential lease expired during sandbox setup");
|
|
5478
|
+
}
|
|
5479
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
5480
|
+
},
|
|
5481
|
+
...(toolCancellationFenceRef.current
|
|
5482
|
+
? {
|
|
5483
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
5484
|
+
toolCancellationFenceRef.current,
|
|
5485
|
+
),
|
|
5486
|
+
}
|
|
5487
|
+
: {}),
|
|
5488
|
+
},
|
|
5489
|
+
),
|
|
5490
|
+
);
|
|
5491
|
+
await attachGitCredentialRenewal(
|
|
5492
|
+
eagerSetupSession as GitCredentialTokenWriterSession,
|
|
5493
|
+
initialGitCredentials,
|
|
5494
|
+
);
|
|
5495
|
+
}
|
|
5496
|
+
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
5164
5497
|
...(activityContext ? { signal: activityContext.cancellationSignal } : {}),
|
|
5165
5498
|
sandboxEnvironment,
|
|
5166
5499
|
onRuntimeEvent: async (event) => {
|
|
@@ -5187,16 +5520,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5187
5520
|
// re-route those execs onto a machine swapped in mid-turn.
|
|
5188
5521
|
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
5189
5522
|
...(fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {}),
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
: {}),
|
|
5194
|
-
...(initialGitCredentials
|
|
5195
|
-
? {
|
|
5196
|
-
onGitCredentialSessionReady: async (
|
|
5197
|
-
tokenSession: GitCredentialTokenWriterSession,
|
|
5198
|
-
) => {
|
|
5199
|
-
await attachGitCredentialRenewal(tokenSession, initialGitCredentials);
|
|
5523
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
5524
|
+
// above under its exact admission, lazy in the provisioner.
|
|
5525
|
+
deferredSetup: true,
|
|
5200
5526
|
},
|
|
5201
5527
|
}
|
|
5202
5528
|
: {}),
|
|
@@ -5237,6 +5563,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5237
5563
|
? { turnToolCancellationFence: toolCancellationFenceRef.current }
|
|
5238
5564
|
: {}),
|
|
5239
5565
|
});
|
|
5566
|
+
};
|
|
5240
5567
|
if (codexLeaseLost) {
|
|
5241
5568
|
throw new Error("Codex credential lease expired before the model run");
|
|
5242
5569
|
}
|
|
@@ -5333,6 +5660,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5333
5660
|
turnId: activeTurnId,
|
|
5334
5661
|
executionGeneration,
|
|
5335
5662
|
attemptId: input.attemptId,
|
|
5663
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
5336
5664
|
...pendingToolCall,
|
|
5337
5665
|
});
|
|
5338
5666
|
if (!registered.accepted) {
|
|
@@ -5357,6 +5685,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5357
5685
|
executionGeneration,
|
|
5358
5686
|
attemptId: input.attemptId,
|
|
5359
5687
|
callId: completedToolCall.callId,
|
|
5688
|
+
modelToolOutputTruncationTokens: modelRunSettings.modelToolOutputTruncationTokens,
|
|
5360
5689
|
resultItem: completedToolCall.resultItem,
|
|
5361
5690
|
});
|
|
5362
5691
|
if (!recorded.accepted) {
|
|
@@ -6668,10 +6997,12 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6668
6997
|
const finalizerSignal = turnFinalizerCancellationSignal(cancellationSignal, activityStatus);
|
|
6669
6998
|
try {
|
|
6670
6999
|
const toolCancellationFence = toolCancellationFenceRef.current;
|
|
6671
|
-
//
|
|
6672
|
-
//
|
|
6673
|
-
//
|
|
6674
|
-
|
|
7000
|
+
// Every renewal controller is an attempt-owned sandbox writer. Capture
|
|
7001
|
+
// and close all of them before the tool/run-writer drain and before the
|
|
7002
|
+
// quiescence receipt; none may start another admitted write afterward.
|
|
7003
|
+
gitCredentialRenewalClosed = true;
|
|
7004
|
+
const gitRenewalsToStop = gitCredentialRenewals;
|
|
7005
|
+
gitCredentialRenewals = [];
|
|
6675
7006
|
toolspaceTokenRenewalClosed = true;
|
|
6676
7007
|
const toolspaceRenewalToStop =
|
|
6677
7008
|
toolspaceTokenRenewal as ToolspaceTokenRenewalController | null;
|
|
@@ -6679,9 +7010,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6679
7010
|
runCredentialRenewalClosed = true;
|
|
6680
7011
|
const runRenewalToStop = runCredentialRenewal as RunCredentialRenewalController | null;
|
|
6681
7012
|
runCredentialRenewal = null;
|
|
7013
|
+
|
|
7014
|
+
// Attempt-qualified credential deletion is also a real workspace write.
|
|
7015
|
+
// Perform it under the same admission fence before publishing physical
|
|
7016
|
+
// quiescence; a failure deliberately keeps the receipt closed.
|
|
7017
|
+
const credentialSessionToClear = runCredentialSession;
|
|
7018
|
+
runCredentialSession = null;
|
|
7019
|
+
if (credentialSessionToClear) {
|
|
7020
|
+
await runWorkspaceMutationForSandbox(
|
|
7021
|
+
requireResolvedSandboxForMutation(
|
|
7022
|
+
"Run credential cleanup has no exact sandbox lease target",
|
|
7023
|
+
),
|
|
7024
|
+
"runCredentialAttemptClear",
|
|
7025
|
+
async () =>
|
|
7026
|
+
await clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
7027
|
+
sessionId: input.sessionId,
|
|
7028
|
+
attemptId: input.attemptId,
|
|
7029
|
+
executionGeneration,
|
|
7030
|
+
}),
|
|
7031
|
+
);
|
|
7032
|
+
}
|
|
6682
7033
|
await drainAttemptOwnedSandboxWriters({
|
|
6683
7034
|
toolCancellationFence: acknowledgeQuiescence ? toolCancellationFence : null,
|
|
6684
7035
|
cancellationReason: cancellationSignal?.reason,
|
|
7036
|
+
gitCredentialRenewals: gitRenewalsToStop,
|
|
6685
7037
|
toolspaceTokenRenewal: toolspaceRenewalToStop,
|
|
6686
7038
|
runCredentialRenewal: runRenewalToStop,
|
|
6687
7039
|
});
|
|
@@ -6761,43 +7113,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6761
7113
|
});
|
|
6762
7114
|
}
|
|
6763
7115
|
}
|
|
6764
|
-
// Deleting this attempt's credential generation is post-receipt
|
|
6765
|
-
// housekeeping: pointer mutation is serialized with successor
|
|
6766
|
-
// activation and the attempt-qualified check cannot erase a successor.
|
|
6767
|
-
// It may therefore detach on cancellation without weakening the writer
|
|
6768
|
-
// drain that gates the receipt above.
|
|
6769
|
-
const credentialSessionToClear = runCredentialSession;
|
|
6770
|
-
runCredentialSession = null;
|
|
6771
|
-
if (credentialSessionToClear) {
|
|
6772
|
-
const cleanup = clearRunCredentialsForAttempt(credentialSessionToClear, {
|
|
6773
|
-
sessionId: input.sessionId,
|
|
6774
|
-
attemptId: input.attemptId,
|
|
6775
|
-
executionGeneration,
|
|
6776
|
-
}).catch((error: unknown) => {
|
|
6777
|
-
try {
|
|
6778
|
-
observability.incrementCounter({
|
|
6779
|
-
name: "opengeni_run_credential_cleanup_total",
|
|
6780
|
-
help: "Attempt-owned run credential cleanup outcomes.",
|
|
6781
|
-
labels: { outcome: "error" },
|
|
6782
|
-
});
|
|
6783
|
-
observability.warn("Attempt-owned run credential cleanup failed", {
|
|
6784
|
-
sessionId: input.sessionId,
|
|
6785
|
-
turnId,
|
|
6786
|
-
attemptId: input.attemptId,
|
|
6787
|
-
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
6788
|
-
});
|
|
6789
|
-
} catch {
|
|
6790
|
-
// Cleanup observability must not own finalizer liveness.
|
|
6791
|
-
}
|
|
6792
|
-
});
|
|
6793
|
-
await waitForTurnFinalizerStep(cleanup, finalizerSignal);
|
|
6794
|
-
}
|
|
6795
|
-
gitCredentialRenewalClosed = true;
|
|
6796
|
-
const renewalsToStop = gitCredentialRenewals;
|
|
6797
|
-
gitCredentialRenewals = [];
|
|
6798
|
-
for (const renewal of renewalsToStop) {
|
|
6799
|
-
await waitForTurnFinalizerStep(renewal.stop(), finalizerSignal);
|
|
6800
|
-
}
|
|
6801
7116
|
// Drain the buffered Connected Machine op events (infra failures + healed
|
|
6802
7117
|
// recoveries) to durable session events — awaited, best-effort, never blocking
|
|
6803
7118
|
// the turn. Sync observer → buffer → single awaited append here (no unawaited
|