@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -3,13 +3,14 @@ import {
|
|
|
3
3
|
PostCompactionContinuationEmptyError,
|
|
4
4
|
SandboxDeadlineRotationError,
|
|
5
5
|
agentRunFailurePayload,
|
|
6
|
+
armAndReconcileCodexCapacityWait,
|
|
6
7
|
armTurnQuiescenceWatchdog,
|
|
7
8
|
assertAgentStreamNotCancelled,
|
|
8
9
|
assertPhysicalToolQuiescenceForCancellation,
|
|
9
10
|
assertSessionAttemptQuiescenceRecoveryDurable,
|
|
10
11
|
assertSuccessfulAgentStreamCompletion,
|
|
11
12
|
authoritativeCodexCapacityResetAt,
|
|
12
|
-
|
|
13
|
+
buildXaiTurnRequestAuthorization,
|
|
13
14
|
classifyCodexCredentialFailure,
|
|
14
15
|
classifyCodexPin,
|
|
15
16
|
classifyContextWindowOverflowError,
|
|
@@ -22,7 +23,6 @@ import {
|
|
|
22
23
|
compactionFailureReason,
|
|
23
24
|
compactionFailureReasonFromError,
|
|
24
25
|
computeIdleDelayMs,
|
|
25
|
-
computeReactiveRotationResume,
|
|
26
26
|
createRuntimeBatcher,
|
|
27
27
|
createSharedActivityServices,
|
|
28
28
|
createTurnSandboxProvisioner,
|
|
@@ -30,13 +30,14 @@ import {
|
|
|
30
30
|
deliverChildRequiresActionToParent,
|
|
31
31
|
deliverFailedChildTurnToParent,
|
|
32
32
|
drainAttemptOwnedSandboxWriters,
|
|
33
|
-
earliestCodexReset,
|
|
34
33
|
escapedMcpTimeoutRecoveryFailure,
|
|
34
|
+
establishSelfhostedTurnSession,
|
|
35
35
|
finalizeDurableTurnOpStreams,
|
|
36
36
|
isCodexCredentialEligible,
|
|
37
37
|
isCompactionSummaryFailure,
|
|
38
38
|
isLazySandboxProvisionRetryable,
|
|
39
39
|
isWorkerShutdownCancellation,
|
|
40
|
+
lazyProvisionEnabled,
|
|
40
41
|
maybePersistWarmWorkspaceSnapshot,
|
|
41
42
|
mergeResourceRefs,
|
|
42
43
|
nextStreamEvent,
|
|
@@ -49,12 +50,14 @@ import {
|
|
|
49
50
|
providerRecoveryResult,
|
|
50
51
|
providerRetryAfterMs,
|
|
51
52
|
refreshCodexUsageAndRepairCapacityWaiters,
|
|
53
|
+
refreshExhaustedXaiQuota,
|
|
52
54
|
releaseTurnSandboxAfterWriterDrain,
|
|
53
55
|
requireAgentStreamFinalOutput,
|
|
54
56
|
resolveWorkspaceInstalledSkillRuntime,
|
|
55
57
|
resolveWorkspacePackRuntime,
|
|
56
58
|
resumeBoxForTurn,
|
|
57
59
|
rigProviderImageSourceImage,
|
|
60
|
+
routingEnabled,
|
|
58
61
|
runMandatoryHistoryPersistenceStep,
|
|
59
62
|
safeErrorDiagnostic,
|
|
60
63
|
safeErrorForTelemetry,
|
|
@@ -64,7 +67,6 @@ import {
|
|
|
64
67
|
sandboxRouteTransitionCode,
|
|
65
68
|
selectCodexCredentialLeaseForTurn,
|
|
66
69
|
settingsWithPackSandboxImage,
|
|
67
|
-
shardCredentialForSession,
|
|
68
70
|
shouldRecoverCompactionProviderFailure,
|
|
69
71
|
shouldRunTurnEndWorkspacePersistence,
|
|
70
72
|
shouldStartPeriodicWorkspaceSnapshot,
|
|
@@ -79,8 +81,10 @@ import {
|
|
|
79
81
|
waitForTurnFinalizerStep,
|
|
80
82
|
waitForTurnOperation,
|
|
81
83
|
waitForWarmSnapshot,
|
|
82
|
-
withFirstPartyTools
|
|
83
|
-
|
|
84
|
+
withFirstPartyTools,
|
|
85
|
+
wrapLazyTurnBoxWithRouting,
|
|
86
|
+
wrapTurnBoxWithRouting
|
|
87
|
+
} from "./chunk-O2LQWJIK.js";
|
|
84
88
|
import {
|
|
85
89
|
StreamTimingMetrics,
|
|
86
90
|
makeMachineOpObserver,
|
|
@@ -106,7 +110,7 @@ import {
|
|
|
106
110
|
recordTurnWorkerPreparationTotal,
|
|
107
111
|
runtimeMetricsHooksForObservability,
|
|
108
112
|
turnLifecycleMetricsFor
|
|
109
|
-
} from "./chunk-
|
|
113
|
+
} from "./chunk-D4ZEFOYH.js";
|
|
110
114
|
|
|
111
115
|
// src/activities-turn.ts
|
|
112
116
|
import { createProductionAgentRuntime, summarizeForCompaction as summarizeForCompaction2 } from "@opengeni/runtime";
|
|
@@ -140,100 +144,6 @@ import {
|
|
|
140
144
|
xaiSubscriptionRequestStorage
|
|
141
145
|
} from "@opengeni/xai-subscription";
|
|
142
146
|
|
|
143
|
-
// src/activities/xai-auth.ts
|
|
144
|
-
import { environmentsEncryptionKeyBytes } from "@opengeni/config";
|
|
145
|
-
import {
|
|
146
|
-
materializeXaiCredentialForRun,
|
|
147
|
-
refreshXaiSubscriptionCredentialSerialized
|
|
148
|
-
} from "@opengeni/db";
|
|
149
|
-
import {
|
|
150
|
-
refreshXaiToken,
|
|
151
|
-
XAI_CLIENT_VERSION,
|
|
152
|
-
XAI_SUBSCRIPTION_MODEL_ID_PREFIX,
|
|
153
|
-
XaiSubscriptionReloginRequired,
|
|
154
|
-
xaiAccessTokenExpiry
|
|
155
|
-
} from "@opengeni/xai-subscription";
|
|
156
|
-
async function buildXaiTurnRequestAuthorization(input) {
|
|
157
|
-
const encryptionKey = environmentsEncryptionKeyBytes(input.settings);
|
|
158
|
-
if (!encryptionKey) {
|
|
159
|
-
throw new Error("SuperGrok subscriptions require OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
|
|
160
|
-
}
|
|
161
|
-
let credential = await materializeXaiCredentialForRun(input.db, {
|
|
162
|
-
workspaceId: input.workspaceId,
|
|
163
|
-
subjectId: input.subjectId,
|
|
164
|
-
credentialId: input.credentialId,
|
|
165
|
-
authoritySnapshot: input.authoritySnapshot,
|
|
166
|
-
encryptionKey
|
|
167
|
-
});
|
|
168
|
-
const tokenSnapshot = () => {
|
|
169
|
-
const accessToken = credential.secret.accessToken;
|
|
170
|
-
const userId = credential.providerAccountId;
|
|
171
|
-
if (!accessToken || !userId) {
|
|
172
|
-
throw new XaiSubscriptionReloginRequired(
|
|
173
|
-
"The SuperGrok connection is missing a verified token identity. Reconnect the account."
|
|
174
|
-
);
|
|
175
|
-
}
|
|
176
|
-
return { accessToken, userId };
|
|
177
|
-
};
|
|
178
|
-
const refresh = async () => {
|
|
179
|
-
const observedAccessToken = credential.secret.accessToken;
|
|
180
|
-
const observedRefreshToken = credential.secret.refreshToken;
|
|
181
|
-
if (!observedRefreshToken) {
|
|
182
|
-
throw new XaiSubscriptionReloginRequired(
|
|
183
|
-
"The SuperGrok connection cannot be refreshed. Reconnect the account."
|
|
184
|
-
);
|
|
185
|
-
}
|
|
186
|
-
const result = await refreshXaiSubscriptionCredentialSerialized(input.db, {
|
|
187
|
-
accountId: input.accountId,
|
|
188
|
-
workspaceId: input.workspaceId,
|
|
189
|
-
subjectId: input.subjectId,
|
|
190
|
-
credentialId: input.credentialId,
|
|
191
|
-
authoritySnapshot: input.authoritySnapshot,
|
|
192
|
-
encryptionKey,
|
|
193
|
-
observedAccessToken,
|
|
194
|
-
observedRefreshToken,
|
|
195
|
-
refresh: async (current) => {
|
|
196
|
-
const refreshToken = current.secret.refreshToken;
|
|
197
|
-
if (!refreshToken) {
|
|
198
|
-
throw new XaiSubscriptionReloginRequired(
|
|
199
|
-
"The SuperGrok connection cannot be refreshed. Reconnect the account."
|
|
200
|
-
);
|
|
201
|
-
}
|
|
202
|
-
const tokens = await refreshXaiToken(refreshToken, {
|
|
203
|
-
...input.fetch ? { fetch: input.fetch } : {}
|
|
204
|
-
});
|
|
205
|
-
return {
|
|
206
|
-
secret: {
|
|
207
|
-
version: 1,
|
|
208
|
-
accessToken: tokens.accessToken,
|
|
209
|
-
refreshToken: tokens.refreshToken
|
|
210
|
-
},
|
|
211
|
-
expiresAt: xaiAccessTokenExpiry(tokens.accessToken) ?? new Date(Date.now() + tokens.expiresInSeconds * 1e3)
|
|
212
|
-
};
|
|
213
|
-
}
|
|
214
|
-
});
|
|
215
|
-
credential = result.credential;
|
|
216
|
-
return tokenSnapshot();
|
|
217
|
-
};
|
|
218
|
-
return {
|
|
219
|
-
credentialId: input.credentialId,
|
|
220
|
-
context: {
|
|
221
|
-
clientVersion: XAI_CLIENT_VERSION,
|
|
222
|
-
sessionId: input.sessionId,
|
|
223
|
-
turnId: input.turnId,
|
|
224
|
-
getToken: async () => tokenSnapshot(),
|
|
225
|
-
refresh,
|
|
226
|
-
resolveModel: (model) => model.startsWith(XAI_SUBSCRIPTION_MODEL_ID_PREFIX) ? model.slice(XAI_SUBSCRIPTION_MODEL_ID_PREFIX.length) : model,
|
|
227
|
-
...input.hostedSearch ? { hostedSearch: input.hostedSearch } : {},
|
|
228
|
-
...input.onFinalContextUsage ? { onFinalContextUsage: input.onFinalContextUsage } : {},
|
|
229
|
-
...input.nextRequestId ? { nextRequestId: input.nextRequestId } : {},
|
|
230
|
-
...input.streamIdleTimeoutMs ? { streamIdleTimeoutMs: input.streamIdleTimeoutMs } : {},
|
|
231
|
-
...input.onModelRequestDiagnostic ? { onModelRequestDiagnostic: input.onModelRequestDiagnostic } : {},
|
|
232
|
-
...input.onModelRequestEvent ? { onModelRequestEvent: input.onModelRequestEvent } : {}
|
|
233
|
-
}
|
|
234
|
-
};
|
|
235
|
-
}
|
|
236
|
-
|
|
237
147
|
// src/activities/turn-attempt-fenced.ts
|
|
238
148
|
var TurnAttemptFencedError = class extends Error {
|
|
239
149
|
constructor(message) {
|
|
@@ -1259,12 +1169,21 @@ function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performan
|
|
|
1259
1169
|
}
|
|
1260
1170
|
|
|
1261
1171
|
// src/activities/agent-turn/credential-leases.ts
|
|
1172
|
+
var CodexCredentialLeaseLostError = class extends Error {
|
|
1173
|
+
constructor(reason) {
|
|
1174
|
+
super("Codex credential lease is not usable for provider dispatch");
|
|
1175
|
+
this.reason = reason;
|
|
1176
|
+
this.name = "CodexCredentialLeaseLostError";
|
|
1177
|
+
}
|
|
1178
|
+
code = "codex_credential_lease_lost";
|
|
1179
|
+
};
|
|
1262
1180
|
var CodexTurnLease = class {
|
|
1263
1181
|
constructor(deps) {
|
|
1264
1182
|
this.deps = deps;
|
|
1265
1183
|
}
|
|
1266
1184
|
held = false;
|
|
1267
1185
|
lost = false;
|
|
1186
|
+
lossReason = null;
|
|
1268
1187
|
holderId = null;
|
|
1269
1188
|
generation = null;
|
|
1270
1189
|
// Monotonic worker deadline, not a comparison between the Postgres and
|
|
@@ -1277,6 +1196,7 @@ var CodexTurnLease = class {
|
|
|
1277
1196
|
markLost = (reason) => {
|
|
1278
1197
|
if (this.lost) return;
|
|
1279
1198
|
this.lost = true;
|
|
1199
|
+
this.lossReason = reason;
|
|
1280
1200
|
this.deps.observability.incrementCounter({
|
|
1281
1201
|
name: "opengeni_codex_lease_renewals_total",
|
|
1282
1202
|
help: "Codex lease renewal checkpoints by outcome and reason.",
|
|
@@ -1288,6 +1208,16 @@ var CodexTurnLease = class {
|
|
|
1288
1208
|
reason
|
|
1289
1209
|
});
|
|
1290
1210
|
};
|
|
1211
|
+
/** Fail closed at the last worker-confirmed lease deadline before dispatch. */
|
|
1212
|
+
assertUsable = () => {
|
|
1213
|
+
if (this.lost) {
|
|
1214
|
+
throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
|
|
1215
|
+
}
|
|
1216
|
+
if (!this.held || this.holderId === null || this.generation === null || codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
|
|
1217
|
+
this.markLost("deadline");
|
|
1218
|
+
throw new CodexCredentialLeaseLostError("deadline");
|
|
1219
|
+
}
|
|
1220
|
+
};
|
|
1291
1221
|
renew = async (reason) => {
|
|
1292
1222
|
const turnId = this.deps.getTurnId();
|
|
1293
1223
|
if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
|
|
@@ -1299,6 +1229,7 @@ var CodexTurnLease = class {
|
|
|
1299
1229
|
}
|
|
1300
1230
|
if (this.heartbeatInFlight) return;
|
|
1301
1231
|
this.heartbeatInFlight = true;
|
|
1232
|
+
const priorConfirmedUntilMs = this.confirmedUntilMs;
|
|
1302
1233
|
const renewalStartedAtMs = performance.now();
|
|
1303
1234
|
try {
|
|
1304
1235
|
const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
|
|
@@ -1312,6 +1243,8 @@ var CodexTurnLease = class {
|
|
|
1312
1243
|
);
|
|
1313
1244
|
if (!renewedUntil) {
|
|
1314
1245
|
this.markLost("not_found");
|
|
1246
|
+
} else if (codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) || codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)) {
|
|
1247
|
+
this.markLost("deadline");
|
|
1315
1248
|
} else {
|
|
1316
1249
|
this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
|
|
1317
1250
|
this.deps.observability.incrementCounter({
|
|
@@ -3272,11 +3205,63 @@ function completedToolCallFromSdkEvent(event) {
|
|
|
3272
3205
|
};
|
|
3273
3206
|
}
|
|
3274
3207
|
|
|
3208
|
+
// src/activities/agent-turn/history-prefix.ts
|
|
3209
|
+
import { createHash as createHash5 } from "crypto";
|
|
3210
|
+
import { normalizeProtocolJsonValue as normalizeProtocolJsonValue2, sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2 } from "@opengeni/runtime";
|
|
3211
|
+
var HistoryPrefixGuard = class {
|
|
3212
|
+
persisted = [];
|
|
3213
|
+
seed(items, count, truncationTokens) {
|
|
3214
|
+
const keys = this.keys(items, truncationTokens);
|
|
3215
|
+
if (keys.length < count)
|
|
3216
|
+
throw new Error("Conversation history seed is shorter than its durable prefix");
|
|
3217
|
+
this.persisted = keys.slice(0, count);
|
|
3218
|
+
}
|
|
3219
|
+
verify(items, truncationTokens) {
|
|
3220
|
+
const keys = this.keys(items, truncationTokens);
|
|
3221
|
+
for (let i = 0; i < this.persisted.length; i++) {
|
|
3222
|
+
if (keys[i] !== this.persisted[i]) {
|
|
3223
|
+
throw new Error(
|
|
3224
|
+
`Conversation history durable prefix changed at item ${i}; refusing to skip unsaved work`
|
|
3225
|
+
);
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
return keys;
|
|
3229
|
+
}
|
|
3230
|
+
acknowledge(keys, count) {
|
|
3231
|
+
this.persisted = keys.slice(0, count);
|
|
3232
|
+
}
|
|
3233
|
+
keys(items, truncationTokens) {
|
|
3234
|
+
return sanitizeHistoryItemsForModel2(items, truncationTokens).map(
|
|
3235
|
+
(item) => createHash5("sha256").update(JSON.stringify(sorted(normalizeProtocolJsonValue2(item)))).digest("hex")
|
|
3236
|
+
);
|
|
3237
|
+
}
|
|
3238
|
+
};
|
|
3239
|
+
function sorted(value) {
|
|
3240
|
+
if (Array.isArray(value)) return value.map(sorted);
|
|
3241
|
+
if (value !== null && typeof value === "object") {
|
|
3242
|
+
return Object.fromEntries(
|
|
3243
|
+
Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => [key, sorted(entry)])
|
|
3244
|
+
);
|
|
3245
|
+
}
|
|
3246
|
+
return value;
|
|
3247
|
+
}
|
|
3248
|
+
|
|
3275
3249
|
// src/activities/agent-turn/history-sink.ts
|
|
3276
3250
|
var TurnHistorySink = class {
|
|
3277
3251
|
constructor(deps) {
|
|
3278
3252
|
this.deps = deps;
|
|
3279
3253
|
}
|
|
3254
|
+
prefix = new HistoryPrefixGuard();
|
|
3255
|
+
seedHistory(input, count) {
|
|
3256
|
+
const items = typeof input === "string" && count === 0 ? [] : Array.isArray(input) ? input : input?.history;
|
|
3257
|
+
if (!Array.isArray(items)) throw new Error("Conversation history seed is unavailable");
|
|
3258
|
+
this.prefix.seed(
|
|
3259
|
+
items,
|
|
3260
|
+
count,
|
|
3261
|
+
this.deps.getModelRunSettings().modelToolOutputTruncationTokens
|
|
3262
|
+
);
|
|
3263
|
+
this.persistedHistoryCount = count;
|
|
3264
|
+
}
|
|
3280
3265
|
persistedHistoryCount = 0;
|
|
3281
3266
|
nextHistoryPosition = 0;
|
|
3282
3267
|
providerArtifactCandidates = {
|
|
@@ -3295,6 +3280,10 @@ var TurnHistorySink = class {
|
|
|
3295
3280
|
const rawHistory = stream.state.history;
|
|
3296
3281
|
if (Array.isArray(rawHistory)) {
|
|
3297
3282
|
const typedHistory = rawHistory;
|
|
3283
|
+
const verifiedKeys = this.prefix.verify(
|
|
3284
|
+
typedHistory,
|
|
3285
|
+
this.deps.getModelRunSettings().modelToolOutputTruncationTokens
|
|
3286
|
+
);
|
|
3298
3287
|
await media.retainNativeGeneratedImagesFromHistory(typedHistory);
|
|
3299
3288
|
const durableHistory = compactGeneratedImageHistory(
|
|
3300
3289
|
compactRetainedScreenshotHistory(typedHistory, media.retainedScreenshotReceiptsByCallId),
|
|
@@ -3330,9 +3319,14 @@ var TurnHistorySink = class {
|
|
|
3330
3319
|
});
|
|
3331
3320
|
}
|
|
3332
3321
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
3322
|
+
this.prefix.acknowledge(verifiedKeys, nextWatermark);
|
|
3333
3323
|
this.persistedHistoryCount = nextWatermark;
|
|
3334
3324
|
this.nextHistoryPosition = nextPosition;
|
|
3335
3325
|
}
|
|
3326
|
+
} else if (options.requireDurable) {
|
|
3327
|
+
throw new Error(
|
|
3328
|
+
"SDK conversation history is unavailable at a mandatory persistence boundary"
|
|
3329
|
+
);
|
|
3336
3330
|
}
|
|
3337
3331
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
3338
3332
|
if (envelope) {
|
|
@@ -3347,10 +3341,7 @@ var TurnHistorySink = class {
|
|
|
3347
3341
|
);
|
|
3348
3342
|
}
|
|
3349
3343
|
} catch (persistError) {
|
|
3350
|
-
console.error(
|
|
3351
|
-
"session history dual-write failed (run unaffected)",
|
|
3352
|
-
safeErrorDiagnostic(persistError)
|
|
3353
|
-
);
|
|
3344
|
+
console.error("session history persistence failed", safeErrorDiagnostic(persistError));
|
|
3354
3345
|
if (options.requireDurable) throw persistError;
|
|
3355
3346
|
}
|
|
3356
3347
|
};
|
|
@@ -3458,6 +3449,7 @@ function createTurnContext(input) {
|
|
|
3458
3449
|
},
|
|
3459
3450
|
attempt: {
|
|
3460
3451
|
turnId: void 0,
|
|
3452
|
+
dispatchId: "",
|
|
3461
3453
|
triggerEventId: void 0,
|
|
3462
3454
|
executionGeneration: 0,
|
|
3463
3455
|
providerRecoveryCount: 0,
|
|
@@ -3530,6 +3522,9 @@ function createTurnContext(input) {
|
|
|
3530
3522
|
},
|
|
3531
3523
|
providerTurn: {
|
|
3532
3524
|
effectiveCodexCredentialId: null,
|
|
3525
|
+
effectiveCodexCredentialVersion: null,
|
|
3526
|
+
codexCredentialFailoverLimit: 1,
|
|
3527
|
+
codexPolicySnapshot: null,
|
|
3533
3528
|
effectiveXaiCredentialId: null,
|
|
3534
3529
|
xaiRotationEnabled: false,
|
|
3535
3530
|
xaiAuthoritySnapshot: null,
|
|
@@ -3556,7 +3551,7 @@ import { clearRunCredentialsForAttempt } from "@opengeni/runtime";
|
|
|
3556
3551
|
import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
|
|
3557
3552
|
|
|
3558
3553
|
// src/activities/workspace-capture.ts
|
|
3559
|
-
import { createHash as
|
|
3554
|
+
import { createHash as createHash6 } from "crypto";
|
|
3560
3555
|
import {
|
|
3561
3556
|
deleteWorkspaceCaptureRows,
|
|
3562
3557
|
insertFailedWorkspaceCapture,
|
|
@@ -4310,7 +4305,7 @@ function joinRepoPath(repoRoot, repoRelPath) {
|
|
|
4310
4305
|
return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
|
|
4311
4306
|
}
|
|
4312
4307
|
function sha256(bytes) {
|
|
4313
|
-
return
|
|
4308
|
+
return createHash6("sha256").update(bytes).digest("hex");
|
|
4314
4309
|
}
|
|
4315
4310
|
function utf8(s) {
|
|
4316
4311
|
return new TextEncoder().encode(s);
|
|
@@ -4913,17 +4908,10 @@ async function finalizeTurnAttempt(deps) {
|
|
|
4913
4908
|
import {
|
|
4914
4909
|
requestSessionTurnRecovery,
|
|
4915
4910
|
getSessionGoal,
|
|
4916
|
-
armCodexCapacityWait,
|
|
4917
4911
|
armXaiCapacityWait,
|
|
4918
4912
|
reconcileXaiCapacityWait,
|
|
4919
|
-
getCodexRotationSettings,
|
|
4920
4913
|
listCodexAccountStatuses as listCodexAccountStatuses2,
|
|
4921
|
-
getSessionCodexState,
|
|
4922
|
-
setSessionCodexPinInTransaction,
|
|
4923
4914
|
quarantineCodexCredentialForLease,
|
|
4924
|
-
setCodexCredentialExhaustedWithWakeTargets,
|
|
4925
|
-
withSessionCodexCapacityMutation,
|
|
4926
|
-
countConsecutiveReactiveRotations,
|
|
4927
4915
|
recordUsageEvent as recordUsageEvent2,
|
|
4928
4916
|
getActiveSessionHistoryItemsPaged,
|
|
4929
4917
|
settleCodexCredentialLeaseLoss,
|
|
@@ -4948,7 +4936,7 @@ import {
|
|
|
4948
4936
|
} from "@opengeni/config";
|
|
4949
4937
|
import { selectCodexCredentialId } from "@opengeni/codex";
|
|
4950
4938
|
import { directPersonalConnectionSubjectId } from "@opengeni/core";
|
|
4951
|
-
import { createHash as
|
|
4939
|
+
import { createHash as createHash7 } from "crypto";
|
|
4952
4940
|
|
|
4953
4941
|
// src/activities/agent-turn/model-usage.ts
|
|
4954
4942
|
import {
|
|
@@ -5552,7 +5540,7 @@ function legacyTurnExecutionPolicyInput(turn) {
|
|
|
5552
5540
|
};
|
|
5553
5541
|
}
|
|
5554
5542
|
function stableHumanInputRequestId(sessionId, turnId, toolCallId) {
|
|
5555
|
-
const hex =
|
|
5543
|
+
const hex = createHash7("sha256").update("opengeni-human-input-v1\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
|
|
5556
5544
|
hex[12] = "5";
|
|
5557
5545
|
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
5558
5546
|
const value = hex.join("");
|
|
@@ -5575,7 +5563,7 @@ function stableInteractionInterventionOperationId(sessionId, turnId, toolCallId)
|
|
|
5575
5563
|
);
|
|
5576
5564
|
}
|
|
5577
5565
|
function stableInteractionInterventionUuid(namespace, sessionId, turnId, toolCallId) {
|
|
5578
|
-
const hex =
|
|
5566
|
+
const hex = createHash7("sha256").update(namespace).update("\0").update(sessionId).update("\0").update(turnId).update("\0").update(toolCallId).digest("hex").slice(0, 32).split("");
|
|
5579
5567
|
hex[12] = "5";
|
|
5580
5568
|
hex[16] = ["8", "9", "a", "b"][Number.parseInt(hex[16] ?? "0", 16) % 4] ?? "8";
|
|
5581
5569
|
const value = hex.join("");
|
|
@@ -5633,6 +5621,59 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternal
|
|
|
5633
5621
|
}
|
|
5634
5622
|
|
|
5635
5623
|
// src/activities/agent-turn/failure-settlement.ts
|
|
5624
|
+
function codexDefinitiveFailureDisposition(input) {
|
|
5625
|
+
const alternateAvailable = input.rotationEnabled && input.pinDisposition !== "manual" && input.decisionKind === "active" && input.decisionCredentialId !== null && input.decisionCredentialId !== input.servingCredentialId;
|
|
5626
|
+
if (alternateAvailable) return "failover";
|
|
5627
|
+
if (input.failureKind === "quota" || input.failureKind === "rate_limit" || input.decisionKind === "allCapped" || !input.rotationEnabled || input.pinDisposition === "manual") {
|
|
5628
|
+
return "wait";
|
|
5629
|
+
}
|
|
5630
|
+
return "terminal";
|
|
5631
|
+
}
|
|
5632
|
+
function codexCapacityWaitFailurePayload(input) {
|
|
5633
|
+
if (input.failureKind === "quota") {
|
|
5634
|
+
return codexUsageLimitFailurePayload(
|
|
5635
|
+
input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
|
|
5636
|
+
input.detail,
|
|
5637
|
+
input.allAccounts ? { allAccounts: true } : void 0
|
|
5638
|
+
);
|
|
5639
|
+
}
|
|
5640
|
+
if (input.failureKind === "rate_limit") {
|
|
5641
|
+
return {
|
|
5642
|
+
error: "The serving Codex subscription is temporarily rate limited.",
|
|
5643
|
+
code: "codex_account_rate_limited",
|
|
5644
|
+
detail: input.detail,
|
|
5645
|
+
retryable: false
|
|
5646
|
+
};
|
|
5647
|
+
}
|
|
5648
|
+
if (input.failureKind === "auth") {
|
|
5649
|
+
return {
|
|
5650
|
+
error: "The serving Codex account requires reconnection.",
|
|
5651
|
+
code: "codex_relogin_required",
|
|
5652
|
+
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
5653
|
+
retryable: false
|
|
5654
|
+
};
|
|
5655
|
+
}
|
|
5656
|
+
return {
|
|
5657
|
+
error: "The serving Codex account is not authorized for this request.",
|
|
5658
|
+
code: "codex_account_forbidden",
|
|
5659
|
+
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
5660
|
+
retryable: false
|
|
5661
|
+
};
|
|
5662
|
+
}
|
|
5663
|
+
function codexLeaseAccountsForSelection(accounts) {
|
|
5664
|
+
return accounts.map((account) => ({
|
|
5665
|
+
...account,
|
|
5666
|
+
activeLeaseCount: 0,
|
|
5667
|
+
selectionCount: 0,
|
|
5668
|
+
lastSelectedAt: null
|
|
5669
|
+
}));
|
|
5670
|
+
}
|
|
5671
|
+
function acceptedCodexPolicySnapshot(providerTurn) {
|
|
5672
|
+
if (!providerTurn.codexPolicySnapshot) {
|
|
5673
|
+
throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
|
|
5674
|
+
}
|
|
5675
|
+
return providerTurn.codexPolicySnapshot;
|
|
5676
|
+
}
|
|
5636
5677
|
async function settleTurnFailure(deps) {
|
|
5637
5678
|
const {
|
|
5638
5679
|
error,
|
|
@@ -5642,7 +5683,6 @@ async function settleTurnFailure(deps) {
|
|
|
5642
5683
|
bus,
|
|
5643
5684
|
observability,
|
|
5644
5685
|
wakeSessionWorkflow,
|
|
5645
|
-
signalCodexCapacityWorkflow,
|
|
5646
5686
|
cancellationSignal,
|
|
5647
5687
|
sandboxRotationController,
|
|
5648
5688
|
noteCancellationRequested,
|
|
@@ -5672,6 +5712,12 @@ async function settleTurnFailure(deps) {
|
|
|
5672
5712
|
const rotationPending = fencedLease?.rotationRequestedAt != null || lifecycleTransition?.reason === "rotation_in_progress";
|
|
5673
5713
|
const transitionPending = lifecycleTransition !== null || rotationPending;
|
|
5674
5714
|
const deadlineRotationPending = rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
5715
|
+
const sandboxLifecycleWait = rotationPending ? {
|
|
5716
|
+
version: 1,
|
|
5717
|
+
sandboxGroupId: leaseControlError.sandboxGroupId,
|
|
5718
|
+
leaseEpoch: fencedLease?.leaseEpoch ?? leaseControlError.leaseEpoch,
|
|
5719
|
+
reason: "rotation_in_progress"
|
|
5720
|
+
} : void 0;
|
|
5675
5721
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
5676
5722
|
sessionId: input.sessionId,
|
|
5677
5723
|
turnId: recoveryTurnId,
|
|
@@ -5687,7 +5733,8 @@ async function settleTurnFailure(deps) {
|
|
|
5687
5733
|
} : {},
|
|
5688
5734
|
...lifecycleTransition ? { transitionReason: lifecycleTransition.reason } : {}
|
|
5689
5735
|
}
|
|
5690
|
-
} : {}
|
|
5736
|
+
} : {},
|
|
5737
|
+
...sandboxLifecycleWait ? { sandboxLifecycleWait } : {}
|
|
5691
5738
|
});
|
|
5692
5739
|
if (recovery.action === "stale") {
|
|
5693
5740
|
acknowledgeLostAttemptOwnership();
|
|
@@ -5701,7 +5748,7 @@ async function settleTurnFailure(deps) {
|
|
|
5701
5748
|
control.turnMetricOutcome = "recovering";
|
|
5702
5749
|
return claimedResult({
|
|
5703
5750
|
status: "recovering",
|
|
5704
|
-
...transitionPending ? {
|
|
5751
|
+
...transitionPending && !sandboxLifecycleWait ? {
|
|
5705
5752
|
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings)
|
|
5706
5753
|
} : {}
|
|
5707
5754
|
});
|
|
@@ -5760,6 +5807,12 @@ async function settleTurnFailure(deps) {
|
|
|
5760
5807
|
detail: {
|
|
5761
5808
|
sandboxGroupId: rotation.sandboxGroupId,
|
|
5762
5809
|
leaseEpoch: rotation.leaseEpoch
|
|
5810
|
+
},
|
|
5811
|
+
sandboxLifecycleWait: {
|
|
5812
|
+
version: 1,
|
|
5813
|
+
sandboxGroupId: rotation.sandboxGroupId,
|
|
5814
|
+
leaseEpoch: rotation.leaseEpoch,
|
|
5815
|
+
reason: "rotation_in_progress"
|
|
5763
5816
|
}
|
|
5764
5817
|
});
|
|
5765
5818
|
if (recovery.action === "stale") {
|
|
@@ -5772,10 +5825,7 @@ async function settleTurnFailure(deps) {
|
|
|
5772
5825
|
await publishDurableSessionEvents2(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
5773
5826
|
control.activityStatus = "recovering";
|
|
5774
5827
|
control.turnMetricOutcome = "recovering";
|
|
5775
|
-
return claimedResult({
|
|
5776
|
-
status: "recovering",
|
|
5777
|
-
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings)
|
|
5778
|
-
});
|
|
5828
|
+
return claimedResult({ status: "recovering" });
|
|
5779
5829
|
} catch (recoveryError) {
|
|
5780
5830
|
console.error(
|
|
5781
5831
|
"sandbox deadline rotation recovery failed",
|
|
@@ -5936,7 +5986,7 @@ async function settleTurnFailure(deps) {
|
|
|
5936
5986
|
control.turnMetricOutcome = "recovering";
|
|
5937
5987
|
return claimedResult({ status: "recovering" });
|
|
5938
5988
|
};
|
|
5939
|
-
if (leases.codex.lost
|
|
5989
|
+
if ((leases.codex.lost || error instanceof CodexCredentialLeaseLostError) && billingState.isCodexTurn && eventing.publish && attempt.turnId && eventing.turnStartedPublished && leases.codex.holderId && leases.codex.generation !== null) {
|
|
5940
5990
|
return await settleLostCodexAttempt(
|
|
5941
5991
|
attempt.turnId,
|
|
5942
5992
|
leases.codex.holderId,
|
|
@@ -5966,7 +6016,8 @@ async function settleTurnFailure(deps) {
|
|
|
5966
6016
|
control.turnMetricOutcome = "recovering";
|
|
5967
6017
|
return claimedResult({ status: "recovering" });
|
|
5968
6018
|
}
|
|
5969
|
-
const
|
|
6019
|
+
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
6020
|
+
const codexCredentialFailure = billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId ? classifyCodexCredentialFailure(error) : null;
|
|
5970
6021
|
if (codexCredentialFailure && providerTurn.effectiveCodexCredentialId && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
5971
6022
|
observability.incrementCounter({
|
|
5972
6023
|
name: "opengeni_codex_credential_failures_total",
|
|
@@ -6017,13 +6068,22 @@ async function settleTurnFailure(deps) {
|
|
|
6017
6068
|
} : {}
|
|
6018
6069
|
} : null;
|
|
6019
6070
|
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
6020
|
-
const
|
|
6071
|
+
const quarantineResult = leases.codex.holderId && leases.codex.generation !== null && providerTurn.effectiveCodexCredentialVersion !== null ? await quarantineCodexCredentialForLease(db, {
|
|
6021
6072
|
accountId: input.accountId,
|
|
6022
6073
|
workspaceId: input.workspaceId,
|
|
6074
|
+
sessionId: input.sessionId,
|
|
6023
6075
|
turnId: attempt.turnId,
|
|
6076
|
+
attemptId: input.attemptId,
|
|
6077
|
+
executionGeneration: attempt.executionGeneration,
|
|
6078
|
+
workflowId: input.workflowId,
|
|
6079
|
+
workflowRunId: input.workflowRunId,
|
|
6080
|
+
dispatchId: attempt.dispatchId,
|
|
6081
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6024
6082
|
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
6083
|
+
credentialVersion: providerTurn.effectiveCodexCredentialVersion,
|
|
6025
6084
|
holderId: leases.codex.holderId,
|
|
6026
6085
|
generation: leases.codex.generation,
|
|
6086
|
+
maxFailovers: providerTurn.codexCredentialFailoverLimit,
|
|
6027
6087
|
quarantine: codexCredentialFailure.kind === "auth" ? {
|
|
6028
6088
|
kind: "status",
|
|
6029
6089
|
status: "needs_relogin",
|
|
@@ -6037,7 +6097,42 @@ async function settleTurnFailure(deps) {
|
|
|
6037
6097
|
until: cooldownUntil,
|
|
6038
6098
|
cooldownKind: codexCredentialFailure.kind
|
|
6039
6099
|
}
|
|
6040
|
-
}) :
|
|
6100
|
+
}) : null;
|
|
6101
|
+
if (quarantineResult?.action === "credential_changed" && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6102
|
+
const recovery = await settleCodexCredentialLeaseLoss(db, {
|
|
6103
|
+
accountId: input.accountId,
|
|
6104
|
+
workspaceId: input.workspaceId,
|
|
6105
|
+
sessionId: input.sessionId,
|
|
6106
|
+
turnId: attempt.turnId,
|
|
6107
|
+
attemptId: input.attemptId,
|
|
6108
|
+
holderId: leases.codex.holderId,
|
|
6109
|
+
generation: leases.codex.generation,
|
|
6110
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6111
|
+
checkpointDurable: true,
|
|
6112
|
+
recoveryPayload: {
|
|
6113
|
+
triggerEventId: attempt.triggerEventId,
|
|
6114
|
+
reason: "codex_credential_version_changed"
|
|
6115
|
+
},
|
|
6116
|
+
failedPayload: {}
|
|
6117
|
+
});
|
|
6118
|
+
if (recovery.action === "recovering") {
|
|
6119
|
+
leases.codex.held = false;
|
|
6120
|
+
await publishDurableSessionEvents2(
|
|
6121
|
+
bus,
|
|
6122
|
+
input.workspaceId,
|
|
6123
|
+
input.sessionId,
|
|
6124
|
+
recovery.events
|
|
6125
|
+
);
|
|
6126
|
+
control.activityStatus = "recovering";
|
|
6127
|
+
control.turnMetricOutcome = "recovering";
|
|
6128
|
+
return claimedResult({ status: "recovering" });
|
|
6129
|
+
}
|
|
6130
|
+
acknowledgeLostAttemptOwnership();
|
|
6131
|
+
control.activityStatus = "cancelled";
|
|
6132
|
+
control.turnMetricOutcome = "cancelled";
|
|
6133
|
+
return claimedResult({ status: "cancelled" });
|
|
6134
|
+
}
|
|
6135
|
+
const statePersisted = quarantineResult?.action === "recorded";
|
|
6041
6136
|
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6042
6137
|
leases.codex.lost = true;
|
|
6043
6138
|
return await settleLostCodexAttempt(
|
|
@@ -6047,19 +6142,112 @@ async function settleTurnFailure(deps) {
|
|
|
6047
6142
|
true
|
|
6048
6143
|
);
|
|
6049
6144
|
}
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
|
|
6062
|
-
|
|
6145
|
+
if (quarantineResult?.action === "recorded" && quarantineResult.exhausted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6146
|
+
const settlement = await settleCodexCredentialFailover(db, {
|
|
6147
|
+
accountId: input.accountId,
|
|
6148
|
+
workspaceId: input.workspaceId,
|
|
6149
|
+
sessionId: input.sessionId,
|
|
6150
|
+
turnId: attempt.turnId,
|
|
6151
|
+
attemptId: input.attemptId,
|
|
6152
|
+
holderId: leases.codex.holderId,
|
|
6153
|
+
generation: leases.codex.generation,
|
|
6154
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6155
|
+
maxFailovers: quarantineResult.maxFailovers,
|
|
6156
|
+
recoveryPayload: {
|
|
6157
|
+
triggerEventId: attempt.triggerEventId,
|
|
6158
|
+
reason: "codex_credential_failover",
|
|
6159
|
+
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
6160
|
+
failureKind: codexCredentialFailure.kind
|
|
6161
|
+
},
|
|
6162
|
+
failedPayload: {
|
|
6163
|
+
error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
6164
|
+
code: "codex_credential_failover_exhausted",
|
|
6165
|
+
retryable: false,
|
|
6166
|
+
recovery: "user_message",
|
|
6167
|
+
failoverCount: quarantineResult.failoverCount,
|
|
6168
|
+
maxFailovers: quarantineResult.maxFailovers
|
|
6169
|
+
}
|
|
6170
|
+
});
|
|
6171
|
+
if (settlement.action === "limit_exceeded") {
|
|
6172
|
+
leases.codex.held = false;
|
|
6173
|
+
await publishDurableSessionEvents2(
|
|
6174
|
+
bus,
|
|
6175
|
+
input.workspaceId,
|
|
6176
|
+
input.sessionId,
|
|
6177
|
+
settlement.events
|
|
6178
|
+
);
|
|
6179
|
+
control.activityError = error;
|
|
6180
|
+
control.activityStatus = "idle";
|
|
6181
|
+
control.turnMetricOutcome = "failed";
|
|
6182
|
+
await deliverFailedChildTurnToParent(
|
|
6183
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
6184
|
+
input.workspaceId,
|
|
6185
|
+
input.sessionId,
|
|
6186
|
+
attempt.turnId
|
|
6187
|
+
);
|
|
6188
|
+
return claimedResult({ status: "idle" });
|
|
6189
|
+
}
|
|
6190
|
+
if (settlement.action === "stale") {
|
|
6191
|
+
acknowledgeLostAttemptOwnership();
|
|
6192
|
+
control.activityStatus = "cancelled";
|
|
6193
|
+
control.turnMetricOutcome = "cancelled";
|
|
6194
|
+
return claimedResult({ status: "cancelled" });
|
|
6195
|
+
}
|
|
6196
|
+
throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
|
|
6197
|
+
}
|
|
6198
|
+
let accounts;
|
|
6199
|
+
try {
|
|
6200
|
+
accounts = await listCodexAccountStatuses2(db, input.workspaceId);
|
|
6201
|
+
} catch (metadataError) {
|
|
6202
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
6203
|
+
error: metadataError,
|
|
6204
|
+
turnId: attempt.turnId,
|
|
6205
|
+
triggerEventId: attempt.triggerEventId,
|
|
6206
|
+
executionGeneration: attempt.executionGeneration
|
|
6207
|
+
});
|
|
6208
|
+
if (recoveryFailure) {
|
|
6209
|
+
control.activityStatus = "recovering";
|
|
6210
|
+
control.turnMetricOutcome = "recovering";
|
|
6211
|
+
control.activityError = metadataError;
|
|
6212
|
+
throw recoveryFailure;
|
|
6213
|
+
}
|
|
6214
|
+
throw metadataError;
|
|
6215
|
+
}
|
|
6216
|
+
const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
|
|
6217
|
+
const selected = selectCodexCredentialLeaseForTurn({
|
|
6218
|
+
context: {
|
|
6219
|
+
accounts: codexLeaseAccountsForSelection(accounts),
|
|
6220
|
+
activeCredentialId: acceptedPolicy.activeCredentialId,
|
|
6221
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
6222
|
+
rotationStrategy: acceptedPolicy.rotationStrategy,
|
|
6223
|
+
existingCredentialId: null,
|
|
6224
|
+
failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
|
|
6225
|
+
policyScope: null,
|
|
6226
|
+
unavailableDiagnostics: []
|
|
6227
|
+
},
|
|
6228
|
+
sessionId: input.sessionId,
|
|
6229
|
+
sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
6230
|
+
sessionPinSource: acceptedPolicy.pinSource,
|
|
6231
|
+
sessionLastCredentialId: acceptedPolicy.lastCredentialId,
|
|
6232
|
+
now
|
|
6233
|
+
});
|
|
6234
|
+
const decisionKind = selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
|
|
6235
|
+
const pinDisposition = classifyCodexPin({
|
|
6236
|
+
pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
6237
|
+
pinSource: acceptedPolicy.pinSource,
|
|
6238
|
+
strategy: acceptedPolicy.rotationStrategy,
|
|
6239
|
+
rotationEnabled: acceptedPolicy.rotationEnabled
|
|
6240
|
+
});
|
|
6241
|
+
const failureDisposition = codexDefinitiveFailureDisposition({
|
|
6242
|
+
failureKind: codexCredentialFailure.kind,
|
|
6243
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
6244
|
+
pinDisposition,
|
|
6245
|
+
decisionKind,
|
|
6246
|
+
decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
|
|
6247
|
+
servingCredentialId: providerTurn.effectiveCodexCredentialId
|
|
6248
|
+
});
|
|
6249
|
+
const maxFailovers = providerTurn.codexCredentialFailoverLimit;
|
|
6250
|
+
if (statePersisted && failureDisposition === "failover" && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6063
6251
|
const settlement = await settleCodexCredentialFailover(db, {
|
|
6064
6252
|
accountId: input.accountId,
|
|
6065
6253
|
workspaceId: input.workspaceId,
|
|
@@ -6069,13 +6257,21 @@ async function settleTurnFailure(deps) {
|
|
|
6069
6257
|
holderId: leases.codex.holderId,
|
|
6070
6258
|
generation: leases.codex.generation,
|
|
6071
6259
|
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6072
|
-
maxFailovers
|
|
6260
|
+
maxFailovers,
|
|
6073
6261
|
recoveryPayload: {
|
|
6074
6262
|
triggerEventId: attempt.triggerEventId,
|
|
6075
6263
|
reason: "codex_credential_failover",
|
|
6076
6264
|
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
6077
6265
|
failureKind: codexCredentialFailure.kind,
|
|
6078
6266
|
...cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}
|
|
6267
|
+
},
|
|
6268
|
+
failedPayload: {
|
|
6269
|
+
error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
6270
|
+
code: "codex_credential_failover_exhausted",
|
|
6271
|
+
retryable: false,
|
|
6272
|
+
recovery: "user_message",
|
|
6273
|
+
failoverCount: quarantineResult.failoverCount,
|
|
6274
|
+
maxFailovers: quarantineResult.maxFailovers
|
|
6079
6275
|
}
|
|
6080
6276
|
});
|
|
6081
6277
|
observability.incrementCounter({
|
|
@@ -6111,8 +6307,106 @@ async function settleTurnFailure(deps) {
|
|
|
6111
6307
|
acknowledgeLostAttemptOwnership();
|
|
6112
6308
|
control.activityStatus = "cancelled";
|
|
6113
6309
|
control.turnMetricOutcome = "cancelled";
|
|
6310
|
+
return claimedResult({ status: "cancelled" });
|
|
6311
|
+
}
|
|
6312
|
+
if (settlement.action === "limit_exceeded") {
|
|
6313
|
+
leases.codex.held = false;
|
|
6314
|
+
await publishDurableSessionEvents2(
|
|
6315
|
+
bus,
|
|
6316
|
+
input.workspaceId,
|
|
6317
|
+
input.sessionId,
|
|
6318
|
+
settlement.events
|
|
6319
|
+
);
|
|
6320
|
+
control.activityError = error;
|
|
6321
|
+
control.activityStatus = "idle";
|
|
6322
|
+
control.turnMetricOutcome = "failed";
|
|
6323
|
+
await deliverFailedChildTurnToParent(
|
|
6324
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
6325
|
+
input.workspaceId,
|
|
6326
|
+
input.sessionId,
|
|
6327
|
+
attempt.turnId
|
|
6328
|
+
);
|
|
6329
|
+
return claimedResult({ status: "idle" });
|
|
6330
|
+
}
|
|
6331
|
+
}
|
|
6332
|
+
if (statePersisted && failureDisposition === "wait" && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6333
|
+
let goal;
|
|
6334
|
+
try {
|
|
6335
|
+
goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
6336
|
+
} catch (metadataError) {
|
|
6337
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
6338
|
+
error: metadataError,
|
|
6339
|
+
turnId: attempt.turnId,
|
|
6340
|
+
triggerEventId: attempt.triggerEventId,
|
|
6341
|
+
executionGeneration: attempt.executionGeneration
|
|
6342
|
+
});
|
|
6343
|
+
if (recoveryFailure) {
|
|
6344
|
+
control.activityStatus = "recovering";
|
|
6345
|
+
control.turnMetricOutcome = "recovering";
|
|
6346
|
+
control.activityError = metadataError;
|
|
6347
|
+
throw recoveryFailure;
|
|
6348
|
+
}
|
|
6349
|
+
throw metadataError;
|
|
6350
|
+
}
|
|
6351
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
6352
|
+
const exactProviderReset = codexCredentialFailure.cooldownSeconds !== null && Number.isFinite(codexCredentialFailure.cooldownSeconds) && codexCredentialFailure.cooldownSeconds > 0;
|
|
6353
|
+
const policyCredentialId = pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId ? acceptedPolicy.pinnedCredentialId : !acceptedPolicy.rotationEnabled ? acceptedPolicy.activeCredentialId : null;
|
|
6354
|
+
const capacityAccounts = policyCredentialId ? accounts.filter((account) => account.id === policyCredentialId) : accounts;
|
|
6355
|
+
const authoritativeResetAt = exactProviderReset ? authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil : null;
|
|
6356
|
+
const allAccounts = acceptedPolicy.rotationEnabled && pinDisposition !== "manual" && decisionKind === "allCapped";
|
|
6357
|
+
const failurePayload = codexCapacityWaitFailurePayload({
|
|
6358
|
+
failureKind: codexCredentialFailure.kind,
|
|
6359
|
+
usageLimit,
|
|
6360
|
+
cooldownSeconds: codexCredentialFailure.cooldownSeconds,
|
|
6361
|
+
detail: codexCredentialFailure.kind === "quota" ? error instanceof Error ? error.message : String(error) : "the same accepted turn is waiting for eligible credential capacity",
|
|
6362
|
+
allAccounts
|
|
6363
|
+
});
|
|
6364
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
6365
|
+
{ db, bus },
|
|
6366
|
+
{
|
|
6367
|
+
accountId: input.accountId,
|
|
6368
|
+
workspaceId: input.workspaceId,
|
|
6369
|
+
sessionId: input.sessionId,
|
|
6370
|
+
turnId: attempt.turnId,
|
|
6371
|
+
attemptId: input.attemptId,
|
|
6372
|
+
workflowId: input.workflowId,
|
|
6373
|
+
goalId: activeGoal?.id ?? null,
|
|
6374
|
+
goalVersion: activeGoal?.version ?? null,
|
|
6375
|
+
earliestResetAt: authoritativeResetAt,
|
|
6376
|
+
resetKind: authoritativeResetAt ? "authoritative" : codexCredentialFailure.kind === "auth" || codexCredentialFailure.kind === "forbidden" ? "mutation_only" : "bounded_refresh",
|
|
6377
|
+
failurePayload,
|
|
6378
|
+
leaseFence: {
|
|
6379
|
+
holderId: leases.codex.holderId,
|
|
6380
|
+
generation: leases.codex.generation
|
|
6381
|
+
},
|
|
6382
|
+
expectedRedispatches: attempt.redispatchesAtDispatch
|
|
6383
|
+
},
|
|
6384
|
+
{ onArmed: () => leases.codex.held = false }
|
|
6385
|
+
);
|
|
6386
|
+
control.activityError = error;
|
|
6387
|
+
if (evaluated.action === "resumed") {
|
|
6388
|
+
control.activityStatus = "recovering";
|
|
6389
|
+
control.turnMetricOutcome = "recovering";
|
|
6114
6390
|
return claimedResult({ status: "recovering" });
|
|
6115
6391
|
}
|
|
6392
|
+
if (evaluated.action === "waiting") {
|
|
6393
|
+
control.activityError = error;
|
|
6394
|
+
control.activityStatus = "waiting_capacity";
|
|
6395
|
+
control.turnMetricOutcome = "recovering";
|
|
6396
|
+
return claimedResult({
|
|
6397
|
+
status: "waiting_capacity",
|
|
6398
|
+
capacityWait: {
|
|
6399
|
+
waiterId: evaluated.waiter.id,
|
|
6400
|
+
generation: evaluated.waiter.generation,
|
|
6401
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
6402
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
6403
|
+
}
|
|
6404
|
+
});
|
|
6405
|
+
}
|
|
6406
|
+
acknowledgeLostAttemptOwnership();
|
|
6407
|
+
control.activityStatus = "cancelled";
|
|
6408
|
+
control.turnMetricOutcome = "cancelled";
|
|
6409
|
+
return claimedResult({ status: "cancelled" });
|
|
6116
6410
|
}
|
|
6117
6411
|
}
|
|
6118
6412
|
}
|
|
@@ -6223,201 +6517,20 @@ async function settleTurnFailure(deps) {
|
|
|
6223
6517
|
control.activityError = error;
|
|
6224
6518
|
return claimedResult({ status: "recovering" });
|
|
6225
6519
|
}
|
|
6226
|
-
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
6227
6520
|
if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
6228
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
6229
|
-
const goalActive = Boolean(goal && goal.status === "active");
|
|
6230
6521
|
await flushRuntimeBatcher();
|
|
6231
6522
|
await historySink.reconcileConversationTruth();
|
|
6232
|
-
|
|
6233
|
-
let rotationResumeMs = null;
|
|
6234
|
-
let rotationResumeIdleUntilReset = false;
|
|
6235
|
-
let allCappedResetAt = null;
|
|
6236
|
-
let capacityAuthoritativeResetAt = null;
|
|
6237
|
-
let capacityNeedsBoundedRefresh = false;
|
|
6238
|
-
if (providerTurn.effectiveCodexCredentialId) {
|
|
6239
|
-
const [rotation, sessionCodex] = await Promise.all([
|
|
6240
|
-
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
6241
|
-
getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null)
|
|
6242
|
-
]);
|
|
6243
|
-
const reactiveStrategy = rotation?.rotationStrategy ?? "most_remaining";
|
|
6244
|
-
const reactiveDisposition = classifyCodexPin({
|
|
6245
|
-
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
6246
|
-
pinSource: sessionCodex?.pinSource ?? null,
|
|
6247
|
-
strategy: reactiveStrategy,
|
|
6248
|
-
rotationEnabled: Boolean(rotation?.rotationEnabled)
|
|
6249
|
-
});
|
|
6250
|
-
const reactiveSharded = reactiveDisposition === "sharded";
|
|
6251
|
-
const rotating = Boolean(rotation?.rotationEnabled || rotation?.leaseRotationEnabled) && reactiveDisposition !== "manual";
|
|
6252
|
-
if (rotating && rotation) {
|
|
6253
|
-
const accounts = await listCodexAccountStatuses2(db, input.workspaceId).catch(() => []);
|
|
6254
|
-
const serving = accounts.find((a) => a.id === providerTurn.effectiveCodexCredentialId) ?? null;
|
|
6255
|
-
const until = codexCredentialCooldownUntil(
|
|
6256
|
-
{ kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
|
|
6257
|
-
serving,
|
|
6258
|
-
/* @__PURE__ */ new Date()
|
|
6259
|
-
);
|
|
6260
|
-
const cooldownMutation = await setCodexCredentialExhaustedWithWakeTargets(
|
|
6261
|
-
db,
|
|
6262
|
-
input.workspaceId,
|
|
6263
|
-
providerTurn.effectiveCodexCredentialId,
|
|
6264
|
-
until,
|
|
6265
|
-
"quota"
|
|
6266
|
-
).catch(() => null);
|
|
6267
|
-
const cooldownPersisted = cooldownMutation?.result ?? false;
|
|
6268
|
-
if (cooldownMutation) {
|
|
6269
|
-
await signalCodexCapacityWakeTargets(
|
|
6270
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
6271
|
-
cooldownMutation.wakeTargets
|
|
6272
|
-
);
|
|
6273
|
-
}
|
|
6274
|
-
const fresh = accounts.map(
|
|
6275
|
-
(a) => a.id === providerTurn.effectiveCodexCredentialId ? { ...a, exhaustedUntil: until, exhaustedKind: "quota" } : a
|
|
6276
|
-
);
|
|
6277
|
-
if (reactiveSharded) {
|
|
6278
|
-
const newHome = shardCredentialForSession({
|
|
6279
|
-
sessionId: input.sessionId,
|
|
6280
|
-
accounts: fresh,
|
|
6281
|
-
now: /* @__PURE__ */ new Date()
|
|
6282
|
-
});
|
|
6283
|
-
if (newHome) {
|
|
6284
|
-
rotated = true;
|
|
6285
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
6286
|
-
db,
|
|
6287
|
-
{
|
|
6288
|
-
workspaceId: input.workspaceId,
|
|
6289
|
-
reason: "codex_policy_pin_resharded"
|
|
6290
|
-
},
|
|
6291
|
-
async (tx) => {
|
|
6292
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
6293
|
-
tx,
|
|
6294
|
-
input.workspaceId,
|
|
6295
|
-
input.sessionId,
|
|
6296
|
-
newHome,
|
|
6297
|
-
"policy",
|
|
6298
|
-
{
|
|
6299
|
-
expected: {
|
|
6300
|
-
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
6301
|
-
pinSource: sessionCodex?.pinSource ?? null
|
|
6302
|
-
}
|
|
6303
|
-
}
|
|
6304
|
-
);
|
|
6305
|
-
return { result: changed, changed };
|
|
6306
|
-
}
|
|
6307
|
-
).catch(() => null);
|
|
6308
|
-
if (pinMutation) {
|
|
6309
|
-
await signalCodexCapacityWakeTargets(
|
|
6310
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
6311
|
-
pinMutation.wakeTargets
|
|
6312
|
-
);
|
|
6313
|
-
}
|
|
6314
|
-
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
6315
|
-
db,
|
|
6316
|
-
input.workspaceId,
|
|
6317
|
-
input.sessionId
|
|
6318
|
-
).catch(() => 0);
|
|
6319
|
-
const resume = computeReactiveRotationResume({
|
|
6320
|
-
cooldownPersisted,
|
|
6321
|
-
priorConsecutiveRotations,
|
|
6322
|
-
connectedAccountCount: accounts.length
|
|
6323
|
-
});
|
|
6324
|
-
rotationResumeMs = resume.continueDelayMs;
|
|
6325
|
-
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
6326
|
-
} else {
|
|
6327
|
-
rotated = true;
|
|
6328
|
-
allCappedResetAt = earliestCodexReset(fresh, /* @__PURE__ */ new Date());
|
|
6329
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, /* @__PURE__ */ new Date());
|
|
6330
|
-
capacityNeedsBoundedRefresh = true;
|
|
6331
|
-
}
|
|
6332
|
-
} else {
|
|
6333
|
-
const decision = chooseRotationActive({
|
|
6334
|
-
rotationStrategy: reactiveStrategy,
|
|
6335
|
-
activeCredentialId: rotation.activeCredentialId,
|
|
6336
|
-
priorCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
6337
|
-
accounts: fresh,
|
|
6338
|
-
now: /* @__PURE__ */ new Date()
|
|
6339
|
-
});
|
|
6340
|
-
if (decision.kind === "active") {
|
|
6341
|
-
rotated = true;
|
|
6342
|
-
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
6343
|
-
db,
|
|
6344
|
-
input.workspaceId,
|
|
6345
|
-
input.sessionId
|
|
6346
|
-
).catch(() => 0);
|
|
6347
|
-
const resume = computeReactiveRotationResume({
|
|
6348
|
-
cooldownPersisted,
|
|
6349
|
-
priorConsecutiveRotations,
|
|
6350
|
-
connectedAccountCount: accounts.length
|
|
6351
|
-
});
|
|
6352
|
-
rotationResumeMs = resume.continueDelayMs;
|
|
6353
|
-
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
6354
|
-
} else if (decision.kind === "allCapped") {
|
|
6355
|
-
rotated = true;
|
|
6356
|
-
allCappedResetAt = decision.earliestResetAt;
|
|
6357
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, /* @__PURE__ */ new Date());
|
|
6358
|
-
capacityNeedsBoundedRefresh = true;
|
|
6359
|
-
}
|
|
6360
|
-
}
|
|
6361
|
-
}
|
|
6362
|
-
}
|
|
6363
|
-
const failurePayload = allCappedResetAt ? codexUsageLimitFailurePayload(
|
|
6364
|
-
{
|
|
6365
|
-
resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1e3)
|
|
6366
|
-
},
|
|
6367
|
-
error instanceof Error ? error.message : String(error),
|
|
6368
|
-
{ allAccounts: true }
|
|
6369
|
-
) : codexUsageLimitFailurePayload(
|
|
6523
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
6370
6524
|
usageLimit,
|
|
6371
6525
|
error instanceof Error ? error.message : String(error)
|
|
6372
6526
|
);
|
|
6373
|
-
if (rotationResumeMs === null) {
|
|
6374
|
-
const providerResetAt = capacityAuthoritativeResetAt ?? (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : null);
|
|
6375
|
-
const armed = await armCodexCapacityWait(db, {
|
|
6376
|
-
accountId: input.accountId,
|
|
6377
|
-
workspaceId: input.workspaceId,
|
|
6378
|
-
sessionId: input.sessionId,
|
|
6379
|
-
turnId: attempt.turnId,
|
|
6380
|
-
attemptId: input.attemptId,
|
|
6381
|
-
workflowId: input.workflowId,
|
|
6382
|
-
goalId: goalActive && goal ? goal.id : null,
|
|
6383
|
-
goalVersion: goalActive && goal ? goal.version : null,
|
|
6384
|
-
earliestResetAt: providerResetAt,
|
|
6385
|
-
resetKind: providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
|
|
6386
|
-
failurePayload,
|
|
6387
|
-
...leases.codex.holderId && leases.codex.generation !== null ? {
|
|
6388
|
-
leaseFence: {
|
|
6389
|
-
holderId: leases.codex.holderId,
|
|
6390
|
-
generation: leases.codex.generation
|
|
6391
|
-
},
|
|
6392
|
-
expectedRedispatches: attempt.redispatchesAtDispatch
|
|
6393
|
-
} : {}
|
|
6394
|
-
});
|
|
6395
|
-
if (armed.action === "waiting") {
|
|
6396
|
-
await publishDurableSessionEvents2(bus, input.workspaceId, input.sessionId, armed.events);
|
|
6397
|
-
control.turnMetricOutcome = "recovering";
|
|
6398
|
-
control.activityStatus = "waiting_capacity";
|
|
6399
|
-
control.activityError = error;
|
|
6400
|
-
return claimedResult({
|
|
6401
|
-
status: "waiting_capacity",
|
|
6402
|
-
capacityWait: {
|
|
6403
|
-
waiterId: armed.waiter.id,
|
|
6404
|
-
generation: armed.waiter.generation,
|
|
6405
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
6406
|
-
wakeRevision: armed.waiter.wakeRevision
|
|
6407
|
-
}
|
|
6408
|
-
});
|
|
6409
|
-
}
|
|
6410
|
-
}
|
|
6411
6527
|
if (!await eventing.settle({
|
|
6412
6528
|
events: [
|
|
6413
|
-
// `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
|
|
6414
|
-
// freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
|
|
6415
6529
|
{
|
|
6416
6530
|
type: "turn.failed",
|
|
6417
6531
|
payload: {
|
|
6418
6532
|
...failurePayload,
|
|
6419
|
-
recovery:
|
|
6420
|
-
...rotated ? { rotated: true } : {}
|
|
6533
|
+
recovery: "user_message"
|
|
6421
6534
|
}
|
|
6422
6535
|
},
|
|
6423
6536
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
@@ -6431,21 +6544,6 @@ async function settleTurnFailure(deps) {
|
|
|
6431
6544
|
control.turnMetricOutcome = "failed";
|
|
6432
6545
|
control.activityStatus = "idle";
|
|
6433
6546
|
control.activityError = error;
|
|
6434
|
-
if (goalActive) {
|
|
6435
|
-
if (rotationResumeMs !== null) {
|
|
6436
|
-
return claimedResult({
|
|
6437
|
-
status: "idle",
|
|
6438
|
-
continueDelayMs: rotationResumeMs,
|
|
6439
|
-
...rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}
|
|
6440
|
-
});
|
|
6441
|
-
}
|
|
6442
|
-
const resumeMs = allCappedResetAt ? computeIdleDelayMs(allCappedResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS) : usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1e3, CODEX_USAGE_LIMIT_MAX_RESUME_MS) : CODEX_USAGE_LIMIT_MAX_RESUME_MS;
|
|
6443
|
-
return claimedResult({
|
|
6444
|
-
status: "idle",
|
|
6445
|
-
continueDelayMs: resumeMs,
|
|
6446
|
-
...allCappedResetAt ? { idleUntilReset: true } : {}
|
|
6447
|
-
});
|
|
6448
|
-
}
|
|
6449
6547
|
return claimedResult({ status: "idle" });
|
|
6450
6548
|
}
|
|
6451
6549
|
if (error instanceof BudgetExhaustedError && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
@@ -6690,6 +6788,7 @@ import {
|
|
|
6690
6788
|
clearDurablePendingSessionToolCalls,
|
|
6691
6789
|
isSessionCompactionRequested,
|
|
6692
6790
|
nextSessionHistoryPosition as nextSessionHistoryPosition2,
|
|
6791
|
+
persistModelContextSnapshot,
|
|
6693
6792
|
updateSessionTitleWithEvent
|
|
6694
6793
|
} from "@opengeni/db";
|
|
6695
6794
|
import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
|
|
@@ -6726,7 +6825,7 @@ import {
|
|
|
6726
6825
|
latestCompactionReplacementFingerprint,
|
|
6727
6826
|
prepareCompactionPromptInput,
|
|
6728
6827
|
projectRemoteCompactionOverflowRetryInput,
|
|
6729
|
-
sanitizeHistoryItemsForModel as
|
|
6828
|
+
sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel3,
|
|
6730
6829
|
summarizeForCompaction
|
|
6731
6830
|
} from "@opengeni/runtime";
|
|
6732
6831
|
import { contextInputBudgetTokens } from "@opengeni/config";
|
|
@@ -6737,7 +6836,7 @@ import {
|
|
|
6737
6836
|
FileResourceRef,
|
|
6738
6837
|
resourceMountPath as resourceMountPath2
|
|
6739
6838
|
} from "@opengeni/contracts";
|
|
6740
|
-
import { createHash as
|
|
6839
|
+
import { createHash as createHash8 } from "crypto";
|
|
6741
6840
|
import {
|
|
6742
6841
|
getActiveSessionHistoryItemsPaged as getActiveSessionHistoryItemsPaged2,
|
|
6743
6842
|
getFilesForSubject as getFilesForSubject2,
|
|
@@ -6846,7 +6945,7 @@ async function modelAttachmentContentForFiles(files, readFileBytes) {
|
|
|
6846
6945
|
});
|
|
6847
6946
|
continue;
|
|
6848
6947
|
}
|
|
6849
|
-
if (
|
|
6948
|
+
if (createHash8("sha256").update(bytes).digest("hex") !== checksum) {
|
|
6850
6949
|
console.error("model attachment checksum did not match finalized metadata", {
|
|
6851
6950
|
fileId: file.id
|
|
6852
6951
|
});
|
|
@@ -6975,13 +7074,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
6975
7074
|
options.turnId
|
|
6976
7075
|
)
|
|
6977
7076
|
);
|
|
6978
|
-
if (updates.
|
|
6979
|
-
|
|
6980
|
-
updates.map((update) => update.deliveredHistoryItemId).filter(Boolean)
|
|
6981
|
-
);
|
|
6982
|
-
if (historyItemIds.size !== 1 || updates.some((update) => !update.deliveredHistoryItemId)) {
|
|
6983
|
-
throw new Error("Delivered internal updates have no single durable model-memory batch");
|
|
6984
|
-
}
|
|
7077
|
+
if (updates.some((update) => !update.deliveredHistoryItemId)) {
|
|
7078
|
+
throw new Error("Delivered internal update has no durable model-memory batch");
|
|
6985
7079
|
}
|
|
6986
7080
|
const internalContext = joinInternalContext(
|
|
6987
7081
|
options.recovering ? [
|
|
@@ -7245,7 +7339,7 @@ async function maybeCompactContext(db, settings, scope, lastInputTokens, summari
|
|
|
7245
7339
|
const canonicalItems = projectRejectedProviderArtifacts(active);
|
|
7246
7340
|
const projectForWire = async (input) => {
|
|
7247
7341
|
const materialized = options.materializeHistory ? await options.materializeHistory(input) : input;
|
|
7248
|
-
return
|
|
7342
|
+
return sanitizeHistoryItemsForModel3(
|
|
7249
7343
|
options.projectModelInput ? await options.projectModelInput(materialized) : materialized,
|
|
7250
7344
|
settings.modelToolOutputTruncationTokens
|
|
7251
7345
|
);
|
|
@@ -7531,7 +7625,7 @@ import {
|
|
|
7531
7625
|
import {
|
|
7532
7626
|
functionCallResultItem,
|
|
7533
7627
|
invokePreparedAgentTool,
|
|
7534
|
-
sanitizeHistoryItemsForModel as
|
|
7628
|
+
sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel4
|
|
7535
7629
|
} from "@opengeni/runtime";
|
|
7536
7630
|
function argumentsJsonFromCallItem(callItem) {
|
|
7537
7631
|
const raw = callItem.arguments;
|
|
@@ -7561,7 +7655,7 @@ function remainingRunStatePendingApprovalsFromSuffix(rows) {
|
|
|
7561
7655
|
return remainingSuffixApprovals(rows, /* @__PURE__ */ new Set(["approval", "interaction_intervention"]));
|
|
7562
7656
|
}
|
|
7563
7657
|
function openSuffixHistoryItems(row, resultItem) {
|
|
7564
|
-
return
|
|
7658
|
+
return sanitizeHistoryItemsForModel4([
|
|
7565
7659
|
...row.tiedReasoningItems,
|
|
7566
7660
|
row.callItem,
|
|
7567
7661
|
resultItem
|
|
@@ -8269,7 +8363,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8269
8363
|
});
|
|
8270
8364
|
runInput = prepared.input;
|
|
8271
8365
|
historySink.providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
8272
|
-
historySink.
|
|
8366
|
+
historySink.seedHistory(prepared.input.input, prepared.persistedHistoryCount);
|
|
8273
8367
|
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
8274
8368
|
const historyPositionStartedAt = performance.now();
|
|
8275
8369
|
let historyPositionOutcome = "completed";
|
|
@@ -8520,6 +8614,17 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8520
8614
|
return await runtime.runStream(agent, runInput, eventing.modelRunSettings, {
|
|
8521
8615
|
signal: runtimeCancellationSignal,
|
|
8522
8616
|
sandboxEnvironment,
|
|
8617
|
+
onModelVisibleContext: async (snapshot) => {
|
|
8618
|
+
await persistModelContextSnapshot(db, {
|
|
8619
|
+
accountId: input.accountId,
|
|
8620
|
+
workspaceId: input.workspaceId,
|
|
8621
|
+
sessionId: input.sessionId,
|
|
8622
|
+
turnId: activeTurnId,
|
|
8623
|
+
attemptId: input.attemptId,
|
|
8624
|
+
executionGeneration: attempt.executionGeneration,
|
|
8625
|
+
snapshot
|
|
8626
|
+
});
|
|
8627
|
+
},
|
|
8523
8628
|
onRuntimeEvent: async (event) => {
|
|
8524
8629
|
await leases.renewServing("runtime_event");
|
|
8525
8630
|
if (leases.servingLost()) {
|
|
@@ -8605,7 +8710,9 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8605
8710
|
});
|
|
8606
8711
|
}
|
|
8607
8712
|
};
|
|
8608
|
-
if (
|
|
8713
|
+
if (billingState.isCodexTurn) {
|
|
8714
|
+
leases.codex.assertUsable();
|
|
8715
|
+
} else if (leases.codex.lost) {
|
|
8609
8716
|
throw new Error("Codex credential lease expired before the model run");
|
|
8610
8717
|
}
|
|
8611
8718
|
if (leases.xai.lost) {
|
|
@@ -8657,6 +8764,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8657
8764
|
]);
|
|
8658
8765
|
};
|
|
8659
8766
|
const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
|
|
8767
|
+
const closeStreamWaitAdmission = eventing.preparedTools?.inputWaitYield?.captureStreamClose();
|
|
8660
8768
|
let streamDone = false;
|
|
8661
8769
|
try {
|
|
8662
8770
|
while (true) {
|
|
@@ -8957,6 +9065,7 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8957
9065
|
}
|
|
8958
9066
|
}
|
|
8959
9067
|
} catch (error) {
|
|
9068
|
+
closeStreamWaitAdmission?.();
|
|
8960
9069
|
if (fallbackProviderRequestLifecycleStartedAt !== null) {
|
|
8961
9070
|
const durationMs = Math.max(
|
|
8962
9071
|
0,
|
|
@@ -9015,7 +9124,8 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9015
9124
|
temporalCancellationSignal: cancellationSignal,
|
|
9016
9125
|
runtimeCancellationSignal
|
|
9017
9126
|
});
|
|
9018
|
-
|
|
9127
|
+
await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
|
|
9128
|
+
if (options.requireTerminalModelResponse && !eventing.preparedTools?.inputWaitYield?.yielded && eventing.stream.interruptions.length === 0 && modelResponseState.responseCount === responseCountBeforeStream) {
|
|
9019
9129
|
throwIfWorkerShuttingDown();
|
|
9020
9130
|
throwIfTurnCancelled();
|
|
9021
9131
|
throw new PostCompactionContinuationEmptyError();
|
|
@@ -9239,16 +9349,16 @@ async function runTurnStreamAttempt(deps) {
|
|
|
9239
9349
|
control.activityStatus = "requires_action";
|
|
9240
9350
|
return claimedResult({ status: "requires_action" });
|
|
9241
9351
|
}
|
|
9242
|
-
const
|
|
9352
|
+
const inputWaitYielded = eventing.preparedTools?.inputWaitYield?.yielded === true;
|
|
9353
|
+
const finalOutput = String(
|
|
9354
|
+
requireAgentStreamFinalOutput(eventing.stream.finalOutput, inputWaitYielded)
|
|
9355
|
+
);
|
|
9243
9356
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
9244
9357
|
await finalizeTurnOpStreamOps();
|
|
9245
9358
|
await finishParallelSessionTitle();
|
|
9246
9359
|
if (!await eventing.settle({
|
|
9247
9360
|
events: [
|
|
9248
|
-
{
|
|
9249
|
-
type: "agent.message.completed",
|
|
9250
|
-
payload: { text: finalOutput }
|
|
9251
|
-
},
|
|
9361
|
+
...inputWaitYielded ? [] : [{ type: "agent.message.completed", payload: { text: finalOutput } }],
|
|
9252
9362
|
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
9253
9363
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
9254
9364
|
],
|
|
@@ -9481,7 +9591,7 @@ import {
|
|
|
9481
9591
|
|
|
9482
9592
|
// src/activities/capabilities.ts
|
|
9483
9593
|
import {
|
|
9484
|
-
environmentsEncryptionKeyBytes
|
|
9594
|
+
environmentsEncryptionKeyBytes,
|
|
9485
9595
|
WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
|
|
9486
9596
|
WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
|
|
9487
9597
|
withCodexCatalogProvider,
|
|
@@ -9513,7 +9623,7 @@ import {
|
|
|
9513
9623
|
loadOrganizationModelProviderApiKey
|
|
9514
9624
|
} from "@opengeni/db";
|
|
9515
9625
|
async function settingsWithSessionMcpServersForRun(db, workspaceId, sessionId, attemptId, settings, options) {
|
|
9516
|
-
const encryptionKey =
|
|
9626
|
+
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
9517
9627
|
if (!encryptionKey) {
|
|
9518
9628
|
const metadata = await listSessionMcpServerMetadata(db, workspaceId, sessionId);
|
|
9519
9629
|
if (metadata.length === 0) {
|
|
@@ -9664,6 +9774,13 @@ import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
|
9664
9774
|
function turnAttemptProducerId(input, turnId) {
|
|
9665
9775
|
return `${input.workflowId}:${turnId}:${input.attemptId}`;
|
|
9666
9776
|
}
|
|
9777
|
+
function codexCredentialLeaseHolderId(input, turnId) {
|
|
9778
|
+
const parts = [input.workflowId, turnId, input.attemptId];
|
|
9779
|
+
if (parts.some((part) => part.trim().length === 0)) {
|
|
9780
|
+
throw new Error("Codex credential lease holder requires a complete turn attempt identity");
|
|
9781
|
+
}
|
|
9782
|
+
return `codex-turn:${parts.map((part) => encodeURIComponent(part)).join(":")}`;
|
|
9783
|
+
}
|
|
9667
9784
|
async function claimTurnAttempt(deps) {
|
|
9668
9785
|
const {
|
|
9669
9786
|
input,
|
|
@@ -9712,9 +9829,11 @@ async function claimTurnAttempt(deps) {
|
|
|
9712
9829
|
}
|
|
9713
9830
|
const turn = claim.turn;
|
|
9714
9831
|
attempt.turnId = turn.id;
|
|
9832
|
+
attempt.dispatchId = dispatchId;
|
|
9715
9833
|
attempt.executionGeneration = turn.executionGeneration;
|
|
9716
9834
|
attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
9717
9835
|
attempt.triggerEventId = turn.triggerEventId;
|
|
9836
|
+
leases.codex.holderId = codexCredentialLeaseHolderId(input, turn.id);
|
|
9718
9837
|
attempt.redispatchesAtDispatch = Number(
|
|
9719
9838
|
turn.metadata?.workerDeathRedispatches ?? 0
|
|
9720
9839
|
);
|
|
@@ -9863,7 +9982,6 @@ async function claimTurnAttempt(deps) {
|
|
|
9863
9982
|
eventing.heartbeatTimer = startActivityHeartbeat(activityContext, heartbeatDetails);
|
|
9864
9983
|
let producerSeq = 0;
|
|
9865
9984
|
const producerId = turnAttemptProducerId(input, attempt.turnId);
|
|
9866
|
-
leases.codex.holderId = dispatchId;
|
|
9867
9985
|
const modelUsageDispatchId = input.attemptId;
|
|
9868
9986
|
const claimedModelUsageSourceKeys = /* @__PURE__ */ new Set();
|
|
9869
9987
|
const emittedModelUsageSourceKeys = /* @__PURE__ */ new Set();
|
|
@@ -9949,6 +10067,7 @@ async function claimTurnAttempt(deps) {
|
|
|
9949
10067
|
turnStatus: inputSettlement.turnStatus,
|
|
9950
10068
|
sessionStatus: inputSettlement.sessionStatus,
|
|
9951
10069
|
activeTurnId: inputSettlement.activeTurnId,
|
|
10070
|
+
...inputSettlement.suppressGoalContinuation !== void 0 ? { suppressGoalContinuation: inputSettlement.suppressGoalContinuation } : {},
|
|
9952
10071
|
events: inputs,
|
|
9953
10072
|
...runState ? { runState } : {},
|
|
9954
10073
|
...compactionRequestFailure ? { compactionRequestFailure } : {}
|
|
@@ -10042,16 +10161,15 @@ async function claimTurnAttempt(deps) {
|
|
|
10042
10161
|
import {
|
|
10043
10162
|
getSessionGoal as getSessionGoal2,
|
|
10044
10163
|
acquireCodexCredentialLease,
|
|
10045
|
-
|
|
10164
|
+
CodexCredentialLeaseAttemptFencedError,
|
|
10165
|
+
CodexCredentialFailoverExhaustedError,
|
|
10046
10166
|
CODEX_CREDENTIAL_LEASE_TTL_MS as CODEX_CREDENTIAL_LEASE_TTL_MS2,
|
|
10047
|
-
getCodexRotationSettings as getCodexRotationSettings2,
|
|
10048
|
-
listCodexAccountStatuses as listCodexAccountStatuses3,
|
|
10049
|
-
getSessionCodexState as getSessionCodexState2,
|
|
10050
10167
|
recordSessionActiveCodexCredential,
|
|
10051
|
-
setSessionCodexPinInTransaction
|
|
10052
|
-
|
|
10053
|
-
withSessionCodexCapacityMutation
|
|
10168
|
+
setSessionCodexPinInTransaction,
|
|
10169
|
+
settleCodexCredentialFailover as settleCodexCredentialFailover2,
|
|
10170
|
+
withSessionCodexCapacityMutation
|
|
10054
10171
|
} from "@opengeni/db";
|
|
10172
|
+
import { CodexReloginRequired } from "@opengeni/codex";
|
|
10055
10173
|
import { publishDurableSessionEvents as publishDurableSessionEvents5 } from "@opengeni/events";
|
|
10056
10174
|
|
|
10057
10175
|
// src/activities/codex-fleet-shadow.ts
|
|
@@ -10255,6 +10373,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10255
10373
|
observability,
|
|
10256
10374
|
wakeSessionWorkflow,
|
|
10257
10375
|
signalCodexCapacityWorkflow,
|
|
10376
|
+
dispatchId,
|
|
10258
10377
|
control,
|
|
10259
10378
|
attempt,
|
|
10260
10379
|
billingState,
|
|
@@ -10262,6 +10381,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10262
10381
|
providerTurn,
|
|
10263
10382
|
leases,
|
|
10264
10383
|
claimedResult,
|
|
10384
|
+
acknowledgeLostAttemptOwnership,
|
|
10265
10385
|
turn,
|
|
10266
10386
|
codexWorkspaceKey
|
|
10267
10387
|
} = deps;
|
|
@@ -10277,123 +10397,62 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10277
10397
|
const credentialSelectionStartedAt = performance.now();
|
|
10278
10398
|
let credentialSelectionOutcome = "completed";
|
|
10279
10399
|
try {
|
|
10280
|
-
const
|
|
10281
|
-
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
10282
|
-
const sessionPinSource = sessionCodex?.pinSource ?? null;
|
|
10283
|
-
const selectForTurn = (context) => selectCodexCredentialLeaseForTurn({
|
|
10400
|
+
const selectForTurn = (context, lockedSessionCodexState2) => selectCodexCredentialLeaseForTurn({
|
|
10284
10401
|
context,
|
|
10285
|
-
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
10286
10402
|
sessionId: input.sessionId,
|
|
10287
|
-
sessionPinnedCredentialId:
|
|
10288
|
-
sessionPinSource,
|
|
10289
|
-
sessionLastCredentialId:
|
|
10403
|
+
sessionPinnedCredentialId: lockedSessionCodexState2.pinnedCredentialId,
|
|
10404
|
+
sessionPinSource: lockedSessionCodexState2.pinSource,
|
|
10405
|
+
sessionLastCredentialId: lockedSessionCodexState2.lastCredentialId,
|
|
10290
10406
|
now: /* @__PURE__ */ new Date()
|
|
10291
10407
|
});
|
|
10292
|
-
let
|
|
10293
|
-
let
|
|
10294
|
-
|
|
10408
|
+
let leaseAcquisitionStartedAtMs = performance.now();
|
|
10409
|
+
let leased = await acquireCodexCredentialLease(
|
|
10410
|
+
db,
|
|
10411
|
+
{
|
|
10412
|
+
accountId: input.accountId,
|
|
10413
|
+
workspaceId: input.workspaceId,
|
|
10414
|
+
sessionId: input.sessionId,
|
|
10415
|
+
turnId,
|
|
10416
|
+
attemptId: input.attemptId,
|
|
10417
|
+
executionGeneration: attempt.executionGeneration,
|
|
10418
|
+
workflowId: input.workflowId,
|
|
10419
|
+
workflowRunId: input.workflowRunId,
|
|
10420
|
+
dispatchId,
|
|
10421
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
10422
|
+
holderId,
|
|
10423
|
+
advanceActivePointer: true
|
|
10424
|
+
},
|
|
10425
|
+
selectForTurn
|
|
10426
|
+
);
|
|
10427
|
+
if (leased.decision.kind === "allCapped") {
|
|
10428
|
+
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
10429
|
+
signalCodexCapacityWorkflow,
|
|
10430
|
+
wakeSessionWorkflow
|
|
10431
|
+
});
|
|
10295
10432
|
leaseAcquisitionStartedAtMs = performance.now();
|
|
10296
10433
|
leased = await acquireCodexCredentialLease(
|
|
10297
10434
|
db,
|
|
10298
10435
|
{
|
|
10299
10436
|
accountId: input.accountId,
|
|
10300
10437
|
workspaceId: input.workspaceId,
|
|
10438
|
+
sessionId: input.sessionId,
|
|
10301
10439
|
turnId,
|
|
10440
|
+
attemptId: input.attemptId,
|
|
10441
|
+
executionGeneration: attempt.executionGeneration,
|
|
10442
|
+
workflowId: input.workflowId,
|
|
10443
|
+
workflowRunId: input.workflowRunId,
|
|
10444
|
+
dispatchId,
|
|
10445
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
10302
10446
|
holderId,
|
|
10303
|
-
advanceActivePointer:
|
|
10447
|
+
advanceActivePointer: true
|
|
10304
10448
|
},
|
|
10305
10449
|
selectForTurn
|
|
10306
10450
|
);
|
|
10307
|
-
} else {
|
|
10308
|
-
const [rotation, accounts] = await Promise.all([
|
|
10309
|
-
getCodexRotationSettings2(db, input.workspaceId),
|
|
10310
|
-
listCodexAccountStatuses3(db, input.workspaceId)
|
|
10311
|
-
]);
|
|
10312
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
10313
|
-
...account,
|
|
10314
|
-
activeLeaseCount: 0,
|
|
10315
|
-
selectionCount: 0,
|
|
10316
|
-
lastSelectedAt: null
|
|
10317
|
-
}));
|
|
10318
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
10319
|
-
const selected = selectForTurn({
|
|
10320
|
-
accounts: leaseAccounts,
|
|
10321
|
-
activeCredentialId,
|
|
10322
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10323
|
-
leaseRotationEnabled: false,
|
|
10324
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10325
|
-
existingCredentialId: null,
|
|
10326
|
-
policyScope: null,
|
|
10327
|
-
unavailableDiagnostics: []
|
|
10328
|
-
});
|
|
10329
|
-
leased = {
|
|
10330
|
-
...selected,
|
|
10331
|
-
accounts: leaseAccounts,
|
|
10332
|
-
activeCredentialId,
|
|
10333
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10334
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10335
|
-
reused: false,
|
|
10336
|
-
holderId: null,
|
|
10337
|
-
generation: null,
|
|
10338
|
-
leasedUntil: null,
|
|
10339
|
-
unavailableDiagnostics: [],
|
|
10340
|
-
advanceActivePointer: selected.advanceActivePointer !== false
|
|
10341
|
-
};
|
|
10342
|
-
}
|
|
10343
|
-
if (leased.decision.kind === "allCapped") {
|
|
10344
|
-
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
10345
|
-
signalCodexCapacityWorkflow,
|
|
10346
|
-
wakeSessionWorkflow
|
|
10347
|
-
});
|
|
10348
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
10349
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
10350
|
-
leased = await acquireCodexCredentialLease(
|
|
10351
|
-
db,
|
|
10352
|
-
{
|
|
10353
|
-
accountId: input.accountId,
|
|
10354
|
-
workspaceId: input.workspaceId,
|
|
10355
|
-
turnId,
|
|
10356
|
-
holderId,
|
|
10357
|
-
advanceActivePointer: sessionPin === null
|
|
10358
|
-
},
|
|
10359
|
-
selectForTurn
|
|
10360
|
-
);
|
|
10361
|
-
} else {
|
|
10362
|
-
const [rotation, accounts] = await Promise.all([
|
|
10363
|
-
getCodexRotationSettings2(db, input.workspaceId),
|
|
10364
|
-
listCodexAccountStatuses3(db, input.workspaceId)
|
|
10365
|
-
]);
|
|
10366
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
10367
|
-
...account,
|
|
10368
|
-
activeLeaseCount: 0,
|
|
10369
|
-
selectionCount: 0,
|
|
10370
|
-
lastSelectedAt: null
|
|
10371
|
-
}));
|
|
10372
|
-
const selected = selectForTurn({
|
|
10373
|
-
accounts: leaseAccounts,
|
|
10374
|
-
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
10375
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10376
|
-
leaseRotationEnabled: false,
|
|
10377
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10378
|
-
existingCredentialId: null,
|
|
10379
|
-
policyScope: null,
|
|
10380
|
-
unavailableDiagnostics: []
|
|
10381
|
-
});
|
|
10382
|
-
leased = {
|
|
10383
|
-
...selected,
|
|
10384
|
-
accounts: leaseAccounts,
|
|
10385
|
-
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
10386
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10387
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10388
|
-
reused: false,
|
|
10389
|
-
holderId: null,
|
|
10390
|
-
generation: null,
|
|
10391
|
-
leasedUntil: null,
|
|
10392
|
-
unavailableDiagnostics: [],
|
|
10393
|
-
advanceActivePointer: selected.advanceActivePointer !== false
|
|
10394
|
-
};
|
|
10395
|
-
}
|
|
10396
10451
|
}
|
|
10452
|
+
const lockedSessionCodexState = leased.sessionCodexState;
|
|
10453
|
+
providerTurn.codexPolicySnapshot = leased.codexPolicySnapshot;
|
|
10454
|
+
const sessionPin = lockedSessionCodexState.pinnedCredentialId;
|
|
10455
|
+
const sessionPinSource = lockedSessionCodexState.pinSource;
|
|
10397
10456
|
const rotationDecision = leased.decision;
|
|
10398
10457
|
const selectedPinDisposition = classifyCodexPin({
|
|
10399
10458
|
pinnedCredentialId: sessionPin,
|
|
@@ -10401,15 +10460,15 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10401
10460
|
strategy: leased.rotationStrategy,
|
|
10402
10461
|
rotationEnabled: leased.rotationEnabled
|
|
10403
10462
|
});
|
|
10404
|
-
if (selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
|
|
10405
|
-
const pinMutation = await
|
|
10463
|
+
if (!leased.codexPolicySnapshotReused && selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
|
|
10464
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
10406
10465
|
db,
|
|
10407
10466
|
{
|
|
10408
10467
|
workspaceId: input.workspaceId,
|
|
10409
10468
|
reason: "codex_policy_pin_changed"
|
|
10410
10469
|
},
|
|
10411
10470
|
async (tx) => {
|
|
10412
|
-
const changed = await
|
|
10471
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
10413
10472
|
tx,
|
|
10414
10473
|
input.workspaceId,
|
|
10415
10474
|
input.sessionId,
|
|
@@ -10430,14 +10489,14 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10430
10489
|
pinMutation.wakeTargets
|
|
10431
10490
|
);
|
|
10432
10491
|
} else if (selectedPinDisposition === "clearStale") {
|
|
10433
|
-
const pinMutation = await
|
|
10492
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
10434
10493
|
db,
|
|
10435
10494
|
{
|
|
10436
10495
|
workspaceId: input.workspaceId,
|
|
10437
10496
|
reason: "codex_stale_policy_pin_cleared"
|
|
10438
10497
|
},
|
|
10439
10498
|
async (tx) => {
|
|
10440
|
-
const changed = await
|
|
10499
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
10441
10500
|
tx,
|
|
10442
10501
|
input.workspaceId,
|
|
10443
10502
|
input.sessionId,
|
|
@@ -10458,16 +10517,14 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10458
10517
|
pinMutation.wakeTargets
|
|
10459
10518
|
);
|
|
10460
10519
|
}
|
|
10461
|
-
if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
|
|
10462
|
-
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
10463
|
-
}
|
|
10464
10520
|
providerTurn.effectiveCodexCredentialId = leased.credentialId;
|
|
10521
|
+
providerTurn.codexCredentialFailoverLimit = leased.failoverLimit;
|
|
10465
10522
|
leases.codex.generation = leased.generation;
|
|
10466
10523
|
leases.codex.confirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS2 : null;
|
|
10467
10524
|
leases.codex.held = providerTurn.effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && leases.codex.confirmedUntilMs !== null;
|
|
10468
10525
|
if (leases.codex.held) leases.codex.startHeartbeat();
|
|
10469
|
-
const actualOutcome = providerTurn.effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
|
|
10470
|
-
const actualReason = providerTurn.effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === providerTurn.effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
|
|
10526
|
+
const actualOutcome = providerTurn.effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" || rotationDecision.kind === "allocatorDisabled" ? "waiting" : "none";
|
|
10527
|
+
const actualReason = providerTurn.effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === providerTurn.effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : rotationDecision.kind === "allocatorDisabled" ? "allocator_disabled" : "none";
|
|
10471
10528
|
const fencedInFlight = leased.reused;
|
|
10472
10529
|
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
10473
10530
|
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
@@ -10476,7 +10533,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10476
10533
|
actualCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
10477
10534
|
actualOutcome,
|
|
10478
10535
|
actualReason,
|
|
10479
|
-
affinityCredentialId: fencedInFlight ? providerTurn.effectiveCodexCredentialId : sessionPin ??
|
|
10536
|
+
affinityCredentialId: fencedInFlight ? providerTurn.effectiveCodexCredentialId : sessionPin ?? lockedSessionCodexState.lastCredentialId ?? null,
|
|
10480
10537
|
fencedInFlight,
|
|
10481
10538
|
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
10482
10539
|
now: /* @__PURE__ */ new Date(),
|
|
@@ -10540,7 +10597,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10540
10597
|
depth: poolDepth
|
|
10541
10598
|
});
|
|
10542
10599
|
}
|
|
10543
|
-
if (providerTurn.effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
|
|
10600
|
+
if (providerTurn.effectiveCodexCredentialId === null && leased.accounts.length > 0 && (rotationDecision.kind === "allocatorDisabled" || leased.accounts.every((account) => !account.allocatorEnabled)) && turnId) {
|
|
10544
10601
|
if (turn.source === "compaction") {
|
|
10545
10602
|
if (!await eventing.settle({
|
|
10546
10603
|
events: [
|
|
@@ -10567,66 +10624,146 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10567
10624
|
control.activityStatus = "idle";
|
|
10568
10625
|
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
10569
10626
|
}
|
|
10570
|
-
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId)
|
|
10627
|
+
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
|
|
10571
10628
|
const activeGoal = goal?.status === "active" ? goal : null;
|
|
10572
|
-
const
|
|
10573
|
-
|
|
10574
|
-
|
|
10575
|
-
|
|
10576
|
-
|
|
10577
|
-
|
|
10578
|
-
|
|
10579
|
-
|
|
10580
|
-
|
|
10581
|
-
|
|
10582
|
-
|
|
10583
|
-
|
|
10584
|
-
|
|
10585
|
-
|
|
10586
|
-
|
|
10629
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
10630
|
+
{ db, bus },
|
|
10631
|
+
{
|
|
10632
|
+
accountId: input.accountId,
|
|
10633
|
+
workspaceId: input.workspaceId,
|
|
10634
|
+
sessionId: input.sessionId,
|
|
10635
|
+
turnId,
|
|
10636
|
+
attemptId: input.attemptId,
|
|
10637
|
+
workflowId: input.workflowId,
|
|
10638
|
+
goalId: activeGoal?.id ?? null,
|
|
10639
|
+
goalVersion: activeGoal?.version ?? null,
|
|
10640
|
+
earliestResetAt: null,
|
|
10641
|
+
resetKind: "mutation_only",
|
|
10642
|
+
failurePayload: {
|
|
10643
|
+
error: rotationDecision.kind === "allocatorDisabled" ? "The policy-selected Codex subscription is disabled for new allocations." : "All connected Codex subscriptions are disabled for new allocations.",
|
|
10644
|
+
code: "codex_allocator_disabled",
|
|
10645
|
+
detail: rotationDecision.kind === "allocatorDisabled" ? "waiting for the selected credential to be re-enabled under the accepted source policy" : "waiting for a credential in the accepted source pool to be re-enabled, reconnected, or added"
|
|
10646
|
+
}
|
|
10587
10647
|
}
|
|
10588
|
-
|
|
10589
|
-
if (
|
|
10590
|
-
|
|
10648
|
+
);
|
|
10649
|
+
if (evaluated.action === "resumed") {
|
|
10650
|
+
control.turnMetricOutcome = "recovering";
|
|
10651
|
+
control.activityStatus = "recovering";
|
|
10652
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
10653
|
+
}
|
|
10654
|
+
if (evaluated.action === "waiting") {
|
|
10591
10655
|
control.turnMetricOutcome = "recovering";
|
|
10592
10656
|
control.activityStatus = "waiting_capacity";
|
|
10593
10657
|
return {
|
|
10594
10658
|
exit: claimedResult({
|
|
10595
10659
|
status: "waiting_capacity",
|
|
10596
10660
|
capacityWait: {
|
|
10597
|
-
waiterId:
|
|
10598
|
-
generation:
|
|
10599
|
-
nextCheckAt:
|
|
10600
|
-
wakeRevision:
|
|
10661
|
+
waiterId: evaluated.waiter.id,
|
|
10662
|
+
generation: evaluated.waiter.generation,
|
|
10663
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10664
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
10601
10665
|
}
|
|
10602
10666
|
})
|
|
10603
10667
|
};
|
|
10604
10668
|
}
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
|
|
10613
|
-
|
|
10669
|
+
acknowledgeLostAttemptOwnership();
|
|
10670
|
+
control.turnMetricOutcome = "cancelled";
|
|
10671
|
+
control.activityStatus = "cancelled";
|
|
10672
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10673
|
+
}
|
|
10674
|
+
if (rotationDecision.kind === "none") {
|
|
10675
|
+
if (leased.poolAccountCount === 0 && !leased.codexPolicySnapshotReused && leased.codexPolicySnapshot?.source !== "disabled") {
|
|
10676
|
+
throw new CodexReloginRequired("No Codex subscription is connected for this workspace.");
|
|
10677
|
+
}
|
|
10678
|
+
const noneReason = sessionPinSource === "manual" && sessionPin !== null ? {
|
|
10679
|
+
code: "codex_manual_pin_unavailable",
|
|
10680
|
+
error: "The pinned Codex subscription is unavailable for this turn.",
|
|
10681
|
+
detail: "waiting for the pinned account to reconnect or become allocatable under the accepted source policy"
|
|
10682
|
+
} : !leased.rotationEnabled && leased.activeCredentialId === null ? {
|
|
10683
|
+
code: "codex_active_pointer_unavailable",
|
|
10684
|
+
error: "The Codex active subscription pointer is unavailable for this turn.",
|
|
10685
|
+
detail: "waiting for the accepted active account to be restored or become allocatable"
|
|
10686
|
+
} : leased.poolAccountCount > leased.accounts.length ? {
|
|
10687
|
+
code: "codex_policy_pool_unavailable",
|
|
10688
|
+
error: "No Codex subscription matches the accepted account-selection policy.",
|
|
10689
|
+
detail: "waiting for a matching credential to reconnect or become allocatable under the accepted source policy"
|
|
10690
|
+
} : {
|
|
10691
|
+
code: "codex_credential_unavailable",
|
|
10692
|
+
error: "No Codex subscription is currently available under the accepted policy.",
|
|
10693
|
+
detail: "waiting for a matching credential to reconnect or become allocatable under the accepted source policy"
|
|
10694
|
+
};
|
|
10695
|
+
if (turn.source === "compaction") {
|
|
10696
|
+
if (!await eventing.settle({
|
|
10697
|
+
events: [
|
|
10698
|
+
{
|
|
10699
|
+
type: "turn.cancelled",
|
|
10700
|
+
payload: {
|
|
10701
|
+
maintenance: "context_compaction",
|
|
10702
|
+
reason: noneReason.code,
|
|
10703
|
+
requestPreserved: true
|
|
10704
|
+
}
|
|
10705
|
+
},
|
|
10706
|
+
{
|
|
10707
|
+
type: "session.status.changed",
|
|
10708
|
+
payload: { status: "idle" }
|
|
10614
10709
|
}
|
|
10615
|
-
|
|
10616
|
-
|
|
10617
|
-
|
|
10618
|
-
|
|
10710
|
+
],
|
|
10711
|
+
turnStatus: "cancelled",
|
|
10712
|
+
sessionStatus: "idle",
|
|
10713
|
+
activeTurnId: null
|
|
10714
|
+
})) {
|
|
10715
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10716
|
+
}
|
|
10717
|
+
control.turnMetricOutcome = "cancelled";
|
|
10718
|
+
control.activityStatus = "idle";
|
|
10719
|
+
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
10720
|
+
}
|
|
10721
|
+
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
|
|
10722
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
10723
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
10724
|
+
{ db, bus },
|
|
10725
|
+
{
|
|
10726
|
+
accountId: input.accountId,
|
|
10727
|
+
workspaceId: input.workspaceId,
|
|
10728
|
+
sessionId: input.sessionId,
|
|
10729
|
+
turnId,
|
|
10730
|
+
attemptId: input.attemptId,
|
|
10731
|
+
workflowId: input.workflowId,
|
|
10732
|
+
goalId: activeGoal?.id ?? null,
|
|
10733
|
+
goalVersion: activeGoal?.version ?? null,
|
|
10734
|
+
earliestResetAt: null,
|
|
10735
|
+
resetKind: "mutation_only",
|
|
10736
|
+
failurePayload: {
|
|
10737
|
+
error: noneReason.error,
|
|
10738
|
+
code: noneReason.code,
|
|
10739
|
+
detail: noneReason.detail
|
|
10619
10740
|
}
|
|
10620
|
-
|
|
10621
|
-
|
|
10622
|
-
|
|
10623
|
-
|
|
10624
|
-
|
|
10625
|
-
return { exit: claimedResult({ status: "
|
|
10741
|
+
}
|
|
10742
|
+
);
|
|
10743
|
+
if (evaluated.action === "resumed") {
|
|
10744
|
+
control.turnMetricOutcome = "recovering";
|
|
10745
|
+
control.activityStatus = "recovering";
|
|
10746
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
10626
10747
|
}
|
|
10627
|
-
|
|
10628
|
-
|
|
10629
|
-
|
|
10748
|
+
if (evaluated.action === "waiting") {
|
|
10749
|
+
control.turnMetricOutcome = "recovering";
|
|
10750
|
+
control.activityStatus = "waiting_capacity";
|
|
10751
|
+
return {
|
|
10752
|
+
exit: claimedResult({
|
|
10753
|
+
status: "waiting_capacity",
|
|
10754
|
+
capacityWait: {
|
|
10755
|
+
waiterId: evaluated.waiter.id,
|
|
10756
|
+
generation: evaluated.waiter.generation,
|
|
10757
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10758
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
10759
|
+
}
|
|
10760
|
+
})
|
|
10761
|
+
};
|
|
10762
|
+
}
|
|
10763
|
+
acknowledgeLostAttemptOwnership();
|
|
10764
|
+
control.turnMetricOutcome = "cancelled";
|
|
10765
|
+
control.activityStatus = "cancelled";
|
|
10766
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10630
10767
|
}
|
|
10631
10768
|
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
10632
10769
|
if (turn.source === "compaction") {
|
|
@@ -10655,88 +10792,64 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10655
10792
|
control.activityStatus = "idle";
|
|
10656
10793
|
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
10657
10794
|
}
|
|
10658
|
-
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId)
|
|
10795
|
+
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
|
|
10659
10796
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
10660
10797
|
const resumeMs = computeIdleDelayMs(
|
|
10661
10798
|
rotationDecision.earliestResetAt,
|
|
10662
10799
|
/* @__PURE__ */ new Date(),
|
|
10663
10800
|
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
10664
10801
|
);
|
|
10802
|
+
const allConnectedAccountsWereConsidered = leased.rotationEnabled && selectedPinDisposition !== "manual";
|
|
10803
|
+
const capacityDetail = allConnectedAccountsWereConsidered ? "all connected Codex subscriptions are rate-limited" : selectedPinDisposition === "manual" ? "the pinned Codex subscription is rate-limited" : "the active Codex subscription is rate-limited";
|
|
10665
10804
|
const failurePayload = codexUsageLimitFailurePayload(
|
|
10666
10805
|
{ resetsInSeconds: Math.ceil(resumeMs / 1e3) },
|
|
10667
|
-
|
|
10668
|
-
{ allAccounts:
|
|
10806
|
+
capacityDetail,
|
|
10807
|
+
{ allAccounts: allConnectedAccountsWereConsidered }
|
|
10669
10808
|
);
|
|
10670
10809
|
const authoritativeResetAt = authoritativeCodexCapacityResetAt(leased.accounts, /* @__PURE__ */ new Date());
|
|
10671
|
-
const
|
|
10672
|
-
|
|
10673
|
-
|
|
10674
|
-
|
|
10675
|
-
|
|
10676
|
-
|
|
10677
|
-
|
|
10678
|
-
|
|
10679
|
-
|
|
10680
|
-
|
|
10681
|
-
|
|
10682
|
-
|
|
10683
|
-
|
|
10684
|
-
|
|
10685
|
-
|
|
10810
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
10811
|
+
{ db, bus },
|
|
10812
|
+
{
|
|
10813
|
+
accountId: input.accountId,
|
|
10814
|
+
workspaceId: input.workspaceId,
|
|
10815
|
+
sessionId: input.sessionId,
|
|
10816
|
+
turnId,
|
|
10817
|
+
attemptId: input.attemptId,
|
|
10818
|
+
workflowId: input.workflowId,
|
|
10819
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
10820
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
10821
|
+
earliestResetAt: authoritativeResetAt,
|
|
10822
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
10823
|
+
failurePayload
|
|
10824
|
+
}
|
|
10825
|
+
);
|
|
10826
|
+
if (evaluated.action === "resumed") {
|
|
10827
|
+
control.turnMetricOutcome = "recovering";
|
|
10828
|
+
control.activityStatus = "recovering";
|
|
10829
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
10830
|
+
}
|
|
10831
|
+
if (evaluated.action === "waiting") {
|
|
10686
10832
|
control.turnMetricOutcome = "recovering";
|
|
10687
10833
|
control.activityStatus = "waiting_capacity";
|
|
10688
10834
|
return {
|
|
10689
10835
|
exit: claimedResult({
|
|
10690
10836
|
status: "waiting_capacity",
|
|
10691
10837
|
capacityWait: {
|
|
10692
|
-
waiterId:
|
|
10693
|
-
generation:
|
|
10694
|
-
nextCheckAt:
|
|
10695
|
-
wakeRevision:
|
|
10838
|
+
waiterId: evaluated.waiter.id,
|
|
10839
|
+
generation: evaluated.waiter.generation,
|
|
10840
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10841
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
10696
10842
|
}
|
|
10697
10843
|
})
|
|
10698
10844
|
};
|
|
10699
10845
|
}
|
|
10700
|
-
|
|
10701
|
-
|
|
10702
|
-
|
|
10703
|
-
|
|
10704
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
10705
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
10706
|
-
// the proactive path while the reactive path spares it.
|
|
10707
|
-
{
|
|
10708
|
-
type: "turn.failed",
|
|
10709
|
-
payload: {
|
|
10710
|
-
...failurePayload,
|
|
10711
|
-
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
10712
|
-
rotated: true
|
|
10713
|
-
}
|
|
10714
|
-
},
|
|
10715
|
-
{
|
|
10716
|
-
type: "session.status.changed",
|
|
10717
|
-
payload: { status: "idle" }
|
|
10718
|
-
}
|
|
10719
|
-
],
|
|
10720
|
-
turnStatus: "failed",
|
|
10721
|
-
sessionStatus: "idle",
|
|
10722
|
-
activeTurnId: null
|
|
10723
|
-
})) {
|
|
10724
|
-
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10725
|
-
}
|
|
10726
|
-
control.turnMetricOutcome = "failed";
|
|
10727
|
-
control.activityStatus = "idle";
|
|
10728
|
-
return {
|
|
10729
|
-
exit: claimedResult(
|
|
10730
|
-
goalActive ? {
|
|
10731
|
-
status: "idle",
|
|
10732
|
-
continueDelayMs: resumeMs,
|
|
10733
|
-
idleUntilReset: true
|
|
10734
|
-
} : { status: "idle" }
|
|
10735
|
-
)
|
|
10736
|
-
};
|
|
10846
|
+
acknowledgeLostAttemptOwnership();
|
|
10847
|
+
control.turnMetricOutcome = "cancelled";
|
|
10848
|
+
control.activityStatus = "cancelled";
|
|
10849
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10737
10850
|
}
|
|
10738
10851
|
if (providerTurn.effectiveCodexCredentialId) {
|
|
10739
|
-
const priorAccountId =
|
|
10852
|
+
const priorAccountId = lockedSessionCodexState.lastCredentialId;
|
|
10740
10853
|
if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
|
|
10741
10854
|
await recordSessionActiveCodexCredential(
|
|
10742
10855
|
db,
|
|
@@ -10782,8 +10895,60 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10782
10895
|
}
|
|
10783
10896
|
} catch (error) {
|
|
10784
10897
|
credentialSelectionOutcome = "failed";
|
|
10785
|
-
|
|
10786
|
-
|
|
10898
|
+
if (error instanceof CodexCredentialLeaseAttemptFencedError) {
|
|
10899
|
+
acknowledgeLostAttemptOwnership();
|
|
10900
|
+
control.activityStatus = "cancelled";
|
|
10901
|
+
control.turnMetricOutcome = "cancelled";
|
|
10902
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10903
|
+
}
|
|
10904
|
+
if (error instanceof CodexCredentialFailoverExhaustedError) {
|
|
10905
|
+
const settlement = await settleCodexCredentialFailover2(db, {
|
|
10906
|
+
accountId: input.accountId,
|
|
10907
|
+
workspaceId: input.workspaceId,
|
|
10908
|
+
sessionId: input.sessionId,
|
|
10909
|
+
turnId,
|
|
10910
|
+
attemptId: input.attemptId,
|
|
10911
|
+
holderId: null,
|
|
10912
|
+
generation: null,
|
|
10913
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
10914
|
+
maxFailovers: error.maxFailovers,
|
|
10915
|
+
recoveryPayload: {
|
|
10916
|
+
triggerEventId: attempt.triggerEventId,
|
|
10917
|
+
reason: "codex_credential_failover_exhausted"
|
|
10918
|
+
},
|
|
10919
|
+
failedPayload: {
|
|
10920
|
+
error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
10921
|
+
code: "codex_credential_failover_exhausted",
|
|
10922
|
+
retryable: false,
|
|
10923
|
+
recovery: "user_message",
|
|
10924
|
+
failoverCount: error.failoverCount,
|
|
10925
|
+
maxFailovers: error.maxFailovers
|
|
10926
|
+
}
|
|
10927
|
+
});
|
|
10928
|
+
if (settlement.action === "limit_exceeded") {
|
|
10929
|
+
await publishDurableSessionEvents5(
|
|
10930
|
+
bus,
|
|
10931
|
+
input.workspaceId,
|
|
10932
|
+
input.sessionId,
|
|
10933
|
+
settlement.events
|
|
10934
|
+
);
|
|
10935
|
+
control.activityStatus = "idle";
|
|
10936
|
+
control.turnMetricOutcome = "failed";
|
|
10937
|
+
await deliverFailedChildTurnToParent(
|
|
10938
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
10939
|
+
input.workspaceId,
|
|
10940
|
+
input.sessionId,
|
|
10941
|
+
turnId
|
|
10942
|
+
);
|
|
10943
|
+
return { exit: claimedResult({ status: "idle" }) };
|
|
10944
|
+
}
|
|
10945
|
+
acknowledgeLostAttemptOwnership();
|
|
10946
|
+
control.activityStatus = "cancelled";
|
|
10947
|
+
control.turnMetricOutcome = "cancelled";
|
|
10948
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10949
|
+
}
|
|
10950
|
+
throw error;
|
|
10951
|
+
} finally {
|
|
10787
10952
|
recordTurnStartupPhase(observability, {
|
|
10788
10953
|
phase: "credential_selection",
|
|
10789
10954
|
provider: "codex-subscription",
|
|
@@ -10841,9 +11006,10 @@ import {
|
|
|
10841
11006
|
getGeneratedImageArtifact,
|
|
10842
11007
|
markImageGenerationOperationOutcomeUnknown,
|
|
10843
11008
|
markImageGenerationOperationRetentionFailed,
|
|
10844
|
-
prepareImageGenerationOperation
|
|
11009
|
+
prepareImageGenerationOperation,
|
|
11010
|
+
resetImageGenerationOperationBeforeProviderDispatch
|
|
10845
11011
|
} from "@opengeni/db";
|
|
10846
|
-
import { createHash as
|
|
11012
|
+
import { createHash as createHash9 } from "crypto";
|
|
10847
11013
|
var ImageGenerationOutcomeUnknownError = class extends Error {
|
|
10848
11014
|
constructor(operationId) {
|
|
10849
11015
|
super(
|
|
@@ -10866,6 +11032,7 @@ var ImageGenerationRetentionFailedError = class extends Error {
|
|
|
10866
11032
|
var imageGenerationOperationPorts = {
|
|
10867
11033
|
prepare: prepareImageGenerationOperation,
|
|
10868
11034
|
begin: beginImageGenerationOperation,
|
|
11035
|
+
resetBeforeProviderDispatch: resetImageGenerationOperationBeforeProviderDispatch,
|
|
10869
11036
|
retain: retainGeneratedImage,
|
|
10870
11037
|
complete: completeImageGenerationOperation,
|
|
10871
11038
|
markOutcomeUnknown: markImageGenerationOperationOutcomeUnknown,
|
|
@@ -10912,7 +11079,9 @@ async function executeImageGenerationOperation(input, ports = imageGenerationOpe
|
|
|
10912
11079
|
accountId: input.accountId,
|
|
10913
11080
|
workspaceId: input.workspaceId,
|
|
10914
11081
|
operationId: identity.operationId,
|
|
10915
|
-
operationKey: identity.operationKey
|
|
11082
|
+
operationKey: identity.operationKey,
|
|
11083
|
+
providerBindingHash: input.providerBindingHash,
|
|
11084
|
+
expectedArtifactId: identity.artifactId
|
|
10916
11085
|
});
|
|
10917
11086
|
if (!begun.started || begun.operation.status !== "provider_started") {
|
|
10918
11087
|
const recovered = await ports.recover(input, {
|
|
@@ -10927,6 +11096,16 @@ async function executeImageGenerationOperation(input, ports = imageGenerationOpe
|
|
|
10927
11096
|
try {
|
|
10928
11097
|
output = await input.generate();
|
|
10929
11098
|
} catch (error) {
|
|
11099
|
+
if (input.isProviderDispatchRejected?.(error) === true) {
|
|
11100
|
+
await ports.resetBeforeProviderDispatch(input.db, {
|
|
11101
|
+
accountId: input.accountId,
|
|
11102
|
+
workspaceId: input.workspaceId,
|
|
11103
|
+
operationId: identity.operationId,
|
|
11104
|
+
operationKey: identity.operationKey,
|
|
11105
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11106
|
+
}).catch(() => void 0);
|
|
11107
|
+
throw error;
|
|
11108
|
+
}
|
|
10930
11109
|
await ports.markOutcomeUnknown(input.db, {
|
|
10931
11110
|
accountId: input.accountId,
|
|
10932
11111
|
workspaceId: input.workspaceId,
|
|
@@ -11043,7 +11222,7 @@ async function recoverCompletedImage(input, identity) {
|
|
|
11043
11222
|
return retainedGeneratedImageFromArtifact(recovered).receipt;
|
|
11044
11223
|
}
|
|
11045
11224
|
function digest(prefix, ...values) {
|
|
11046
|
-
const hash =
|
|
11225
|
+
const hash = createHash9("sha256").update(prefix);
|
|
11047
11226
|
for (const value of values) hash.update(value).update("\0");
|
|
11048
11227
|
return hash.digest("hex");
|
|
11049
11228
|
}
|
|
@@ -11058,13 +11237,9 @@ function uuidFromDigest3(digestValue) {
|
|
|
11058
11237
|
// src/activities/agent-turn/tool-policy.ts
|
|
11059
11238
|
function structuredToolTransportForTurn(resolvedModel) {
|
|
11060
11239
|
if (!resolvedModel) return true;
|
|
11061
|
-
|
|
11062
|
-
return
|
|
11063
|
-
|
|
11064
|
-
"xai-subscription",
|
|
11065
|
-
"vercel-gateway-managed",
|
|
11066
|
-
"vercel-gateway-workspace"
|
|
11067
|
-
].includes(resolvedModel.provider.kind);
|
|
11240
|
+
const provider = resolvedModel.provider;
|
|
11241
|
+
if (provider.api === "chat" || provider.kind === "codex-subscription") return false;
|
|
11242
|
+
return provider.wireProfile === "azure-openai" || provider.builtin && provider.id === "openai" && provider.baseUrl === void 0;
|
|
11068
11243
|
}
|
|
11069
11244
|
function lazyToolTransportForTurn(resolvedModel) {
|
|
11070
11245
|
if (!resolvedModel) return "openai_native";
|
|
@@ -12307,905 +12482,39 @@ function createSandboxTurnRuntime(deps) {
|
|
|
12307
12482
|
};
|
|
12308
12483
|
return {
|
|
12309
12484
|
releaseLateSandbox,
|
|
12310
|
-
requireResolvedSandboxForMutation,
|
|
12311
|
-
finalizeTurnOpStreamOps,
|
|
12312
|
-
onHomeSandboxRebound,
|
|
12313
|
-
runWorkspaceMutationForSandbox,
|
|
12314
|
-
stopLeaseHeartbeat,
|
|
12315
|
-
flushRuntimeBatcher,
|
|
12316
|
-
publishSandboxLifecycleEvents,
|
|
12317
|
-
publishSandboxLost,
|
|
12318
|
-
startLeaseHeartbeat
|
|
12319
|
-
};
|
|
12320
|
-
}
|
|
12321
|
-
|
|
12322
|
-
// src/activities/agent-turn/sandbox-establish.ts
|
|
12323
|
-
import {
|
|
12324
|
-
getEnrollment,
|
|
12325
|
-
getSandbox as getSandbox2,
|
|
12326
|
-
readActiveSandbox as readActiveSandbox3,
|
|
12327
|
-
getLiveEnrollmentConnection as getLiveEnrollmentConnection2,
|
|
12328
|
-
assertPersonalMachineForAttempt
|
|
12329
|
-
} from "@opengeni/db";
|
|
12330
|
-
import {
|
|
12331
|
-
withRunCredentialsSession as withRunCredentialsSession2,
|
|
12332
|
-
runOwnedSandboxSetup as runOwnedSandboxSetup2,
|
|
12333
|
-
markModelPreparationFirstSandboxOperation,
|
|
12334
|
-
recordModelPreparationMeasurement,
|
|
12335
|
-
sdkBackendIdForSandboxBackend,
|
|
12336
|
-
resolveConnectedMachineWorkspaceRoot as resolveConnectedMachineWorkspaceRoot2
|
|
12337
|
-
} from "@opengeni/runtime";
|
|
12338
|
-
import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
|
|
12339
|
-
|
|
12340
|
-
// src/sandbox-routing.ts
|
|
12341
|
-
import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
|
|
12342
|
-
import {
|
|
12343
|
-
adoptConnectedMachineSessionBackgroundCommand,
|
|
12344
|
-
advanceWorkspaceGenerationForRetainedProcess,
|
|
12345
|
-
advanceWorkspaceGeneration as advanceWorkspaceGeneration2,
|
|
12346
|
-
getRetainedProcess,
|
|
12347
|
-
retainWorkspaceMutationProcess,
|
|
12348
|
-
retainedProcessSettlementIdentity,
|
|
12349
|
-
settleRetainedProcess,
|
|
12350
|
-
verifyRetainedProcessMutationSettlement,
|
|
12351
|
-
verifyWorkspaceMutationSettlement as verifyWorkspaceMutationSettlement2,
|
|
12352
|
-
getSandbox,
|
|
12353
|
-
getLiveEnrollmentConnection,
|
|
12354
|
-
markWarmLeaseInstanceLost,
|
|
12355
|
-
readLease as readLease3,
|
|
12356
|
-
readActiveSandbox,
|
|
12357
|
-
resolvePersonalMachineConnectionForAttempt,
|
|
12358
|
-
SandboxRetainedProcessPromotionFencedError
|
|
12359
|
-
} from "@opengeni/db";
|
|
12360
|
-
import {
|
|
12361
|
-
settleConnectedMachineSessionBackgroundCommand,
|
|
12362
|
-
settleSessionBackgroundCommandForRetainedProcess
|
|
12363
|
-
} from "@opengeni/db/session-background-commands";
|
|
12364
|
-
import {
|
|
12365
|
-
buildSelfhostedBackendSession,
|
|
12366
|
-
ActiveBackendUnresolvableError,
|
|
12367
|
-
establishSandboxSessionFromEnvelope,
|
|
12368
|
-
isProviderSandboxGoneDuringRoutedOperation,
|
|
12369
|
-
isProviderSandboxNotFoundError,
|
|
12370
|
-
makeActiveBackendResolver,
|
|
12371
|
-
NatsControlRpc,
|
|
12372
|
-
NatsOpStreamTransport,
|
|
12373
|
-
RoutingBackendRecoveryRequiredError,
|
|
12374
|
-
RoutingSandboxSession,
|
|
12375
|
-
resolveModalCheckpointProviderBindingForSession,
|
|
12376
|
-
resolveConnectedMachineWorkspaceRoot,
|
|
12377
|
-
sandboxProviderInstanceIdFromEnvelope,
|
|
12378
|
-
sandboxBackendForSdkBackendId,
|
|
12379
|
-
selectBackend,
|
|
12380
|
-
verifySandboxExecReadiness
|
|
12381
|
-
} from "@opengeni/runtime";
|
|
12382
|
-
function relayConfigFromSettings(settings) {
|
|
12383
|
-
const raw = settings.selfhostedRelayUrl?.trim();
|
|
12384
|
-
if (!raw) {
|
|
12385
|
-
return { host: "relay.opengeni.local", port: 443, tls: true };
|
|
12386
|
-
}
|
|
12387
|
-
try {
|
|
12388
|
-
const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
|
|
12389
|
-
const tls = url.protocol === "wss:" || url.protocol === "https:";
|
|
12390
|
-
const port = url.port ? Number(url.port) : tls ? 443 : 80;
|
|
12391
|
-
return { host: url.hostname, port, tls };
|
|
12392
|
-
} catch {
|
|
12393
|
-
return { host: raw, port: 443, tls: true };
|
|
12394
|
-
}
|
|
12395
|
-
}
|
|
12396
|
-
function selfhostedTimeoutsFromSettings(settings) {
|
|
12397
|
-
return {
|
|
12398
|
-
timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
|
|
12399
|
-
execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs
|
|
12400
|
-
};
|
|
12401
|
-
}
|
|
12402
|
-
function selfhostedResolverTimeouts(settings) {
|
|
12403
|
-
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
12404
|
-
return {
|
|
12405
|
-
selfhostedTimeoutMs: timeoutMs,
|
|
12406
|
-
selfhostedExecTimeoutMs: execTimeoutMs
|
|
12407
|
-
};
|
|
12408
|
-
}
|
|
12409
|
-
function homeRouteRecoveryDisposition(lease) {
|
|
12410
|
-
if (!lease) return "unrecoverable";
|
|
12411
|
-
if (lease.liveness === "warming") return "pending";
|
|
12412
|
-
if (lease.liveness === "draining") return "superseded";
|
|
12413
|
-
if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
|
|
12414
|
-
return "pending";
|
|
12415
|
-
}
|
|
12416
|
-
if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
|
|
12417
|
-
return "degraded";
|
|
12418
|
-
}
|
|
12419
|
-
return "unrecoverable";
|
|
12420
|
-
}
|
|
12421
|
-
function homeRouteRecoveryError(lease, fallbackEpoch) {
|
|
12422
|
-
return new RoutingBackendRecoveryRequiredError(
|
|
12423
|
-
"resolve_home_backend",
|
|
12424
|
-
lease?.leaseEpoch ?? fallbackEpoch,
|
|
12425
|
-
homeRouteRecoveryDisposition(lease)
|
|
12426
|
-
);
|
|
12427
|
-
}
|
|
12428
|
-
function providerIdentityFromResumeState(resumeState) {
|
|
12429
|
-
return sandboxProviderInstanceIdFromEnvelope(resumeState);
|
|
12430
|
-
}
|
|
12431
|
-
async function resolveCurrentHomeBackend(services, ids, established) {
|
|
12432
|
-
const lease = await readLease3(services.db, ids.workspaceId, ids.sandboxGroupId);
|
|
12433
|
-
const fallbackEpoch = lease?.leaseEpoch ?? 0;
|
|
12434
|
-
if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
|
|
12435
|
-
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
12436
|
-
}
|
|
12437
|
-
if (lease.instanceId === established.instanceId) {
|
|
12438
|
-
services.onHomeSandboxRebound?.({
|
|
12439
|
-
established,
|
|
12440
|
-
leaseEpoch: lease.leaseEpoch
|
|
12441
|
-
});
|
|
12442
|
-
return {
|
|
12443
|
-
session: established.session,
|
|
12444
|
-
sandboxId: null,
|
|
12445
|
-
kind: established.backendId,
|
|
12446
|
-
leaseEpoch: lease.leaseEpoch,
|
|
12447
|
-
providerInstanceId: lease.instanceId
|
|
12448
|
-
};
|
|
12449
|
-
}
|
|
12450
|
-
const resumeState = lease.resumeState;
|
|
12451
|
-
const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
12452
|
-
const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
|
|
12453
|
-
if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
|
|
12454
|
-
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
12455
|
-
}
|
|
12456
|
-
let rebound;
|
|
12457
|
-
try {
|
|
12458
|
-
rebound = await establishSandboxSessionFromEnvelope(services.settings, resumeState, {
|
|
12459
|
-
sessionId: ids.sessionId,
|
|
12460
|
-
recovery: "resume-only",
|
|
12461
|
-
backendOverride: resumeBackend
|
|
12462
|
-
});
|
|
12463
|
-
await verifySandboxExecReadiness(rebound);
|
|
12464
|
-
} catch (error) {
|
|
12465
|
-
if (!isProviderSandboxNotFoundError(resumeBackend, error)) {
|
|
12466
|
-
throw new RoutingBackendRecoveryRequiredError(
|
|
12467
|
-
"resolve_home_backend",
|
|
12468
|
-
lease.leaseEpoch,
|
|
12469
|
-
"pending"
|
|
12470
|
-
);
|
|
12471
|
-
}
|
|
12472
|
-
const marked = await markWarmLeaseInstanceLost(services.db, {
|
|
12473
|
-
accountId: ids.accountId,
|
|
12474
|
-
workspaceId: ids.workspaceId,
|
|
12475
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
12476
|
-
expectedEpoch: lease.leaseEpoch,
|
|
12477
|
-
expectedInstanceId: lease.instanceId,
|
|
12478
|
-
diagnostic: "provider_not_found_during_home_route_rebind"
|
|
12479
|
-
});
|
|
12480
|
-
if (marked.status === "marked") {
|
|
12481
|
-
await services.onHomeSandboxLost?.({
|
|
12482
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
12483
|
-
instanceId: lease.instanceId,
|
|
12484
|
-
leaseEpoch: marked.lease.leaseEpoch
|
|
12485
|
-
});
|
|
12486
|
-
}
|
|
12487
|
-
throw new RoutingBackendRecoveryRequiredError(
|
|
12488
|
-
"resolve_home_backend",
|
|
12489
|
-
marked.lease?.leaseEpoch ?? lease.leaseEpoch,
|
|
12490
|
-
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
|
|
12491
|
-
);
|
|
12492
|
-
}
|
|
12493
|
-
const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
|
|
12494
|
-
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
12495
|
-
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
12496
|
-
}
|
|
12497
|
-
services.onHomeSandboxRebound?.({
|
|
12498
|
-
established: rebound,
|
|
12499
|
-
leaseEpoch: lease.leaseEpoch
|
|
12500
|
-
});
|
|
12501
|
-
Object.assign(established, rebound);
|
|
12502
|
-
return {
|
|
12503
|
-
session: rebound.session,
|
|
12504
|
-
sandboxId: null,
|
|
12505
|
-
kind: rebound.backendId,
|
|
12506
|
-
leaseEpoch: lease.leaseEpoch,
|
|
12507
|
-
providerInstanceId: lease.instanceId
|
|
12508
|
-
};
|
|
12509
|
-
}
|
|
12510
|
-
function beforePersistableHomeMutation(services, ids, home) {
|
|
12511
|
-
const fence = ids.workspaceMutationFence;
|
|
12512
|
-
if (!home || !fence) return void 0;
|
|
12513
|
-
return async ({ op, backend, onCaptureWait }) => {
|
|
12514
|
-
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
12515
|
-
return null;
|
|
12516
|
-
}
|
|
12517
|
-
const providerBinding = home.backend === "modal" ? await resolveModalCheckpointProviderBindingForSession(services.settings, backend.session) : null;
|
|
12518
|
-
const admission = await advanceWorkspaceGeneration2(services.db, {
|
|
12519
|
-
accountId: fence.accountId,
|
|
12520
|
-
workspaceId: ids.workspaceId,
|
|
12521
|
-
sessionId: ids.sessionId,
|
|
12522
|
-
turnId: fence.turnId,
|
|
12523
|
-
executionGeneration: fence.executionGeneration,
|
|
12524
|
-
attemptId: fence.attemptId,
|
|
12525
|
-
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
12526
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
12527
|
-
expectedEpoch: backend.leaseEpoch,
|
|
12528
|
-
expectedInstanceId: backend.providerInstanceId,
|
|
12529
|
-
operation: op,
|
|
12530
|
-
captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
|
|
12531
|
-
...services.waitSignal ? { waitSignal: services.waitSignal } : {},
|
|
12532
|
-
...onCaptureWait ? { onCaptureWait } : {}
|
|
12533
|
-
});
|
|
12534
|
-
return { admission, providerBinding };
|
|
12535
|
-
};
|
|
12536
|
-
}
|
|
12537
|
-
function afterPersistableHomeMutation(services, ids, home) {
|
|
12538
|
-
const fence = ids.workspaceMutationFence;
|
|
12539
|
-
if (!home || !fence) return void 0;
|
|
12540
|
-
return async ({ op, backend, admission, outcome, retainedProcess }) => {
|
|
12541
|
-
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object") {
|
|
12542
|
-
return;
|
|
12543
|
-
}
|
|
12544
|
-
const boundAdmission = admission;
|
|
12545
|
-
const exactAdmission = boundAdmission.admission;
|
|
12546
|
-
if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
|
|
12547
|
-
throw new Error("Persistable home mutation settlement lacked its bound admission");
|
|
12548
|
-
}
|
|
12549
|
-
if (outcome === "resolved" && retainedProcess) {
|
|
12550
|
-
try {
|
|
12551
|
-
await retainWorkspaceMutationProcess(services.db, {
|
|
12552
|
-
accountId: fence.accountId,
|
|
12553
|
-
workspaceId: ids.workspaceId,
|
|
12554
|
-
sessionId: ids.sessionId,
|
|
12555
|
-
processId: retainedProcess.id,
|
|
12556
|
-
providerSessionId: retainedProcess.providerSessionId,
|
|
12557
|
-
admissionId: exactAdmission.id,
|
|
12558
|
-
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
12559
|
-
operation: op,
|
|
12560
|
-
providerBinding: boundAdmission.providerBinding ?? null,
|
|
12561
|
-
backgroundCommand: {
|
|
12562
|
-
commandId: retainedProcess.id,
|
|
12563
|
-
command: op
|
|
12564
|
-
},
|
|
12565
|
-
owner: {
|
|
12566
|
-
kind: "turn",
|
|
12567
|
-
turnId: fence.turnId,
|
|
12568
|
-
executionGeneration: fence.executionGeneration,
|
|
12569
|
-
attemptId: fence.attemptId,
|
|
12570
|
-
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
12571
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
12572
|
-
expectedEpoch: backend.leaseEpoch,
|
|
12573
|
-
expectedInstanceId: backend.providerInstanceId,
|
|
12574
|
-
routeKind: exactAdmission.routeKind,
|
|
12575
|
-
routeTargetId: exactAdmission.routeTargetId,
|
|
12576
|
-
routeEpoch: exactAdmission.routeEpoch
|
|
12577
|
-
}
|
|
12578
|
-
});
|
|
12579
|
-
} catch (error) {
|
|
12580
|
-
if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
|
|
12581
|
-
const durable = error.process;
|
|
12582
|
-
if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
|
|
12583
|
-
throw new Error("Durable retained-process promotion returned a mismatched identity", {
|
|
12584
|
-
cause: error
|
|
12585
|
-
});
|
|
12586
|
-
}
|
|
12587
|
-
return {
|
|
12588
|
-
status: "retained_process_durable_output_rejected",
|
|
12589
|
-
retainedProcess: {
|
|
12590
|
-
id: durable.id,
|
|
12591
|
-
providerSessionId: durable.providerSessionId
|
|
12592
|
-
}
|
|
12593
|
-
};
|
|
12594
|
-
}
|
|
12595
|
-
return;
|
|
12596
|
-
}
|
|
12597
|
-
await verifyWorkspaceMutationSettlement2(services.db, {
|
|
12598
|
-
accountId: fence.accountId,
|
|
12599
|
-
workspaceId: ids.workspaceId,
|
|
12600
|
-
sessionId: ids.sessionId,
|
|
12601
|
-
turnId: fence.turnId,
|
|
12602
|
-
executionGeneration: fence.executionGeneration,
|
|
12603
|
-
attemptId: fence.attemptId,
|
|
12604
|
-
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
12605
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
12606
|
-
expectedEpoch: backend.leaseEpoch,
|
|
12607
|
-
expectedInstanceId: backend.providerInstanceId,
|
|
12608
|
-
admission: exactAdmission,
|
|
12609
|
-
operation: op,
|
|
12610
|
-
outcome
|
|
12611
|
-
});
|
|
12612
|
-
};
|
|
12613
|
-
}
|
|
12614
|
-
function beforeRetainedProcessMutation(services, ids) {
|
|
12615
|
-
const fence = ids.workspaceMutationFence;
|
|
12616
|
-
if (!fence) return void 0;
|
|
12617
|
-
return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
|
|
12618
|
-
accountId: fence.accountId,
|
|
12619
|
-
workspaceId: ids.workspaceId,
|
|
12620
|
-
sessionId: ids.sessionId,
|
|
12621
|
-
processId: process2.id,
|
|
12622
|
-
operation: op,
|
|
12623
|
-
captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
|
|
12624
|
-
...services.waitSignal ? { waitSignal: services.waitSignal } : {}
|
|
12625
|
-
});
|
|
12626
|
-
}
|
|
12627
|
-
function afterRetainedProcessMutation(services, ids) {
|
|
12628
|
-
const fence = ids.workspaceMutationFence;
|
|
12629
|
-
if (!fence) return void 0;
|
|
12630
|
-
return async ({ op, process: process2, admission, outcome }) => {
|
|
12631
|
-
if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
|
|
12632
|
-
throw new Error("Retained-process mutation settlement lacked its exact admission");
|
|
12633
|
-
}
|
|
12634
|
-
await verifyRetainedProcessMutationSettlement(services.db, {
|
|
12635
|
-
accountId: fence.accountId,
|
|
12636
|
-
workspaceId: ids.workspaceId,
|
|
12637
|
-
sessionId: ids.sessionId,
|
|
12638
|
-
processId: process2.id,
|
|
12639
|
-
admission,
|
|
12640
|
-
operation: op,
|
|
12641
|
-
outcome
|
|
12642
|
-
});
|
|
12643
|
-
};
|
|
12644
|
-
}
|
|
12645
|
-
function settleRetainedProcessForTurn(services, ids) {
|
|
12646
|
-
const fence = ids.workspaceMutationFence;
|
|
12647
|
-
if (!fence) return void 0;
|
|
12648
|
-
return async ({ backend, process: process2, proof }) => {
|
|
12649
|
-
const durable = await getRetainedProcess(services.db, {
|
|
12650
|
-
workspaceId: ids.workspaceId,
|
|
12651
|
-
sessionId: ids.sessionId,
|
|
12652
|
-
processId: process2.id
|
|
12653
|
-
});
|
|
12654
|
-
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
12655
|
-
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
12656
|
-
}
|
|
12657
|
-
const settlement = await settleRetainedProcess(services.db, {
|
|
12658
|
-
accountId: fence.accountId,
|
|
12659
|
-
workspaceId: ids.workspaceId,
|
|
12660
|
-
sessionId: ids.sessionId,
|
|
12661
|
-
processId: process2.id,
|
|
12662
|
-
expected: retainedProcessSettlementIdentity(durable),
|
|
12663
|
-
outcome: proof.outcome,
|
|
12664
|
-
exitCode: proof.exitCode,
|
|
12665
|
-
reason: proof.reason,
|
|
12666
|
-
idleGraceMs: services.settings.sandboxIdleGraceMs
|
|
12667
|
-
});
|
|
12668
|
-
if (settlement.process.state === "active") {
|
|
12669
|
-
throw new Error("Retained-process settlement returned an active durable process");
|
|
12670
|
-
}
|
|
12671
|
-
await settleSessionBackgroundCommandForRetainedProcess(services.db, {
|
|
12672
|
-
accountId: fence.accountId,
|
|
12673
|
-
workspaceId: ids.workspaceId,
|
|
12674
|
-
sessionId: ids.sessionId,
|
|
12675
|
-
retainedProcessId: process2.id,
|
|
12676
|
-
outcome: settlement.process.state,
|
|
12677
|
-
exitCode: settlement.process.exitCode,
|
|
12678
|
-
reason: settlement.process.settlementReason ?? proof.reason
|
|
12679
|
-
});
|
|
12680
|
-
};
|
|
12681
|
-
}
|
|
12682
|
-
function controlRpcFactory(bus) {
|
|
12683
|
-
return () => new NatsControlRpc(async () => {
|
|
12684
|
-
if (!bus) {
|
|
12685
|
-
return null;
|
|
12686
|
-
}
|
|
12687
|
-
return bus.getRequestConnection();
|
|
12688
|
-
});
|
|
12689
|
-
}
|
|
12690
|
-
function wrapTurnBoxWithRouting(services, ids, established) {
|
|
12691
|
-
const { db, settings, bus, onOp } = services;
|
|
12692
|
-
const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
|
|
12693
|
-
const backgroundCommandAttempt = ids.workspaceMutationFence;
|
|
12694
|
-
const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
|
|
12695
|
-
const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
|
|
12696
|
-
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
12697
|
-
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
12698
|
-
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
12699
|
-
const resolver = makeActiveBackendResolver({
|
|
12700
|
-
workspaceId: ids.workspaceId,
|
|
12701
|
-
defaultBackend: established.session,
|
|
12702
|
-
defaultKind: established.backendId,
|
|
12703
|
-
getSandbox: async (sandboxId) => {
|
|
12704
|
-
const sandbox = await getSandbox(
|
|
12705
|
-
db,
|
|
12706
|
-
ids.resourceAccountId && ids.resourceSubjectId ? {
|
|
12707
|
-
accountId: ids.resourceAccountId,
|
|
12708
|
-
workspaceId: ids.workspaceId,
|
|
12709
|
-
subjectId: ids.resourceSubjectId
|
|
12710
|
-
} : ids.workspaceId,
|
|
12711
|
-
sandboxId
|
|
12712
|
-
);
|
|
12713
|
-
return sandbox ? {
|
|
12714
|
-
id: sandbox.id,
|
|
12715
|
-
workspaceId: sandbox.workspaceId,
|
|
12716
|
-
kind: sandbox.kind,
|
|
12717
|
-
name: sandbox.name,
|
|
12718
|
-
...sandbox.scope ? { scope: sandbox.scope } : {},
|
|
12719
|
-
enrollmentId: sandbox.enrollmentId
|
|
12720
|
-
} : null;
|
|
12721
|
-
},
|
|
12722
|
-
controlRpcFactory: controlRpcFactory(bus),
|
|
12723
|
-
resolveSelfhostedConnection: async (sandbox) => {
|
|
12724
|
-
if (!sandbox.enrollmentId) return null;
|
|
12725
|
-
const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
|
|
12726
|
-
return connectionBindingFor(services, enrollment);
|
|
12727
|
-
},
|
|
12728
|
-
relay: relayConfigFromSettings(settings),
|
|
12729
|
-
// A selfhosted swap target runs real commands too, so give it the same split
|
|
12730
|
-
// deadlines the machine-primary establish path uses (short control, long exec).
|
|
12731
|
-
...selfhostedResolverTimeouts(settings),
|
|
12732
|
-
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
12733
|
-
...backgroundCommandAccountId && backgroundCommandAttempt ? {
|
|
12734
|
-
selfhostedAdoptBackgroundCommand: async (command) => {
|
|
12735
|
-
const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
|
|
12736
|
-
accountId: backgroundCommandAccountId,
|
|
12737
|
-
workspaceId: ids.workspaceId,
|
|
12738
|
-
sessionId: ids.sessionId,
|
|
12739
|
-
turnId: backgroundCommandAttempt.turnId,
|
|
12740
|
-
executionGeneration: backgroundCommandAttempt.executionGeneration,
|
|
12741
|
-
attemptId: backgroundCommandAttempt.attemptId,
|
|
12742
|
-
commandId: crypto.randomUUID(),
|
|
12743
|
-
controlWorkspaceId: command.controlWorkspaceId,
|
|
12744
|
-
enrollmentId: command.enrollmentId,
|
|
12745
|
-
connectionInstanceId: command.connectionInstanceId,
|
|
12746
|
-
opId: command.opId,
|
|
12747
|
-
command: command.command
|
|
12748
|
-
});
|
|
12749
|
-
return { commandId: adopted.id };
|
|
12750
|
-
},
|
|
12751
|
-
selfhostedSettleBackgroundCommand: async (command) => {
|
|
12752
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
12753
|
-
accountId: backgroundCommandAccountId,
|
|
12754
|
-
workspaceId: ids.workspaceId,
|
|
12755
|
-
sessionId: ids.sessionId,
|
|
12756
|
-
commandId: command.commandId,
|
|
12757
|
-
controlWorkspaceId: command.controlWorkspaceId,
|
|
12758
|
-
enrollmentId: command.enrollmentId,
|
|
12759
|
-
connectionInstanceId: command.connectionInstanceId,
|
|
12760
|
-
opId: command.opId,
|
|
12761
|
-
outcome: command.outcome,
|
|
12762
|
-
exitCode: command.exitCode,
|
|
12763
|
-
reason: command.reason
|
|
12764
|
-
});
|
|
12765
|
-
}
|
|
12766
|
-
} : {},
|
|
12767
|
-
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
12768
|
-
// SDK's per-turn manifest-env delta is empty (no "cannot change manifest
|
|
12769
|
-
// environment variables" throw when the turn pins to a vm). Mirrors the group
|
|
12770
|
-
// box, which is created WITH this same environment (resumeBoxForTurn).
|
|
12771
|
-
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
12772
|
-
...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {},
|
|
12773
|
-
// Stage D machine-primary: pin THIS established SelfhostedSession for the machine
|
|
12774
|
-
// pointer so the resolver returns the SAME instance (no two-instance manifest
|
|
12775
|
-
// divergence). `established.session` is the SelfhostedSession the establish branch
|
|
12776
|
-
// bound to (sandboxId, epoch).
|
|
12777
|
-
...ids.pinnedSelfhosted ? {
|
|
12778
|
-
pinnedSelfhosted: {
|
|
12779
|
-
sandboxId: ids.pinnedSelfhosted.sandboxId,
|
|
12780
|
-
epoch: ids.pinnedSelfhosted.epoch,
|
|
12781
|
-
session: established.session
|
|
12782
|
-
}
|
|
12783
|
-
} : {},
|
|
12784
|
-
// A modal swap target in the turn path would need its own lease resume-by-id;
|
|
12785
|
-
// that is a future cross-group-box concern. Until then a modal swap target is
|
|
12786
|
-
// unresolvable (the swap tool validates liveness, so this only triggers if a
|
|
12787
|
-
// session points at a sibling modal box the turn cannot resume here) and the
|
|
12788
|
-
// op surfaces unresolvable — never a silent wrong-box landing.
|
|
12789
|
-
//
|
|
12790
|
-
// For a machine-primary turn of a Modal-HOME session (pinned to a machine, no
|
|
12791
|
-
// group box established this turn), a mid-turn clear-to-null must NOT fall back to
|
|
12792
|
-
// the pinned machine — passing defaultIsHome:false makes the null branch throw typed
|
|
12793
|
-
// `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
|
|
12794
|
-
...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
|
|
12795
|
-
...ids.homeLease ? {
|
|
12796
|
-
resolveDefaultBackend: () => resolveCurrentHomeBackend(
|
|
12797
|
-
services,
|
|
12798
|
-
{
|
|
12799
|
-
workspaceId: ids.workspaceId,
|
|
12800
|
-
sessionId: ids.sessionId,
|
|
12801
|
-
accountId: ids.homeLease.accountId,
|
|
12802
|
-
sandboxGroupId: ids.homeLease.sandboxGroupId,
|
|
12803
|
-
backend: ids.homeLease.backend
|
|
12804
|
-
},
|
|
12805
|
-
established
|
|
12806
|
-
)
|
|
12807
|
-
} : {}
|
|
12808
|
-
});
|
|
12809
|
-
const proxy = new RoutingSandboxSession({
|
|
12810
|
-
// Seed the DEFAULT backend (the established group box) at construction so
|
|
12811
|
-
// `session.state` is the real backend's state object BEFORE the first op. The
|
|
12812
|
-
// SDK reads `session.state.manifest` at turn START (and writes it back); an
|
|
12813
|
-
// empty `{}` there crashes serializeManifestEnvironment /
|
|
12814
|
-
// validateProvidedSessionManifestUpdate. This is byte-identical to what the
|
|
12815
|
-
// resolver returns for the default pointer (`activeSandboxId === null`).
|
|
12816
|
-
defaultResolved: {
|
|
12817
|
-
session: established.session,
|
|
12818
|
-
sandboxId: null,
|
|
12819
|
-
kind: established.backendId,
|
|
12820
|
-
...ids.homeLease ? {
|
|
12821
|
-
leaseEpoch: ids.homeLease.leaseEpoch,
|
|
12822
|
-
providerInstanceId: ids.homeLease.instanceId
|
|
12823
|
-
} : {}
|
|
12824
|
-
},
|
|
12825
|
-
readPointer: async () => {
|
|
12826
|
-
if (!routingEnabled(settings)) {
|
|
12827
|
-
return { activeSandboxId: null, activeEpoch: 0 };
|
|
12828
|
-
}
|
|
12829
|
-
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
12830
|
-
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
12831
|
-
},
|
|
12832
|
-
resolveActiveBackend: resolver,
|
|
12833
|
-
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
12834
|
-
...beforeMutation ? { beforeMutation } : {},
|
|
12835
|
-
...afterMutation ? { afterMutation } : {},
|
|
12836
|
-
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
12837
|
-
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
12838
|
-
...settleProcess ? { settleProcess } : {},
|
|
12839
|
-
...ids.homeLease ? {
|
|
12840
|
-
onDefaultBackendError: async ({
|
|
12841
|
-
error,
|
|
12842
|
-
backend
|
|
12843
|
-
}) => {
|
|
12844
|
-
const home = ids.homeLease;
|
|
12845
|
-
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
12846
|
-
const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
|
|
12847
|
-
const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
|
|
12848
|
-
const marked = await markWarmLeaseInstanceLost(db, {
|
|
12849
|
-
accountId: home.accountId,
|
|
12850
|
-
workspaceId: ids.workspaceId,
|
|
12851
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
12852
|
-
expectedEpoch,
|
|
12853
|
-
expectedInstanceId,
|
|
12854
|
-
diagnostic: "provider_not_found_during_routed_operation"
|
|
12855
|
-
});
|
|
12856
|
-
if (marked.status === "marked") {
|
|
12857
|
-
await services.onHomeSandboxLost?.({
|
|
12858
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
12859
|
-
instanceId: expectedInstanceId,
|
|
12860
|
-
leaseEpoch: marked.lease.leaseEpoch
|
|
12861
|
-
});
|
|
12862
|
-
}
|
|
12863
|
-
const lease = marked.lease;
|
|
12864
|
-
const restore = lease?.recovery.restore.status;
|
|
12865
|
-
return {
|
|
12866
|
-
leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
|
|
12867
|
-
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
12868
|
-
};
|
|
12869
|
-
}
|
|
12870
|
-
} : {}
|
|
12871
|
-
});
|
|
12872
|
-
return { ...established, session: proxy };
|
|
12873
|
-
}
|
|
12874
|
-
function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
12875
|
-
const { db, settings, bus, onOp } = services;
|
|
12876
|
-
const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
|
|
12877
|
-
const backgroundCommandAttempt = ids.workspaceMutationFence;
|
|
12878
|
-
const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
12879
|
-
const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
12880
|
-
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
12881
|
-
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
12882
|
-
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
12883
|
-
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
12884
|
-
const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
|
|
12885
|
-
const syntheticSession = {
|
|
12886
|
-
state: { manifest: args.agentDefaultManifest },
|
|
12887
|
-
// The SDK binds shell tools synchronously before lazy provisioning. Preserve
|
|
12888
|
-
// the authoritative descriptor's PTY shape so a Modal home exposes
|
|
12889
|
-
// write_stdin on its first turn operation; the async cancellation transport
|
|
12890
|
-
// below still resolves the exact live route before executing a command.
|
|
12891
|
-
supportsPty: () => defaultSupportsPty
|
|
12892
|
-
};
|
|
12893
|
-
const routedResolver = makeActiveBackendResolver({
|
|
12894
|
-
workspaceId: ids.workspaceId,
|
|
12895
|
-
defaultBackend: syntheticSession,
|
|
12896
|
-
defaultKind: "unprovisioned",
|
|
12897
|
-
getSandbox: async (sandboxId) => {
|
|
12898
|
-
const sandbox = await getSandbox(
|
|
12899
|
-
db,
|
|
12900
|
-
ids.resourceAccountId && ids.resourceSubjectId ? {
|
|
12901
|
-
accountId: ids.resourceAccountId,
|
|
12902
|
-
workspaceId: ids.workspaceId,
|
|
12903
|
-
subjectId: ids.resourceSubjectId
|
|
12904
|
-
} : ids.workspaceId,
|
|
12905
|
-
sandboxId
|
|
12906
|
-
);
|
|
12907
|
-
return sandbox ? {
|
|
12908
|
-
id: sandbox.id,
|
|
12909
|
-
workspaceId: sandbox.workspaceId,
|
|
12910
|
-
kind: sandbox.kind,
|
|
12911
|
-
name: sandbox.name,
|
|
12912
|
-
...sandbox.scope ? { scope: sandbox.scope } : {},
|
|
12913
|
-
enrollmentId: sandbox.enrollmentId
|
|
12914
|
-
} : null;
|
|
12915
|
-
},
|
|
12916
|
-
controlRpcFactory: controlRpcFactory(bus),
|
|
12917
|
-
resolveSelfhostedConnection: async (sandbox) => {
|
|
12918
|
-
if (!sandbox.enrollmentId) return null;
|
|
12919
|
-
const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
|
|
12920
|
-
return connectionBindingFor(services, enrollment);
|
|
12921
|
-
},
|
|
12922
|
-
relay: relayConfigFromSettings(settings),
|
|
12923
|
-
...selfhostedResolverTimeouts(settings),
|
|
12924
|
-
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
12925
|
-
...backgroundCommandAccountId && backgroundCommandAttempt ? {
|
|
12926
|
-
selfhostedAdoptBackgroundCommand: async (command) => {
|
|
12927
|
-
const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
|
|
12928
|
-
accountId: backgroundCommandAccountId,
|
|
12929
|
-
workspaceId: ids.workspaceId,
|
|
12930
|
-
sessionId: ids.sessionId,
|
|
12931
|
-
turnId: backgroundCommandAttempt.turnId,
|
|
12932
|
-
executionGeneration: backgroundCommandAttempt.executionGeneration,
|
|
12933
|
-
attemptId: backgroundCommandAttempt.attemptId,
|
|
12934
|
-
commandId: crypto.randomUUID(),
|
|
12935
|
-
controlWorkspaceId: command.controlWorkspaceId,
|
|
12936
|
-
enrollmentId: command.enrollmentId,
|
|
12937
|
-
connectionInstanceId: command.connectionInstanceId,
|
|
12938
|
-
opId: command.opId,
|
|
12939
|
-
command: command.command
|
|
12940
|
-
});
|
|
12941
|
-
return { commandId: adopted.id };
|
|
12942
|
-
},
|
|
12943
|
-
selfhostedSettleBackgroundCommand: async (command) => {
|
|
12944
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
12945
|
-
accountId: backgroundCommandAccountId,
|
|
12946
|
-
workspaceId: ids.workspaceId,
|
|
12947
|
-
sessionId: ids.sessionId,
|
|
12948
|
-
commandId: command.commandId,
|
|
12949
|
-
controlWorkspaceId: command.controlWorkspaceId,
|
|
12950
|
-
enrollmentId: command.enrollmentId,
|
|
12951
|
-
connectionInstanceId: command.connectionInstanceId,
|
|
12952
|
-
opId: command.opId,
|
|
12953
|
-
outcome: command.outcome,
|
|
12954
|
-
exitCode: command.exitCode,
|
|
12955
|
-
reason: command.reason
|
|
12956
|
-
});
|
|
12957
|
-
}
|
|
12958
|
-
} : {},
|
|
12959
|
-
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
12960
|
-
...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {}
|
|
12961
|
-
});
|
|
12962
|
-
const proxy = new RoutingSandboxSession({
|
|
12963
|
-
// Before the first op the SDK reads `state.manifest`; the synthetic backend
|
|
12964
|
-
// points at agent.defaultManifest BY REFERENCE so the provided-session delta is
|
|
12965
|
-
// empty. The first default-pointer op resolves the real box through the
|
|
12966
|
-
// provisioner and `state` switches to that real backend by reference.
|
|
12967
|
-
defaultResolved: {
|
|
12968
|
-
session: syntheticSession,
|
|
12969
|
-
sandboxId: null,
|
|
12970
|
-
kind: "unprovisioned"
|
|
12971
|
-
},
|
|
12972
|
-
readPointer: async () => {
|
|
12973
|
-
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
12974
|
-
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
12975
|
-
},
|
|
12976
|
-
resolveActiveBackend: async (pointer) => {
|
|
12977
|
-
if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
|
|
12978
|
-
const provisioned = await args.provisioner.get();
|
|
12979
|
-
if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
|
|
12980
|
-
return resolveCurrentHomeBackend(
|
|
12981
|
-
services,
|
|
12982
|
-
{
|
|
12983
|
-
workspaceId: ids.workspaceId,
|
|
12984
|
-
sessionId: ids.sessionId,
|
|
12985
|
-
accountId: args.homeLeaseIdentity.accountId,
|
|
12986
|
-
sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
|
|
12987
|
-
backend: args.homeLeaseIdentity.backend
|
|
12988
|
-
},
|
|
12989
|
-
provisioned.established
|
|
12990
|
-
);
|
|
12991
|
-
}
|
|
12992
|
-
return {
|
|
12993
|
-
session: provisioned.established.session,
|
|
12994
|
-
sandboxId: null,
|
|
12995
|
-
kind: provisioned.established.backendId
|
|
12996
|
-
};
|
|
12997
|
-
}
|
|
12998
|
-
return routedResolver(pointer);
|
|
12999
|
-
},
|
|
13000
|
-
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
13001
|
-
...args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {},
|
|
13002
|
-
...beforeMutation ? { beforeMutation } : {},
|
|
13003
|
-
...afterMutation ? { afterMutation } : {},
|
|
13004
|
-
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
13005
|
-
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
13006
|
-
...settleProcess ? { settleProcess } : {},
|
|
13007
|
-
...args.homeLeaseIdentity ? {
|
|
13008
|
-
onDefaultBackendError: async ({
|
|
13009
|
-
error,
|
|
13010
|
-
backend
|
|
13011
|
-
}) => {
|
|
13012
|
-
const home = args.homeLeaseIdentity;
|
|
13013
|
-
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
13014
|
-
if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
13015
|
-
return null;
|
|
13016
|
-
}
|
|
13017
|
-
const marked = await markWarmLeaseInstanceLost(db, {
|
|
13018
|
-
accountId: home.accountId,
|
|
13019
|
-
workspaceId: ids.workspaceId,
|
|
13020
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
13021
|
-
expectedEpoch: backend.leaseEpoch,
|
|
13022
|
-
expectedInstanceId: backend.providerInstanceId,
|
|
13023
|
-
diagnostic: "provider_not_found_during_routed_operation"
|
|
13024
|
-
});
|
|
13025
|
-
if (marked.status === "marked") {
|
|
13026
|
-
await services.onHomeSandboxLost?.({
|
|
13027
|
-
sandboxGroupId: home.sandboxGroupId,
|
|
13028
|
-
instanceId: backend.providerInstanceId,
|
|
13029
|
-
leaseEpoch: marked.lease.leaseEpoch
|
|
13030
|
-
});
|
|
13031
|
-
}
|
|
13032
|
-
const lease = marked.lease;
|
|
13033
|
-
const restore = lease?.recovery.restore.status;
|
|
13034
|
-
return {
|
|
13035
|
-
leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
|
|
13036
|
-
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
13037
|
-
};
|
|
13038
|
-
}
|
|
13039
|
-
} : {}
|
|
13040
|
-
});
|
|
13041
|
-
return {
|
|
13042
|
-
client: args.client,
|
|
13043
|
-
session: proxy,
|
|
13044
|
-
sessionState: void 0,
|
|
13045
|
-
instanceId: "unprovisioned",
|
|
13046
|
-
backendId: args.backendId
|
|
13047
|
-
};
|
|
13048
|
-
}
|
|
13049
|
-
function opStreamDepsFor(services, machineAdvertisesOpStream) {
|
|
13050
|
-
const { settings, bus, opJournal } = services;
|
|
13051
|
-
if (!machineAdvertisesOpStream || settings.agentOpStreamEnabled !== true || !bus) {
|
|
13052
|
-
return void 0;
|
|
13053
|
-
}
|
|
13054
|
-
return {
|
|
13055
|
-
transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
|
|
13056
|
-
...opJournal !== void 0 ? { journal: opJournal } : {}
|
|
13057
|
-
};
|
|
13058
|
-
}
|
|
13059
|
-
function connectionBindingFor(services, enrollment) {
|
|
13060
|
-
if (!enrollment?.connectionInstanceId) return null;
|
|
13061
|
-
const opStream = opStreamDepsFor(services, enrollment.opStream === true);
|
|
13062
|
-
return {
|
|
13063
|
-
workspaceId: enrollment.workspaceId,
|
|
13064
|
-
connectionInstanceId: enrollment.connectionInstanceId,
|
|
13065
|
-
workspaceRoot: enrollment.workspaceRoot,
|
|
13066
|
-
...opStream !== void 0 ? { opStream } : {},
|
|
13067
|
-
operationResourcePolicy: enrollment.operationPolicy,
|
|
13068
|
-
operationResourcePolicySupported: enrollment.agentCapabilities.operationResourcePolicy === true,
|
|
13069
|
-
operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true
|
|
13070
|
-
};
|
|
13071
|
-
}
|
|
13072
|
-
async function resolveRoutedMachineConnection(db, ids, sandbox) {
|
|
13073
|
-
if (!sandbox.enrollmentId) return null;
|
|
13074
|
-
if (sandbox.scope !== "user") {
|
|
13075
|
-
return await getLiveEnrollmentConnection(
|
|
13076
|
-
db,
|
|
13077
|
-
ids.resourceAccountId && ids.resourceSubjectId ? {
|
|
13078
|
-
accountId: ids.resourceAccountId,
|
|
13079
|
-
workspaceId: ids.workspaceId,
|
|
13080
|
-
subjectId: ids.resourceSubjectId
|
|
13081
|
-
} : sandbox.workspaceId,
|
|
13082
|
-
sandbox.enrollmentId
|
|
13083
|
-
);
|
|
13084
|
-
}
|
|
13085
|
-
const authority = ids.personalMachineAttempt;
|
|
13086
|
-
if (!authority) return null;
|
|
13087
|
-
return await resolvePersonalMachineConnectionForAttempt(db, {
|
|
13088
|
-
accountId: authority.accountId,
|
|
13089
|
-
workspaceId: ids.workspaceId,
|
|
13090
|
-
subjectId: authority.subjectId,
|
|
13091
|
-
sessionId: ids.sessionId,
|
|
13092
|
-
turnId: authority.turnId,
|
|
13093
|
-
attemptId: authority.attemptId,
|
|
13094
|
-
executionGeneration: authority.executionGeneration,
|
|
13095
|
-
enrollmentId: sandbox.enrollmentId,
|
|
13096
|
-
requireActiveSandbox: true
|
|
13097
|
-
});
|
|
13098
|
-
}
|
|
13099
|
-
async function establishSelfhostedTurnSession(services, args) {
|
|
13100
|
-
const { db, settings, bus, onOp } = services;
|
|
13101
|
-
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
13102
|
-
const opStream = opStreamDepsFor(services, args.opStream);
|
|
13103
|
-
const { client, session } = await buildSelfhostedBackendSession({
|
|
13104
|
-
workspaceId: args.workspaceId,
|
|
13105
|
-
...args.controlWorkspaceId !== void 0 ? { controlWorkspaceId: args.controlWorkspaceId } : {},
|
|
13106
|
-
agentId: args.agentId,
|
|
13107
|
-
connectionInstanceId: args.connectionInstanceId,
|
|
13108
|
-
workspaceRoot: args.workspaceRoot,
|
|
13109
|
-
relay: relayConfigFromSettings(settings),
|
|
13110
|
-
controlRpcFactory: controlRpcFactory(bus),
|
|
13111
|
-
epoch: args.epoch,
|
|
13112
|
-
environment: args.environment,
|
|
13113
|
-
...args.transientExecEnvironment !== void 0 ? { transientExecEnvironment: args.transientExecEnvironment } : {},
|
|
13114
|
-
// Give this turn's exec ops the long deadline (control ops stay short) so a real
|
|
13115
|
-
// command is not killed at the control wall.
|
|
13116
|
-
timeoutMs,
|
|
13117
|
-
execTimeoutMs,
|
|
13118
|
-
operationResourcePolicy: args.operationResourcePolicy,
|
|
13119
|
-
operationResourcePolicySupported: args.operationResourcePolicySupported,
|
|
13120
|
-
operationCpuQuotaSupported: args.operationCpuQuotaSupported,
|
|
13121
|
-
resolveOperationAdmission: async () => {
|
|
13122
|
-
const enrollment = args.personalMachineAttempt ? await resolvePersonalMachineConnectionForAttempt(db, {
|
|
13123
|
-
accountId: args.personalMachineAttempt.accountId,
|
|
13124
|
-
workspaceId: args.workspaceId,
|
|
13125
|
-
subjectId: args.personalMachineAttempt.subjectId,
|
|
13126
|
-
sessionId: args.personalMachineAttempt.sessionId,
|
|
13127
|
-
turnId: args.personalMachineAttempt.turnId,
|
|
13128
|
-
attemptId: args.personalMachineAttempt.attemptId,
|
|
13129
|
-
executionGeneration: args.personalMachineAttempt.executionGeneration,
|
|
13130
|
-
enrollmentId: args.agentId,
|
|
13131
|
-
requireActiveSandbox: true
|
|
13132
|
-
}) : await getLiveEnrollmentConnection(
|
|
13133
|
-
db,
|
|
13134
|
-
args.controlWorkspaceId ?? args.workspaceId,
|
|
13135
|
-
args.agentId
|
|
13136
|
-
);
|
|
13137
|
-
const binding = connectionBindingFor(services, enrollment);
|
|
13138
|
-
if (!binding) return null;
|
|
13139
|
-
if (!binding.workspaceRoot) {
|
|
13140
|
-
throw new ActiveBackendUnresolvableError(
|
|
13141
|
-
"workspace_root_unavailable",
|
|
13142
|
-
"Connected Machine no longer has a reported workspace root"
|
|
13143
|
-
);
|
|
13144
|
-
}
|
|
13145
|
-
return {
|
|
13146
|
-
...binding,
|
|
13147
|
-
workspaceRoot: resolveConnectedMachineWorkspaceRoot(binding.workspaceRoot, args.workingDir)
|
|
13148
|
-
};
|
|
13149
|
-
},
|
|
13150
|
-
...args.backgroundCommandAttempt ? {
|
|
13151
|
-
adoptBackgroundCommand: async (command) => {
|
|
13152
|
-
const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
|
|
13153
|
-
accountId: args.accountId,
|
|
13154
|
-
workspaceId: args.workspaceId,
|
|
13155
|
-
sessionId: args.sessionId,
|
|
13156
|
-
turnId: args.backgroundCommandAttempt.turnId,
|
|
13157
|
-
executionGeneration: args.backgroundCommandAttempt.executionGeneration,
|
|
13158
|
-
attemptId: args.backgroundCommandAttempt.attemptId,
|
|
13159
|
-
commandId: crypto.randomUUID(),
|
|
13160
|
-
controlWorkspaceId: command.controlWorkspaceId,
|
|
13161
|
-
enrollmentId: command.enrollmentId,
|
|
13162
|
-
connectionInstanceId: command.connectionInstanceId,
|
|
13163
|
-
opId: command.opId,
|
|
13164
|
-
command: command.command
|
|
13165
|
-
});
|
|
13166
|
-
return { commandId: adopted.id };
|
|
13167
|
-
}
|
|
13168
|
-
} : {},
|
|
13169
|
-
settleBackgroundCommand: async (command) => {
|
|
13170
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
13171
|
-
accountId: args.accountId,
|
|
13172
|
-
workspaceId: args.workspaceId,
|
|
13173
|
-
sessionId: args.sessionId,
|
|
13174
|
-
commandId: command.commandId,
|
|
13175
|
-
controlWorkspaceId: command.controlWorkspaceId,
|
|
13176
|
-
enrollmentId: command.enrollmentId,
|
|
13177
|
-
connectionInstanceId: command.connectionInstanceId,
|
|
13178
|
-
opId: command.opId,
|
|
13179
|
-
outcome: command.outcome,
|
|
13180
|
-
exitCode: command.exitCode,
|
|
13181
|
-
reason: command.reason
|
|
13182
|
-
});
|
|
13183
|
-
},
|
|
13184
|
-
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
13185
|
-
...onOp !== void 0 ? { onOp } : {},
|
|
13186
|
-
// The streaming exec transport — present iff the machine advertised the
|
|
13187
|
-
// capability AND the server flag is on. Connected Machine exec fails closed
|
|
13188
|
-
// when it is absent; there is no request/reply downgrade.
|
|
13189
|
-
...opStream !== void 0 ? { opStream } : {}
|
|
13190
|
-
});
|
|
13191
|
-
return {
|
|
13192
|
-
client,
|
|
13193
|
-
session,
|
|
13194
|
-
sessionState: { agentId: args.agentId },
|
|
13195
|
-
instanceId: args.agentId,
|
|
13196
|
-
backendId: "selfhosted"
|
|
12485
|
+
requireResolvedSandboxForMutation,
|
|
12486
|
+
finalizeTurnOpStreamOps,
|
|
12487
|
+
onHomeSandboxRebound,
|
|
12488
|
+
runWorkspaceMutationForSandbox,
|
|
12489
|
+
stopLeaseHeartbeat,
|
|
12490
|
+
flushRuntimeBatcher,
|
|
12491
|
+
publishSandboxLifecycleEvents,
|
|
12492
|
+
publishSandboxLost,
|
|
12493
|
+
startLeaseHeartbeat
|
|
13197
12494
|
};
|
|
13198
12495
|
}
|
|
13199
|
-
function routingEnabled(settings) {
|
|
13200
|
-
return settings.sandboxSelfhostedEnabled === true;
|
|
13201
|
-
}
|
|
13202
|
-
function lazyProvisionEnabled(settings) {
|
|
13203
|
-
return settings.sandboxLazyProvisionEnabled === true && settings.sandboxOwnershipEnabled === true;
|
|
13204
|
-
}
|
|
13205
12496
|
|
|
13206
|
-
// src/activities/agent-turn/sandbox-
|
|
12497
|
+
// src/activities/agent-turn/sandbox-establish.ts
|
|
13207
12498
|
import {
|
|
12499
|
+
getEnrollment,
|
|
12500
|
+
getSandbox,
|
|
13208
12501
|
readActiveSandbox as readActiveSandbox2,
|
|
12502
|
+
getLiveEnrollmentConnection,
|
|
12503
|
+
assertPersonalMachineForAttempt
|
|
12504
|
+
} from "@opengeni/db";
|
|
12505
|
+
import {
|
|
12506
|
+
withRunCredentialsSession as withRunCredentialsSession2,
|
|
12507
|
+
runOwnedSandboxSetup as runOwnedSandboxSetup2,
|
|
12508
|
+
markModelPreparationFirstSandboxOperation,
|
|
12509
|
+
recordModelPreparationMeasurement,
|
|
12510
|
+
sdkBackendIdForSandboxBackend,
|
|
12511
|
+
resolveConnectedMachineWorkspaceRoot
|
|
12512
|
+
} from "@opengeni/runtime";
|
|
12513
|
+
import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/core";
|
|
12514
|
+
|
|
12515
|
+
// src/activities/agent-turn/sandbox-route.ts
|
|
12516
|
+
import {
|
|
12517
|
+
readActiveSandbox,
|
|
13209
12518
|
setActiveSandbox
|
|
13210
12519
|
} from "@opengeni/db";
|
|
13211
12520
|
import {
|
|
@@ -13318,7 +12627,7 @@ async function reconcileActiveSandboxPointer(db, ids, pointer, loadRecord, publi
|
|
|
13318
12627
|
).catch(() => void 0);
|
|
13319
12628
|
return { pointer: reset.pointer, record: null };
|
|
13320
12629
|
}
|
|
13321
|
-
const reread = await
|
|
12630
|
+
const reread = await readActiveSandbox(db, ids.workspaceId, ids.sessionId).catch(() => null);
|
|
13322
12631
|
if (!reread) {
|
|
13323
12632
|
return { pointer, record: null };
|
|
13324
12633
|
}
|
|
@@ -13387,7 +12696,7 @@ async function resolveSandboxRoute(deps) {
|
|
|
13387
12696
|
logicalSandboxSettings
|
|
13388
12697
|
} = deps;
|
|
13389
12698
|
const routingOn = routingEnabled(settings);
|
|
13390
|
-
let activeSandboxPointer = routingOn ? await
|
|
12699
|
+
let activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
|
|
13391
12700
|
let activeSandboxRecord = null;
|
|
13392
12701
|
if (routingOn) {
|
|
13393
12702
|
const reconciled = await reconcileActiveSandboxPointer(
|
|
@@ -13398,7 +12707,7 @@ async function resolveSandboxRoute(deps) {
|
|
|
13398
12707
|
sessionId: input.sessionId
|
|
13399
12708
|
},
|
|
13400
12709
|
activeSandboxPointer,
|
|
13401
|
-
(sandboxId) =>
|
|
12710
|
+
(sandboxId) => getSandbox(
|
|
13402
12711
|
db,
|
|
13403
12712
|
fileAuthoritySubjectId ? {
|
|
13404
12713
|
accountId: input.accountId,
|
|
@@ -13541,7 +12850,7 @@ async function establishTurnSandbox(deps) {
|
|
|
13541
12850
|
throw new Error("personal Connected Machine use was not admitted for this turn");
|
|
13542
12851
|
}
|
|
13543
12852
|
}
|
|
13544
|
-
const liveMachineEnrollment = await
|
|
12853
|
+
const liveMachineEnrollment = await getLiveEnrollmentConnection(
|
|
13545
12854
|
db,
|
|
13546
12855
|
activeSandboxRecord.workspaceId,
|
|
13547
12856
|
activeSandboxRecord.enrollmentId
|
|
@@ -13556,7 +12865,7 @@ async function establishTurnSandbox(deps) {
|
|
|
13556
12865
|
"Connected Machine has not reported an absolute workspace root; reconnect it with a current agent"
|
|
13557
12866
|
);
|
|
13558
12867
|
}
|
|
13559
|
-
const machineWorkspaceRoot =
|
|
12868
|
+
const machineWorkspaceRoot = resolveConnectedMachineWorkspaceRoot(
|
|
13560
12869
|
machineEnrollment.workspaceRoot,
|
|
13561
12870
|
activeSandboxPointer.workingDir
|
|
13562
12871
|
);
|
|
@@ -14281,6 +13590,16 @@ async function selectXaiTurnCapacity(deps) {
|
|
|
14281
13590
|
authoritySnapshot
|
|
14282
13591
|
});
|
|
14283
13592
|
const leaseStartedAtMs = performance.now();
|
|
13593
|
+
await refreshExhaustedXaiQuota({
|
|
13594
|
+
db,
|
|
13595
|
+
settings: deps.settings,
|
|
13596
|
+
accountId: input.accountId,
|
|
13597
|
+
workspaceId: input.workspaceId,
|
|
13598
|
+
subjectId,
|
|
13599
|
+
sessionId: input.sessionId,
|
|
13600
|
+
turnId: turn.id,
|
|
13601
|
+
authoritySnapshot
|
|
13602
|
+
});
|
|
14284
13603
|
const leased = await acquireXaiCredentialLease(db, {
|
|
14285
13604
|
accountId: input.accountId,
|
|
14286
13605
|
workspaceId: input.workspaceId,
|
|
@@ -15354,6 +14673,8 @@ async function prepareRunCredentials(deps) {
|
|
|
15354
14673
|
|
|
15355
14674
|
// src/activities/agent-turn/tool-environment.ts
|
|
15356
14675
|
import {
|
|
14676
|
+
beginConnectorActionExecution,
|
|
14677
|
+
completeConnectorActionExecution,
|
|
15357
14678
|
getScheduledVariableSetExpectedGenerationForAttempt,
|
|
15358
14679
|
getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
|
|
15359
14680
|
listWorkspaceGatewayCustomModels as listWorkspaceGatewayCustomModels2,
|
|
@@ -15361,6 +14682,8 @@ import {
|
|
|
15361
14682
|
listOrganizationModelProviderCustomModelsForWorkspace as listOrganizationModelProviderCustomModelsForWorkspace2,
|
|
15362
14683
|
organizationModelProviderConnectionActiveForWorkspace,
|
|
15363
14684
|
persistAttemptToolCatalog,
|
|
14685
|
+
prepareConnectorActionApproval,
|
|
14686
|
+
previewConnectorActionApproval,
|
|
15364
14687
|
namedSubjectHasLiveWorkspaceAuthority as namedSubjectHasLiveWorkspaceAuthority2,
|
|
15365
14688
|
updateSessionTitleWithEvent as updateSessionTitleWithEvent2,
|
|
15366
14689
|
withCodexAppsRequestAuthorization,
|
|
@@ -15375,8 +14698,7 @@ import {
|
|
|
15375
14698
|
} from "@opengeni/runtime";
|
|
15376
14699
|
|
|
15377
14700
|
// src/activities/google-drive-publication.ts
|
|
15378
|
-
import { createHash as
|
|
15379
|
-
import { AttemptToolApprovalRequiredError } from "@opengeni/codemode";
|
|
14701
|
+
import { createHash as createHash10 } from "crypto";
|
|
15380
14702
|
import {
|
|
15381
14703
|
GOOGLE_DRIVE_FILE_SCOPE,
|
|
15382
14704
|
GOOGLE_DRIVE_FULL_SCOPE,
|
|
@@ -15390,10 +14712,7 @@ import {
|
|
|
15390
14712
|
googleDriveScopesAllowCapability
|
|
15391
14713
|
} from "@opengeni/contracts/google-drive";
|
|
15392
14714
|
import {
|
|
15393
|
-
beginConnectorActionExecution,
|
|
15394
|
-
completeConnectorActionExecution,
|
|
15395
14715
|
getConnectionMetadata,
|
|
15396
|
-
prepareConnectorActionApproval,
|
|
15397
14716
|
requireFile as requireFile2,
|
|
15398
14717
|
namedSubjectHasLiveWorkspaceAuthority
|
|
15399
14718
|
} from "@opengeni/db";
|
|
@@ -15401,6 +14720,7 @@ import {
|
|
|
15401
14720
|
readAuthorizedEditableArtifactMaterialization
|
|
15402
14721
|
} from "@opengeni/db/editable-artifact-durable-export";
|
|
15403
14722
|
import { readResponseJsonBounded, undiciFetch } from "@opengeni/network";
|
|
14723
|
+
import { ConnectorActionExecutionError } from "@opengeni/runtime";
|
|
15404
14724
|
import { retryWhileMissing as retryWhileMissing4 } from "@opengeni/storage";
|
|
15405
14725
|
var DRIVE_API = "https://www.googleapis.com/drive/v3";
|
|
15406
14726
|
var DRIVE_UPLOAD_API = "https://www.googleapis.com/upload/drive/v3";
|
|
@@ -15416,9 +14736,6 @@ var defaultPorts = {
|
|
|
15416
14736
|
getMembership: namedSubjectHasLiveWorkspaceAuthority,
|
|
15417
14737
|
readMaterialization: readAuthorizedEditableArtifactMaterialization,
|
|
15418
14738
|
requireFile: requireFile2,
|
|
15419
|
-
prepare: prepareConnectorActionApproval,
|
|
15420
|
-
begin: beginConnectorActionExecution,
|
|
15421
|
-
complete: completeConnectorActionExecution,
|
|
15422
14739
|
fetch: undiciFetch
|
|
15423
14740
|
};
|
|
15424
14741
|
async function resolveGoogleDrivePublicationTarget(db, workspace, delegations, ports = defaultPorts) {
|
|
@@ -15481,7 +14798,7 @@ function googleDrivePublicationConnectorCall(target, rawRequest, approvalId) {
|
|
|
15481
14798
|
sourceSha256: request.file.sha256
|
|
15482
14799
|
},
|
|
15483
14800
|
canonicalReceipt: { ...request.file },
|
|
15484
|
-
idempotencyKeyDigest:
|
|
14801
|
+
idempotencyKeyDigest: createHash10("sha256").update(request.idempotencyKey, "utf8").digest("hex")
|
|
15485
14802
|
}
|
|
15486
14803
|
};
|
|
15487
14804
|
}
|
|
@@ -15589,24 +14906,6 @@ function createGoogleDrivePublicationAttemptTool(input) {
|
|
|
15589
14906
|
approval: "policy",
|
|
15590
14907
|
execute: async (raw, context) => {
|
|
15591
14908
|
const request = GoogleDrivePublicationToolInput.parse(raw);
|
|
15592
|
-
const connectorCall = googleDrivePublicationConnectorCall(
|
|
15593
|
-
input.target,
|
|
15594
|
-
request,
|
|
15595
|
-
context.operationId
|
|
15596
|
-
);
|
|
15597
|
-
let requestId = null;
|
|
15598
|
-
if (context.caller.kind === "codemode") {
|
|
15599
|
-
const admission = await ports.begin(input.db, input.identity, connectorCall);
|
|
15600
|
-
if (!admission.allowed) {
|
|
15601
|
-
throw new Error(
|
|
15602
|
-
admission.reason === "uncertain_retry" || admission.reason === "already_executed" ? "Google Drive publication outcome is unknown: a previous attempt may have already sent this publication to Google Drive. Verify in Drive before retrying with a new idempotency key." : admission.reason === "not_executed" ? "Google Drive publication was not executed: the previous attempt failed before any request reached Google Drive. Retrying with a new call is safe." : `Google Drive publication was not executed: ${admission.reason}. No request reached Google Drive.`
|
|
15603
|
-
);
|
|
15604
|
-
}
|
|
15605
|
-
if (!admission.managed) {
|
|
15606
|
-
throw new Error("Google Drive publication has no explicit connector action policy");
|
|
15607
|
-
}
|
|
15608
|
-
requestId = admission.requestId;
|
|
15609
|
-
}
|
|
15610
14909
|
let providerRequestStarted = false;
|
|
15611
14910
|
try {
|
|
15612
14911
|
const receipt = await executeGoogleDrivePublication(
|
|
@@ -15625,53 +14924,21 @@ function createGoogleDrivePublicationAttemptTool(input) {
|
|
|
15625
14924
|
},
|
|
15626
14925
|
ports
|
|
15627
14926
|
);
|
|
15628
|
-
if (requestId) {
|
|
15629
|
-
await ports.complete(input.db, {
|
|
15630
|
-
accountId: input.identity.accountId,
|
|
15631
|
-
workspaceId: input.identity.workspaceId,
|
|
15632
|
-
requestId,
|
|
15633
|
-
attemptId: input.identity.attemptId,
|
|
15634
|
-
outcome: "completed"
|
|
15635
|
-
});
|
|
15636
|
-
}
|
|
15637
14927
|
return {
|
|
15638
14928
|
content: [{ type: "text", text: JSON.stringify(receipt) }],
|
|
15639
14929
|
structuredContent: receipt
|
|
15640
14930
|
};
|
|
15641
14931
|
} catch (error) {
|
|
15642
|
-
if (requestId) {
|
|
15643
|
-
await ports.complete(input.db, {
|
|
15644
|
-
accountId: input.identity.accountId,
|
|
15645
|
-
workspaceId: input.identity.workspaceId,
|
|
15646
|
-
requestId,
|
|
15647
|
-
attemptId: input.identity.attemptId,
|
|
15648
|
-
outcome: providerRequestStarted ? "uncertain" : "not_executed"
|
|
15649
|
-
});
|
|
15650
|
-
}
|
|
15651
14932
|
const detail = error instanceof Error ? error.message : String(error);
|
|
15652
|
-
throw new
|
|
14933
|
+
throw new ConnectorActionExecutionError(
|
|
15653
14934
|
providerRequestStarted ? `Google Drive publication outcome is unknown: a provider request may have been sent before the failure. Verify in Drive before retrying. Cause: ${detail}` : `Google Drive publication was not executed: no request reached Google Drive. Cause: ${detail}`,
|
|
14935
|
+
providerRequestStarted ? "uncertain" : "not_executed",
|
|
15654
14936
|
{ cause: error }
|
|
15655
14937
|
);
|
|
15656
14938
|
}
|
|
15657
14939
|
}
|
|
15658
14940
|
};
|
|
15659
14941
|
}
|
|
15660
|
-
async function authorizeGoogleDrivePublicationAttempt(input) {
|
|
15661
|
-
const request = GoogleDrivePublicationToolInput.parse(input.arguments);
|
|
15662
|
-
const preparation = await (input.ports ?? defaultPorts).prepare(
|
|
15663
|
-
input.db,
|
|
15664
|
-
input.identity,
|
|
15665
|
-
googleDrivePublicationConnectorCall(input.target, request, input.approvalId)
|
|
15666
|
-
);
|
|
15667
|
-
if (!preparation.managed) {
|
|
15668
|
-
throw new Error("Google Drive publication has no explicit connector action policy");
|
|
15669
|
-
}
|
|
15670
|
-
if (preparation.decision === "ask") throw new AttemptToolApprovalRequiredError();
|
|
15671
|
-
if (preparation.decision === "block") {
|
|
15672
|
-
throw new Error("Google Drive publication is blocked by connector action policy");
|
|
15673
|
-
}
|
|
15674
|
-
}
|
|
15675
14942
|
async function executeGoogleDrivePublication(input, ports = defaultPorts) {
|
|
15676
14943
|
const request = GoogleDrivePublicationToolInput.parse(input.request);
|
|
15677
14944
|
const connection = await ports.getConnection(
|
|
@@ -15719,7 +14986,7 @@ async function executeGoogleDrivePublication(input, ports = defaultPorts) {
|
|
|
15719
14986
|
);
|
|
15720
14987
|
if (!object) throw new Error("Editable artifact export object is missing");
|
|
15721
14988
|
const bytes = object.bytes;
|
|
15722
|
-
if (bytes.byteLength !== request.file.sizeBytes ||
|
|
14989
|
+
if (bytes.byteLength !== request.file.sizeBytes || createHash10("sha256").update(bytes).digest("hex") !== request.file.sha256) {
|
|
15723
14990
|
throw new Error("Editable artifact export bytes failed verification");
|
|
15724
14991
|
}
|
|
15725
14992
|
const authorizedFetch = async (url, init) => {
|
|
@@ -15801,7 +15068,7 @@ function assertPublicationConnection(input, connection) {
|
|
|
15801
15068
|
}
|
|
15802
15069
|
}
|
|
15803
15070
|
function publicationActorReplicaId(identity) {
|
|
15804
|
-
const value =
|
|
15071
|
+
const value = createHash10("sha256").update(
|
|
15805
15072
|
`google-drive-publication:${identity.sessionId}:${identity.turnId}:${identity.attemptId}:${identity.executionGeneration}`,
|
|
15806
15073
|
"utf8"
|
|
15807
15074
|
).digest("hex").slice(0, 16);
|
|
@@ -15846,7 +15113,7 @@ function officeFormat(modality, contentType) {
|
|
|
15846
15113
|
return expected.format;
|
|
15847
15114
|
}
|
|
15848
15115
|
function deterministicUuid(seed) {
|
|
15849
|
-
const bytes =
|
|
15116
|
+
const bytes = createHash10("sha256").update(seed, "utf8").digest().subarray(0, 16);
|
|
15850
15117
|
bytes[6] = (bytes[6] ?? 0) & 15 | 80;
|
|
15851
15118
|
bytes[8] = (bytes[8] ?? 0) & 63 | 128;
|
|
15852
15119
|
const hex = bytes.toString("hex");
|
|
@@ -15984,7 +15251,7 @@ function driveReceipt(input, file, replayed) {
|
|
|
15984
15251
|
});
|
|
15985
15252
|
}
|
|
15986
15253
|
function publicationOperationKey(input) {
|
|
15987
|
-
return
|
|
15254
|
+
return createHash10("sha256").update(
|
|
15988
15255
|
JSON.stringify({
|
|
15989
15256
|
workspaceId: input.workspaceId,
|
|
15990
15257
|
connectionId: input.connectionId,
|
|
@@ -16202,100 +15469,6 @@ function isGitHubApiDestination(value) {
|
|
|
16202
15469
|
}
|
|
16203
15470
|
}
|
|
16204
15471
|
|
|
16205
|
-
// src/activities/api-integrations.ts
|
|
16206
|
-
import {
|
|
16207
|
-
createGraphqlMcpServer,
|
|
16208
|
-
createOpenApiMcpServer,
|
|
16209
|
-
createPinnedIntegrationTransport
|
|
16210
|
-
} from "@opengeni/capabilities";
|
|
16211
|
-
function buildApiIntegrationServersForTurn(input) {
|
|
16212
|
-
const transport = createPinnedIntegrationTransport({
|
|
16213
|
-
network: input.settings,
|
|
16214
|
-
...input.fetchImpl ? { fetchImpl: input.fetchImpl } : {}
|
|
16215
|
-
});
|
|
16216
|
-
return input.integrations.map((integration) => {
|
|
16217
|
-
const authority = {
|
|
16218
|
-
...input.authority,
|
|
16219
|
-
...integration.connectionRef?.connectionId ? { connectionRef: integration.connectionRef.connectionId } : {}
|
|
16220
|
-
};
|
|
16221
|
-
const credentialResolver = integration.connectionRef ? integrationCredentialResolver(input, integration) : void 0;
|
|
16222
|
-
const server = integration.revision.protocol === "openapi" ? createOpenApiMcpServer({
|
|
16223
|
-
revision: integration.revision,
|
|
16224
|
-
transport,
|
|
16225
|
-
authority,
|
|
16226
|
-
...credentialResolver ? { credentialResolver } : {}
|
|
16227
|
-
}) : createGraphqlMcpServer({
|
|
16228
|
-
revision: integration.revision,
|
|
16229
|
-
endpoint: integration.baseUrl,
|
|
16230
|
-
transport,
|
|
16231
|
-
authority,
|
|
16232
|
-
...credentialResolver ? { credentialResolver } : {}
|
|
16233
|
-
});
|
|
16234
|
-
return {
|
|
16235
|
-
id: integration.serverId,
|
|
16236
|
-
server,
|
|
16237
|
-
...integration.connectionRef?.connectionId ? { resolvedConnectionId: integration.connectionRef.connectionId } : {}
|
|
16238
|
-
};
|
|
16239
|
-
});
|
|
16240
|
-
}
|
|
16241
|
-
function integrationCredentialResolver(input, integration) {
|
|
16242
|
-
const connectionRef = integration.connectionRef;
|
|
16243
|
-
if (!connectionRef) throw new Error("Integration credential resolver requires a connection");
|
|
16244
|
-
return {
|
|
16245
|
-
resolve: async (request) => {
|
|
16246
|
-
const result = await input.resolveCredential({
|
|
16247
|
-
workspaceId: input.authority.workspaceId,
|
|
16248
|
-
serverId: integration.serverId,
|
|
16249
|
-
toolName: request.operationKey,
|
|
16250
|
-
connectionRef,
|
|
16251
|
-
destinationUrl: request.destinationUrl,
|
|
16252
|
-
credentialTarget: "http_api",
|
|
16253
|
-
forceRefresh: request.forceRefresh === true
|
|
16254
|
-
});
|
|
16255
|
-
if (result.status === "auth_needed") {
|
|
16256
|
-
await publishAuthNeeded(
|
|
16257
|
-
input,
|
|
16258
|
-
integration.serverId,
|
|
16259
|
-
request.operationKey,
|
|
16260
|
-
result,
|
|
16261
|
-
connectionRef
|
|
16262
|
-
);
|
|
16263
|
-
return null;
|
|
16264
|
-
}
|
|
16265
|
-
const destination = new URL(request.destinationUrl);
|
|
16266
|
-
return {
|
|
16267
|
-
audience: { origin: destination.origin, pathPrefix: "/" },
|
|
16268
|
-
placements: result.placements ?? Object.entries(result.headers).map(([name, value]) => ({
|
|
16269
|
-
carrier: "header",
|
|
16270
|
-
name,
|
|
16271
|
-
value
|
|
16272
|
-
})),
|
|
16273
|
-
...result.authorizeProviderRequest ? { authorizeProviderRequest: result.authorizeProviderRequest } : {},
|
|
16274
|
-
...result.expiresAt ? { expiresAt: result.expiresAt.toISOString() } : {},
|
|
16275
|
-
...connectionRef.scopes ? { scope: [...connectionRef.scopes] } : {}
|
|
16276
|
-
};
|
|
16277
|
-
}
|
|
16278
|
-
};
|
|
16279
|
-
}
|
|
16280
|
-
async function publishAuthNeeded(input, serverId, toolName, result, connectionRef) {
|
|
16281
|
-
try {
|
|
16282
|
-
await input.onAuthNeeded?.({
|
|
16283
|
-
serverId,
|
|
16284
|
-
toolName,
|
|
16285
|
-
providerDomain: result.providerDomain,
|
|
16286
|
-
...result.provider ? { provider: result.provider } : {},
|
|
16287
|
-
reason: result.reason,
|
|
16288
|
-
...result.connectionId ? { connectionId: result.connectionId } : {},
|
|
16289
|
-
...result.authoritySource === "host" || connectionRef.authoritySource === "host" ? { authoritySource: "host" } : {},
|
|
16290
|
-
...result.scopes ? { scopes: result.scopes } : {},
|
|
16291
|
-
...result.resource ? { resource: result.resource } : {},
|
|
16292
|
-
...result.selectedResources ? { selectedResources: result.selectedResources } : {},
|
|
16293
|
-
...result.authorizationUrl ? { authorizationUrl: result.authorizationUrl } : {}
|
|
16294
|
-
});
|
|
16295
|
-
} catch {
|
|
16296
|
-
}
|
|
16297
|
-
}
|
|
16298
|
-
|
|
16299
15472
|
// src/github-rest-mcp.ts
|
|
16300
15473
|
import {
|
|
16301
15474
|
mergeToolRefs
|
|
@@ -16306,18 +15479,12 @@ import {
|
|
|
16306
15479
|
} from "@opengeni/contracts/personal-github";
|
|
16307
15480
|
import { personalGitHubRepositoryResources as personalGitHubRepositoryResources2 } from "@opengeni/core";
|
|
16308
15481
|
import {
|
|
16309
|
-
beginConnectorActionExecution as beginConnectorActionExecution2,
|
|
16310
|
-
completeConnectorActionExecution as completeConnectorActionExecution2,
|
|
16311
15482
|
getConnectionMetadata as getConnectionMetadata2,
|
|
16312
15483
|
getPersonalGitHubRepositorySelectionState,
|
|
16313
15484
|
hasAuditableGitHubInstallationAuthority,
|
|
16314
15485
|
listGitHubInstallationAccessForWorkspace,
|
|
16315
|
-
prepareConnectorActionApproval as prepareConnectorActionApproval2,
|
|
16316
15486
|
resolveAcceptedConnectionUse as resolveAcceptedConnectionUse2
|
|
16317
15487
|
} from "@opengeni/db";
|
|
16318
|
-
import {
|
|
16319
|
-
AttemptToolApprovalRequiredError as AttemptToolApprovalRequiredError2
|
|
16320
|
-
} from "@opengeni/codemode";
|
|
16321
15488
|
import { createGitHubAppInstallationTokenWithExpiry as createGitHubAppInstallationTokenWithExpiry2, githubAppBotIdentity as githubAppBotIdentity2 } from "@opengeni/github";
|
|
16322
15489
|
import {
|
|
16323
15490
|
ConnectorActionBindingRejectedError,
|
|
@@ -16331,8 +15498,7 @@ import {
|
|
|
16331
15498
|
GITHUB_REST_TOOL_NAMES,
|
|
16332
15499
|
GitHubRestAuthorityError,
|
|
16333
15500
|
githubRestConnectorActionOutcome,
|
|
16334
|
-
GitHubRestMcpServer
|
|
16335
|
-
GitHubRestMutationNotExecutedError
|
|
15501
|
+
GitHubRestMcpServer
|
|
16336
15502
|
} from "@opengeni/runtime/github-rest-mcp";
|
|
16337
15503
|
var PERSONAL_GITHUB_DELEGATION_SERVER_ID = "github:personal";
|
|
16338
15504
|
var INTERNAL_GITHUB_MCP_TIMEOUT_MS = 2e4;
|
|
@@ -16342,8 +15508,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16342
15508
|
settings: input.settings,
|
|
16343
15509
|
tools: input.tools,
|
|
16344
15510
|
localMcpServers: [],
|
|
16345
|
-
connectorBindings: []
|
|
16346
|
-
authorizeCodemodeCall: null
|
|
15511
|
+
connectorBindings: []
|
|
16347
15512
|
};
|
|
16348
15513
|
}
|
|
16349
15514
|
const appRepositories = appRepositoryAuthority(input.resources);
|
|
@@ -16356,8 +15521,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16356
15521
|
settings: input.settings,
|
|
16357
15522
|
tools: input.tools,
|
|
16358
15523
|
localMcpServers: [],
|
|
16359
|
-
connectorBindings: []
|
|
16360
|
-
authorizeCodemodeCall: null
|
|
15524
|
+
connectorBindings: []
|
|
16361
15525
|
};
|
|
16362
15526
|
}
|
|
16363
15527
|
const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
|
|
@@ -16372,7 +15536,6 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16372
15536
|
const localMcpServers = [];
|
|
16373
15537
|
const toolRefs = [];
|
|
16374
15538
|
const connectorBindings = [];
|
|
16375
|
-
const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
|
|
16376
15539
|
if (appRepositories.length > 0) {
|
|
16377
15540
|
const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
|
|
16378
15541
|
localMcpServers.push({
|
|
@@ -16381,8 +15544,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16381
15544
|
serverId: GITHUB_REST_MCP_APP_SERVER_ID,
|
|
16382
15545
|
authorityKind: "workspace_app",
|
|
16383
15546
|
repositories: appRepositories,
|
|
16384
|
-
resolveAuthority: appResolver
|
|
16385
|
-
completeCodemodeAction: codemodeActions.complete
|
|
15547
|
+
resolveAuthority: appResolver
|
|
16386
15548
|
})
|
|
16387
15549
|
});
|
|
16388
15550
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub \u2014 OpenGeni bot"));
|
|
@@ -16398,8 +15560,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16398
15560
|
serverId: GITHUB_REST_MCP_PERSONAL_SERVER_ID,
|
|
16399
15561
|
authorityKind: "personal_oauth",
|
|
16400
15562
|
repositories: personalRepositories,
|
|
16401
|
-
resolveAuthority: personalAuthorityResolver(input)
|
|
16402
|
-
completeCodemodeAction: codemodeActions.complete
|
|
15563
|
+
resolveAuthority: personalAuthorityResolver(input)
|
|
16403
15564
|
})
|
|
16404
15565
|
});
|
|
16405
15566
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub \u2014 My account"));
|
|
@@ -16415,59 +15576,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16415
15576
|
},
|
|
16416
15577
|
tools: mergeToolRefs(input.tools, toolRefs),
|
|
16417
15578
|
localMcpServers,
|
|
16418
|
-
connectorBindings
|
|
16419
|
-
authorizeCodemodeCall: codemodeActions.authorize
|
|
16420
|
-
};
|
|
16421
|
-
}
|
|
16422
|
-
function createCodemodeActionLifecycle(input, bindings) {
|
|
16423
|
-
const identity = {
|
|
16424
|
-
accountId: input.accountId,
|
|
16425
|
-
workspaceId: input.workspaceId,
|
|
16426
|
-
sessionId: input.sessionId,
|
|
16427
|
-
turnId: input.turn.id,
|
|
16428
|
-
attemptId: input.attemptId,
|
|
16429
|
-
executionGeneration: input.turn.executionGeneration,
|
|
16430
|
-
initiator: input.turn.initiator
|
|
16431
|
-
};
|
|
16432
|
-
const executing = /* @__PURE__ */ new Map();
|
|
16433
|
-
return {
|
|
16434
|
-
authorize: async ({ call, entry }) => {
|
|
16435
|
-
if (call.caller.kind !== "codemode") return;
|
|
16436
|
-
const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
|
|
16437
|
-
if (!binding) return;
|
|
16438
|
-
const invocation = binding.call(call.operationId, call.arguments);
|
|
16439
|
-
const policy = await prepareConnectorActionApproval2(input.db, identity, invocation);
|
|
16440
|
-
if (policy.managed && policy.decision === "ask") {
|
|
16441
|
-
throw new AttemptToolApprovalRequiredError2();
|
|
16442
|
-
}
|
|
16443
|
-
if (policy.managed && policy.decision === "block") {
|
|
16444
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
16445
|
-
"GitHub connector action is blocked by policy"
|
|
16446
|
-
);
|
|
16447
|
-
}
|
|
16448
|
-
const admission = await beginConnectorActionExecution2(input.db, identity, invocation);
|
|
16449
|
-
if (!admission.allowed) {
|
|
16450
|
-
if (admission.reason === "approval_required") {
|
|
16451
|
-
throw new AttemptToolApprovalRequiredError2();
|
|
16452
|
-
}
|
|
16453
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
16454
|
-
`GitHub connector action was not executed: ${admission.reason}`
|
|
16455
|
-
);
|
|
16456
|
-
}
|
|
16457
|
-
if (admission.managed) executing.set(call.operationId, admission.requestId);
|
|
16458
|
-
},
|
|
16459
|
-
complete: async ({ operationId, outcome }) => {
|
|
16460
|
-
const requestId = executing.get(operationId);
|
|
16461
|
-
if (!requestId) return;
|
|
16462
|
-
executing.delete(operationId);
|
|
16463
|
-
await completeConnectorActionExecution2(input.db, {
|
|
16464
|
-
accountId: input.accountId,
|
|
16465
|
-
workspaceId: input.workspaceId,
|
|
16466
|
-
requestId,
|
|
16467
|
-
attemptId: input.attemptId,
|
|
16468
|
-
outcome
|
|
16469
|
-
});
|
|
16470
|
-
}
|
|
15579
|
+
connectorBindings
|
|
16471
15580
|
};
|
|
16472
15581
|
}
|
|
16473
15582
|
function internalMcpConfig(id, name) {
|
|
@@ -16811,7 +15920,7 @@ function canWrite(permissions) {
|
|
|
16811
15920
|
}
|
|
16812
15921
|
|
|
16813
15922
|
// src/activities/connector-attachments.ts
|
|
16814
|
-
import { createHash as
|
|
15923
|
+
import { createHash as createHash11 } from "crypto";
|
|
16815
15924
|
import {
|
|
16816
15925
|
CONNECTOR_ATTACHMENT_RECEIPT_VERSION,
|
|
16817
15926
|
ConnectorAttachmentReceiptEnvelope
|
|
@@ -16831,7 +15940,7 @@ var ConnectorAttachmentMaterializationError = class extends Error {
|
|
|
16831
15940
|
}
|
|
16832
15941
|
};
|
|
16833
15942
|
function digestParts(...parts) {
|
|
16834
|
-
const hash =
|
|
15943
|
+
const hash = createHash11("sha256");
|
|
16835
15944
|
for (const part of parts) {
|
|
16836
15945
|
const bytes = Buffer.from(part, "utf8");
|
|
16837
15946
|
hash.update(String(bytes.byteLength));
|
|
@@ -16943,14 +16052,16 @@ import { allowedFirstPartyMcpToolsForSession } from "@opengeni/config";
|
|
|
16943
16052
|
// src/activities/codemode-dispatcher.ts
|
|
16944
16053
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
16945
16054
|
import {
|
|
16946
|
-
AttemptToolApprovalRequiredError
|
|
16055
|
+
AttemptToolApprovalRequiredError,
|
|
16947
16056
|
AttemptToolCatalogStaleError,
|
|
16057
|
+
AttemptToolInputValidationError,
|
|
16948
16058
|
AttemptToolNotFoundError,
|
|
16949
16059
|
codemodeDispatchSubject,
|
|
16950
16060
|
decodeCodemodeDispatchRequest,
|
|
16951
16061
|
encodeCodemodeDispatchAck
|
|
16952
16062
|
} from "@opengeni/codemode";
|
|
16953
16063
|
import {
|
|
16064
|
+
CODEMODE_CLAIM_LEASE_MS,
|
|
16954
16065
|
CODEMODE_CLAIM_HEARTBEAT_MS,
|
|
16955
16066
|
CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT
|
|
16956
16067
|
} from "@opengeni/contracts";
|
|
@@ -16958,6 +16069,8 @@ import {
|
|
|
16958
16069
|
cancelQueuedCodemodeOperationsForAttempt,
|
|
16959
16070
|
claimCodemodeOperation,
|
|
16960
16071
|
failCodemodeOperation,
|
|
16072
|
+
getSessionEventByClientEventId,
|
|
16073
|
+
getSessionToolCallCreatedEventByOperationId,
|
|
16961
16074
|
markCodemodeOperationExecutionStarted,
|
|
16962
16075
|
renewCodemodeOperationClaim,
|
|
16963
16076
|
settleCodemodeOperationWithOutput
|
|
@@ -16967,7 +16080,7 @@ import {
|
|
|
16967
16080
|
publishDurableSessionEvents as publishDurableSessionEvents8
|
|
16968
16081
|
} from "@opengeni/events";
|
|
16969
16082
|
var CodemodeAttemptDispatcher = class {
|
|
16970
|
-
constructor(db, bus, environment, scope, turnSignal, maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT) {
|
|
16083
|
+
constructor(db, bus, environment, scope, turnSignal, maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT, timings = {}) {
|
|
16971
16084
|
this.db = db;
|
|
16972
16085
|
this.bus = bus;
|
|
16973
16086
|
this.environment = environment;
|
|
@@ -16980,12 +16093,22 @@ var CodemodeAttemptDispatcher = class {
|
|
|
16980
16093
|
if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
|
|
16981
16094
|
throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
|
|
16982
16095
|
}
|
|
16096
|
+
this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
|
|
16097
|
+
this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
|
|
16098
|
+
if (!Number.isSafeInteger(this.claimLeaseMs) || this.claimLeaseMs < 1e3 || this.claimLeaseMs > 10 * 6e4) {
|
|
16099
|
+
throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
|
|
16100
|
+
}
|
|
16101
|
+
if (!Number.isSafeInteger(this.claimHeartbeatMs) || this.claimHeartbeatMs < 1 || this.claimHeartbeatMs >= this.claimLeaseMs) {
|
|
16102
|
+
throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
|
|
16103
|
+
}
|
|
16983
16104
|
}
|
|
16984
16105
|
stopController = new AbortController();
|
|
16985
16106
|
inFlight = /* @__PURE__ */ new Map();
|
|
16986
16107
|
pendingClaims = 0;
|
|
16987
16108
|
unsubscribe = null;
|
|
16988
16109
|
closing = false;
|
|
16110
|
+
claimLeaseMs;
|
|
16111
|
+
claimHeartbeatMs;
|
|
16989
16112
|
start() {
|
|
16990
16113
|
if (this.unsubscribe || this.closing)
|
|
16991
16114
|
throw new Error("Codemode dispatcher cannot be restarted");
|
|
@@ -17013,12 +16136,15 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17013
16136
|
...this.scope,
|
|
17014
16137
|
catalogDigest: request.catalogDigest,
|
|
17015
16138
|
operationId: request.operationId,
|
|
17016
|
-
claimId: randomUUID3()
|
|
16139
|
+
claimId: randomUUID3(),
|
|
16140
|
+
claimLeaseMs: this.claimLeaseMs
|
|
17017
16141
|
});
|
|
17018
16142
|
if (claim.status === "claimed") {
|
|
17019
|
-
const execution = this.execute(claim.operation, claim.claimId).finally(
|
|
17020
|
-
|
|
17021
|
-
|
|
16143
|
+
const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
|
|
16144
|
+
() => {
|
|
16145
|
+
this.inFlight.delete(request.operationId);
|
|
16146
|
+
}
|
|
16147
|
+
);
|
|
17022
16148
|
this.inFlight.set(request.operationId, execution);
|
|
17023
16149
|
return encodeCodemodeDispatchAck({
|
|
17024
16150
|
version: 1,
|
|
@@ -17026,6 +16152,18 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17026
16152
|
status: "accepted"
|
|
17027
16153
|
});
|
|
17028
16154
|
}
|
|
16155
|
+
if (claim.status === "execution_owner_lost") {
|
|
16156
|
+
await this.settleWithOutput(claim.operation, claim.claimId, {
|
|
16157
|
+
state: "outcome_unknown",
|
|
16158
|
+
errorCode: "worker_lost_during_execution",
|
|
16159
|
+
errorMessage: "The execution owner disappeared after the tool call began. Inspect actual state before retrying."
|
|
16160
|
+
});
|
|
16161
|
+
return encodeCodemodeDispatchAck({
|
|
16162
|
+
version: 1,
|
|
16163
|
+
operationId: request.operationId,
|
|
16164
|
+
status: "terminal"
|
|
16165
|
+
});
|
|
16166
|
+
}
|
|
17029
16167
|
return encodeCodemodeDispatchAck({
|
|
17030
16168
|
version: 1,
|
|
17031
16169
|
operationId: request.operationId,
|
|
@@ -17054,7 +16192,26 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17054
16192
|
});
|
|
17055
16193
|
await Promise.allSettled([...this.inFlight.values()]);
|
|
17056
16194
|
}
|
|
17057
|
-
async execute(operation, claimId) {
|
|
16195
|
+
async execute(operation, claimId, reclaimed) {
|
|
16196
|
+
const claimAuthority = {
|
|
16197
|
+
accountId: this.scope.accountId,
|
|
16198
|
+
workspaceId: this.scope.workspaceId,
|
|
16199
|
+
attemptId: this.scope.attemptId,
|
|
16200
|
+
operationId: operation.operationId,
|
|
16201
|
+
claimId,
|
|
16202
|
+
claimLeaseMs: this.claimLeaseMs
|
|
16203
|
+
};
|
|
16204
|
+
const heartbeat = setInterval(() => {
|
|
16205
|
+
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => void 0);
|
|
16206
|
+
}, this.claimHeartbeatMs);
|
|
16207
|
+
heartbeat.unref?.();
|
|
16208
|
+
try {
|
|
16209
|
+
await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
|
|
16210
|
+
} finally {
|
|
16211
|
+
clearInterval(heartbeat);
|
|
16212
|
+
}
|
|
16213
|
+
}
|
|
16214
|
+
async executeClaimed(operation, claimId, reclaimed, claimAuthority) {
|
|
17058
16215
|
const entry = this.environment.catalog.entries.find(
|
|
17059
16216
|
(candidate) => candidate.identity.serverId === operation.identity.serverId && candidate.identity.toolName === operation.identity.toolName
|
|
17060
16217
|
);
|
|
@@ -17077,38 +16234,7 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17077
16234
|
);
|
|
17078
16235
|
return;
|
|
17079
16236
|
}
|
|
17080
|
-
|
|
17081
|
-
this.db,
|
|
17082
|
-
this.bus,
|
|
17083
|
-
this.scope.workspaceId,
|
|
17084
|
-
this.scope.sessionId,
|
|
17085
|
-
this.scope.turnId,
|
|
17086
|
-
this.scope.executionGeneration,
|
|
17087
|
-
this.scope.attemptId,
|
|
17088
|
-
[
|
|
17089
|
-
{
|
|
17090
|
-
type: "agent.toolCall.created",
|
|
17091
|
-
turnId: this.scope.turnId,
|
|
17092
|
-
turnGeneration: this.scope.executionGeneration,
|
|
17093
|
-
turnAttemptId: this.scope.attemptId,
|
|
17094
|
-
producerId: operation.caller.subjectId,
|
|
17095
|
-
payload: {
|
|
17096
|
-
id: operation.operationId,
|
|
17097
|
-
name: entry.modelName,
|
|
17098
|
-
arguments: operation.arguments,
|
|
17099
|
-
origin: "codemode",
|
|
17100
|
-
subjectId: operation.caller.subjectId,
|
|
17101
|
-
raw: {
|
|
17102
|
-
type: "codemode_call",
|
|
17103
|
-
serverId: operation.identity.serverId,
|
|
17104
|
-
toolName: operation.identity.toolName,
|
|
17105
|
-
catalogDigest: operation.catalogDigest
|
|
17106
|
-
}
|
|
17107
|
-
}
|
|
17108
|
-
}
|
|
17109
|
-
]
|
|
17110
|
-
).catch(() => ({ accepted: false }));
|
|
17111
|
-
if (!created.accepted) {
|
|
16237
|
+
if (!await this.ensureToolCallCreated(operation, entry, reclaimed)) {
|
|
17112
16238
|
await this.failBeforeExecution(
|
|
17113
16239
|
operation.operationId,
|
|
17114
16240
|
claimId,
|
|
@@ -17125,24 +16251,9 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17125
16251
|
});
|
|
17126
16252
|
return;
|
|
17127
16253
|
}
|
|
17128
|
-
|
|
17129
|
-
accountId: this.scope.accountId,
|
|
17130
|
-
workspaceId: this.scope.workspaceId,
|
|
17131
|
-
attemptId: this.scope.attemptId,
|
|
17132
|
-
operationId: operation.operationId,
|
|
17133
|
-
claimId
|
|
17134
|
-
};
|
|
17135
|
-
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
17136
|
-
this.db,
|
|
17137
|
-
claimAuthority
|
|
17138
|
-
);
|
|
17139
|
-
if (!crossedExecutionBoundary) return;
|
|
17140
|
-
const heartbeat = setInterval(() => {
|
|
17141
|
-
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => void 0);
|
|
17142
|
-
}, CODEMODE_CLAIM_HEARTBEAT_MS);
|
|
17143
|
-
heartbeat.unref?.();
|
|
16254
|
+
let preparedCall;
|
|
17144
16255
|
try {
|
|
17145
|
-
|
|
16256
|
+
preparedCall = await this.environment.prepareCall(
|
|
17146
16257
|
{
|
|
17147
16258
|
operationId: operation.operationId,
|
|
17148
16259
|
catalogDigest: operation.catalogDigest,
|
|
@@ -17152,16 +16263,111 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17152
16263
|
},
|
|
17153
16264
|
{ signal }
|
|
17154
16265
|
);
|
|
17155
|
-
await this.settleWithOutput(operation, claimId, { state: "completed", result });
|
|
17156
16266
|
} catch (error) {
|
|
17157
|
-
|
|
16267
|
+
await this.settleWithOutput(operation, claimId, {
|
|
16268
|
+
state: "failed",
|
|
16269
|
+
errorCode: errorCode(error, signal, false),
|
|
16270
|
+
errorMessage: safeErrorMessage(error)
|
|
16271
|
+
});
|
|
16272
|
+
return;
|
|
16273
|
+
}
|
|
16274
|
+
if (signal.aborted) {
|
|
16275
|
+
await this.settleWithOutput(operation, claimId, {
|
|
16276
|
+
state: "failed",
|
|
16277
|
+
errorCode: "attempt_cancelled",
|
|
16278
|
+
errorMessage: "Execution attempt ended before the Codemode call started"
|
|
16279
|
+
});
|
|
16280
|
+
return;
|
|
16281
|
+
}
|
|
16282
|
+
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
16283
|
+
this.db,
|
|
16284
|
+
claimAuthority
|
|
16285
|
+
);
|
|
16286
|
+
if (!crossedExecutionBoundary) return;
|
|
16287
|
+
try {
|
|
16288
|
+
const result = await preparedCall.execute();
|
|
16289
|
+
await this.settleWithOutput(operation, claimId, {
|
|
16290
|
+
state: "completed",
|
|
16291
|
+
result
|
|
16292
|
+
});
|
|
16293
|
+
} catch (error) {
|
|
16294
|
+
const knownPreExecution = error instanceof AttemptToolApprovalRequiredError || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolNotFoundError || connectorActionWasNotExecuted(error);
|
|
17158
16295
|
await this.settleWithOutput(operation, claimId, {
|
|
17159
16296
|
state: knownPreExecution ? "failed" : "outcome_unknown",
|
|
17160
|
-
errorCode: errorCode(error, signal),
|
|
16297
|
+
errorCode: errorCode(error, signal, true),
|
|
17161
16298
|
errorMessage: !knownPreExecution ? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying." : safeErrorMessage(error)
|
|
17162
16299
|
});
|
|
17163
|
-
}
|
|
17164
|
-
|
|
16300
|
+
}
|
|
16301
|
+
}
|
|
16302
|
+
async ensureToolCallCreated(operation, entry, reclaimed) {
|
|
16303
|
+
const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
|
|
16304
|
+
if (reclaimed) {
|
|
16305
|
+
const existing = await getSessionEventByClientEventId(
|
|
16306
|
+
this.db,
|
|
16307
|
+
this.scope.workspaceId,
|
|
16308
|
+
this.scope.sessionId,
|
|
16309
|
+
clientEventId
|
|
16310
|
+
).catch(() => null);
|
|
16311
|
+
if (existing) {
|
|
16312
|
+
assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
|
|
16313
|
+
return true;
|
|
16314
|
+
}
|
|
16315
|
+
const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
|
|
16316
|
+
workspaceId: this.scope.workspaceId,
|
|
16317
|
+
sessionId: this.scope.sessionId,
|
|
16318
|
+
turnId: this.scope.turnId,
|
|
16319
|
+
attemptId: this.scope.attemptId,
|
|
16320
|
+
operationId: operation.operationId
|
|
16321
|
+
}).catch(() => null);
|
|
16322
|
+
if (legacyCreated) {
|
|
16323
|
+
assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
|
|
16324
|
+
return true;
|
|
16325
|
+
}
|
|
16326
|
+
}
|
|
16327
|
+
try {
|
|
16328
|
+
const created = await appendAndPublishTurnEventsFenced3(
|
|
16329
|
+
this.db,
|
|
16330
|
+
this.bus,
|
|
16331
|
+
this.scope.workspaceId,
|
|
16332
|
+
this.scope.sessionId,
|
|
16333
|
+
this.scope.turnId,
|
|
16334
|
+
this.scope.executionGeneration,
|
|
16335
|
+
this.scope.attemptId,
|
|
16336
|
+
[
|
|
16337
|
+
{
|
|
16338
|
+
type: "agent.toolCall.created",
|
|
16339
|
+
clientEventId,
|
|
16340
|
+
turnId: this.scope.turnId,
|
|
16341
|
+
turnGeneration: this.scope.executionGeneration,
|
|
16342
|
+
turnAttemptId: this.scope.attemptId,
|
|
16343
|
+
producerId: operation.caller.subjectId,
|
|
16344
|
+
payload: {
|
|
16345
|
+
id: operation.operationId,
|
|
16346
|
+
name: entry.modelName,
|
|
16347
|
+
arguments: operation.arguments,
|
|
16348
|
+
origin: "codemode",
|
|
16349
|
+
subjectId: operation.caller.subjectId,
|
|
16350
|
+
raw: {
|
|
16351
|
+
type: "codemode_call",
|
|
16352
|
+
serverId: operation.identity.serverId,
|
|
16353
|
+
toolName: operation.identity.toolName,
|
|
16354
|
+
catalogDigest: operation.catalogDigest
|
|
16355
|
+
}
|
|
16356
|
+
}
|
|
16357
|
+
}
|
|
16358
|
+
]
|
|
16359
|
+
);
|
|
16360
|
+
return created.accepted;
|
|
16361
|
+
} catch {
|
|
16362
|
+
const raced = await getSessionEventByClientEventId(
|
|
16363
|
+
this.db,
|
|
16364
|
+
this.scope.workspaceId,
|
|
16365
|
+
this.scope.sessionId,
|
|
16366
|
+
clientEventId
|
|
16367
|
+
).catch(() => null);
|
|
16368
|
+
if (!raced) return false;
|
|
16369
|
+
assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
|
|
16370
|
+
return true;
|
|
17165
16371
|
}
|
|
17166
16372
|
}
|
|
17167
16373
|
async settleWithOutput(operation, claimId, settlement) {
|
|
@@ -17193,6 +16399,19 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17193
16399
|
});
|
|
17194
16400
|
}
|
|
17195
16401
|
};
|
|
16402
|
+
function codemodeToolCallCreatedClientEventId(operationId) {
|
|
16403
|
+
return `opengeni:codemode-tool-call-created:${operationId}`;
|
|
16404
|
+
}
|
|
16405
|
+
function assertMatchingCodemodeToolCallCreated(event, operation, entry, scope) {
|
|
16406
|
+
const payload = recordValue(event.payload);
|
|
16407
|
+
const raw = recordValue(payload?.raw);
|
|
16408
|
+
if (event.type !== "agent.toolCall.created" || event.turnId !== scope.turnId || event.turnGeneration !== scope.executionGeneration || event.turnAttemptId !== scope.attemptId || event.turnAssociation !== "current" || payload?.id !== operation.operationId || payload?.name !== entry.modelName || payload?.origin !== "codemode" || payload?.subjectId !== operation.caller.subjectId || raw?.type !== "codemode_call" || raw?.serverId !== operation.identity.serverId || raw?.toolName !== operation.identity.toolName || raw?.catalogDigest !== operation.catalogDigest) {
|
|
16409
|
+
throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
|
|
16410
|
+
}
|
|
16411
|
+
}
|
|
16412
|
+
function recordValue(value) {
|
|
16413
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
16414
|
+
}
|
|
17196
16415
|
function connectorActionWasNotExecuted(error) {
|
|
17197
16416
|
return Boolean(
|
|
17198
16417
|
error && typeof error === "object" && "connectorActionOutcome" in error && error.connectorActionOutcome === "not_executed"
|
|
@@ -17201,8 +16420,10 @@ function connectorActionWasNotExecuted(error) {
|
|
|
17201
16420
|
function combinedSignal(primary, secondary) {
|
|
17202
16421
|
return secondary ? AbortSignal.any([primary, secondary]) : primary;
|
|
17203
16422
|
}
|
|
17204
|
-
function errorCode(error, signal) {
|
|
17205
|
-
if (signal.aborted)
|
|
16423
|
+
function errorCode(error, signal, executionStarted) {
|
|
16424
|
+
if (signal.aborted) {
|
|
16425
|
+
return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
|
|
16426
|
+
}
|
|
17206
16427
|
if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
|
|
17207
16428
|
if (typeof error === "object" && error !== null && "code" in error) {
|
|
17208
16429
|
const code = error.code;
|
|
@@ -17211,7 +16432,7 @@ function errorCode(error, signal) {
|
|
|
17211
16432
|
return "tool_outcome_unknown";
|
|
17212
16433
|
}
|
|
17213
16434
|
function safeErrorMessage(error) {
|
|
17214
|
-
if (error instanceof
|
|
16435
|
+
if (error instanceof AttemptToolApprovalRequiredError || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolInputValidationError || error instanceof AttemptToolNotFoundError) {
|
|
17215
16436
|
return error.message;
|
|
17216
16437
|
}
|
|
17217
16438
|
return "Codemode call failed before tool execution";
|
|
@@ -17223,6 +16444,7 @@ import {
|
|
|
17223
16444
|
defaultSessionMcpServerIds,
|
|
17224
16445
|
loadRigDefaultVariableSetEnvironment,
|
|
17225
16446
|
mergeRigDefaultVariableSetEnvironment,
|
|
16447
|
+
buildApiIntegrationMcpServers,
|
|
17226
16448
|
resolveCatalogSettings as resolveCatalogSettings2,
|
|
17227
16449
|
resolveWorkspaceModelSelection,
|
|
17228
16450
|
withFrozenPersonalConnectionDelegations,
|
|
@@ -17555,7 +16777,7 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17555
16777
|
);
|
|
17556
16778
|
};
|
|
17557
16779
|
const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
|
|
17558
|
-
const apiIntegrationMcpServers =
|
|
16780
|
+
const apiIntegrationMcpServers = buildApiIntegrationMcpServers({
|
|
17559
16781
|
settings: runSettings,
|
|
17560
16782
|
integrations: installedApiIntegrations.filter(
|
|
17561
16783
|
(integration) => selectedApiIntegrationServerIds.has(integration.serverId)
|
|
@@ -17633,6 +16855,22 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17633
16855
|
)
|
|
17634
16856
|
}
|
|
17635
16857
|
] : [];
|
|
16858
|
+
const attemptConnectorActionBindings = [
|
|
16859
|
+
...googleDriveConnectorBindings,
|
|
16860
|
+
...githubRestMcp.connectorBindings
|
|
16861
|
+
];
|
|
16862
|
+
const connectorActionPolicy = {
|
|
16863
|
+
preview: async (call) => await previewConnectorActionApproval(db, connectorActionIdentity, call),
|
|
16864
|
+
prepare: async (call) => await prepareConnectorActionApproval(db, connectorActionIdentity, call),
|
|
16865
|
+
begin: async (call) => await beginConnectorActionExecution(db, connectorActionIdentity, call),
|
|
16866
|
+
complete: async ({ requestId, outcome }) => await completeConnectorActionExecution(db, {
|
|
16867
|
+
accountId: input.accountId,
|
|
16868
|
+
workspaceId: input.workspaceId,
|
|
16869
|
+
requestId,
|
|
16870
|
+
attemptId: input.attemptId,
|
|
16871
|
+
outcome
|
|
16872
|
+
})
|
|
16873
|
+
};
|
|
17636
16874
|
const attemptToolDefinitions = [
|
|
17637
16875
|
createListModelsAttemptToolDefinition({
|
|
17638
16876
|
currentModelId: turnExecutionPolicy.productModelId,
|
|
@@ -17821,6 +17059,12 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17821
17059
|
resolveCredential,
|
|
17822
17060
|
onAuthNeeded: publishToolAuthNeeded,
|
|
17823
17061
|
materializeConnectorAttachments,
|
|
17062
|
+
refreshOwnedCommand: async (commandId) => {
|
|
17063
|
+
throwIfWorkerShuttingDown();
|
|
17064
|
+
throwIfTurnCancelled();
|
|
17065
|
+
const owned = sandboxState.lazyOwnedSandbox?.session ?? sandboxState.resolvedSandbox?.established.session ?? media.sdkOwnedSandboxSession;
|
|
17066
|
+
return await owned?.refreshOwnedCommand?.(commandId) ?? false;
|
|
17067
|
+
},
|
|
17824
17068
|
spillOversizedModelToolResult: async ({ operationId, result }) => await toolResultSpill.spill({ operationId, result }),
|
|
17825
17069
|
localMcpServers,
|
|
17826
17070
|
...deferNonEagerToolPreparation ? { deferNonEagerUntilToolDemand: true } : {},
|
|
@@ -17844,21 +17088,8 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17844
17088
|
nestedAgentDepth: session.nestedAgentDepth,
|
|
17845
17089
|
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
17846
17090
|
attemptToolDefinitions,
|
|
17847
|
-
|
|
17848
|
-
|
|
17849
|
-
const { call } = authorization;
|
|
17850
|
-
if (googleDrivePublicationTarget && googleDrivePublicationAllowed && call.caller.kind === "codemode" && call.identity.serverId === "google-drive-publishing" && call.identity.toolName === "google_drive_publish_file") {
|
|
17851
|
-
await authorizeGoogleDrivePublicationAttempt({
|
|
17852
|
-
db,
|
|
17853
|
-
identity: connectorActionIdentity,
|
|
17854
|
-
target: googleDrivePublicationTarget,
|
|
17855
|
-
approvalId: call.operationId,
|
|
17856
|
-
arguments: call.arguments
|
|
17857
|
-
});
|
|
17858
|
-
}
|
|
17859
|
-
await githubRestMcp.authorizeCodemodeCall?.(authorization);
|
|
17860
|
-
}
|
|
17861
|
-
} : {}
|
|
17091
|
+
connectorActionPolicy,
|
|
17092
|
+
attemptConnectorActionBindings
|
|
17862
17093
|
}),
|
|
17863
17094
|
cancellationSignal,
|
|
17864
17095
|
async (latePreparedTools) => await latePreparedTools.close().catch(() => void 0)
|
|
@@ -17917,11 +17148,8 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17917
17148
|
activatePreparedToolEnvironment(eventing.preparedTools);
|
|
17918
17149
|
}
|
|
17919
17150
|
return {
|
|
17920
|
-
attemptConnectorActionBindings
|
|
17921
|
-
|
|
17922
|
-
...githubRestMcp.connectorBindings
|
|
17923
|
-
],
|
|
17924
|
-
connectorActionIdentity,
|
|
17151
|
+
attemptConnectorActionBindings,
|
|
17152
|
+
connectorActionPolicy,
|
|
17925
17153
|
generateSessionTitleInParallel: titleToolPlan.generateTitleInParallel,
|
|
17926
17154
|
postToolPreparationStartedAt,
|
|
17927
17155
|
preparationIndependentToolNames: titleToolPlan.preparationIndependentToolNames
|
|
@@ -18114,19 +17342,16 @@ async function applyTurnGitHubRepositoryBindings(input) {
|
|
|
18114
17342
|
// src/activities/agent-turn/agent-build.ts
|
|
18115
17343
|
import {
|
|
18116
17344
|
selectXaiCredentialForUse,
|
|
18117
|
-
materializeXaiCredentialForRun
|
|
17345
|
+
materializeXaiCredentialForRun,
|
|
18118
17346
|
resolveXaiProviderAccountAuthoritySnapshotForAcceptance,
|
|
18119
17347
|
getXaiSessionAccountPin as getXaiSessionAccountPin2,
|
|
18120
17348
|
setXaiSessionAccountPin as setXaiSessionAccountPin2,
|
|
18121
17349
|
getWorkspaceVideoGenerationPolicy,
|
|
18122
|
-
loadWorkspaceVercelAiGatewayCredentialLease
|
|
18123
|
-
beginConnectorActionExecution as beginConnectorActionExecution3,
|
|
18124
|
-
completeConnectorActionExecution as completeConnectorActionExecution3,
|
|
18125
|
-
prepareConnectorActionApproval as prepareConnectorActionApproval3
|
|
17350
|
+
loadWorkspaceVercelAiGatewayCredentialLease
|
|
18126
17351
|
} from "@opengeni/db";
|
|
18127
17352
|
import {
|
|
18128
17353
|
serviceTierForLatencyMode as serviceTierForLatencyMode2,
|
|
18129
|
-
environmentsEncryptionKeyBytes as
|
|
17354
|
+
environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes3,
|
|
18130
17355
|
WORKSPACE_GATEWAY_PROVIDER_ID as WORKSPACE_GATEWAY_PROVIDER_ID2,
|
|
18131
17356
|
resolveModelProvider as resolveModelProvider2
|
|
18132
17357
|
} from "@opengeni/config";
|
|
@@ -18205,9 +17430,9 @@ import {
|
|
|
18205
17430
|
} from "@opengeni/db";
|
|
18206
17431
|
import {
|
|
18207
17432
|
calculateVideoGenerationCreditCostMicros,
|
|
18208
|
-
environmentsEncryptionKeyBytes as
|
|
17433
|
+
environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes2
|
|
18209
17434
|
} from "@opengeni/config";
|
|
18210
|
-
import { createHash as
|
|
17435
|
+
import { createHash as createHash13, randomUUID as randomUUID5 } from "crypto";
|
|
18211
17436
|
|
|
18212
17437
|
// src/activities/video-reference-staging.ts
|
|
18213
17438
|
import {
|
|
@@ -18216,7 +17441,7 @@ import {
|
|
|
18216
17441
|
|
|
18217
17442
|
// src/activities/video-media-validation.ts
|
|
18218
17443
|
import { retryWhileMissing as retryWhileMissing5 } from "@opengeni/storage";
|
|
18219
|
-
import { createHash as
|
|
17444
|
+
import { createHash as createHash12, randomUUID as randomUUID4 } from "crypto";
|
|
18220
17445
|
import { open, mkdir, rm, stat } from "fs/promises";
|
|
18221
17446
|
import { join } from "path";
|
|
18222
17447
|
import { spawn } from "child_process";
|
|
@@ -18244,7 +17469,7 @@ async function copyVersionedObjectToVerifiedTemp(input) {
|
|
|
18244
17469
|
const path = join(directory, "payload");
|
|
18245
17470
|
await mkdir(directory, { recursive: false, mode: 448 });
|
|
18246
17471
|
const handle = await open(path, "wx", 384);
|
|
18247
|
-
const hash =
|
|
17472
|
+
const hash = createHash12("sha256");
|
|
18248
17473
|
let offset = 0;
|
|
18249
17474
|
try {
|
|
18250
17475
|
while (offset < input.expectedSizeBytes) {
|
|
@@ -18764,7 +17989,7 @@ function videoCapabilitiesForTurn(input) {
|
|
|
18764
17989
|
}
|
|
18765
17990
|
function managedVideoGenerationCredentialLease(settings) {
|
|
18766
17991
|
if (!settings.vercelAiGatewayApiKey) return null;
|
|
18767
|
-
const encryptionKey =
|
|
17992
|
+
const encryptionKey = environmentsEncryptionKeyBytes2(settings);
|
|
18768
17993
|
if (!encryptionKey) {
|
|
18769
17994
|
throw new Error("Managed video generation requires OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
|
|
18770
17995
|
}
|
|
@@ -18779,7 +18004,7 @@ function managedVideoGenerationCredentialLease(settings) {
|
|
|
18779
18004
|
});
|
|
18780
18005
|
}
|
|
18781
18006
|
function xaiVideoGenerationCredentialLease(input) {
|
|
18782
|
-
const encryptionKey =
|
|
18007
|
+
const encryptionKey = environmentsEncryptionKeyBytes2(input.settings);
|
|
18783
18008
|
if (!encryptionKey) {
|
|
18784
18009
|
throw new Error("SuperGrok video generation requires OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
|
|
18785
18010
|
}
|
|
@@ -18839,7 +18064,7 @@ async function admitVideoGenerationRequest(input) {
|
|
|
18839
18064
|
operationId: candidateOperationId,
|
|
18840
18065
|
requestDigest
|
|
18841
18066
|
});
|
|
18842
|
-
const encryptionKey =
|
|
18067
|
+
const encryptionKey = environmentsEncryptionKeyBytes2(input.settings);
|
|
18843
18068
|
if (!encryptionKey) {
|
|
18844
18069
|
throw new Error("Video generation requires OPENGENI_ENVIRONMENTS_ENCRYPTION_KEY");
|
|
18845
18070
|
}
|
|
@@ -18863,7 +18088,7 @@ async function admitVideoGenerationRequest(input) {
|
|
|
18863
18088
|
toolCallId: input.toolCallId,
|
|
18864
18089
|
admissionKey,
|
|
18865
18090
|
requestDigest,
|
|
18866
|
-
promptDigest:
|
|
18091
|
+
promptDigest: createHash13("sha256").update(canonical.prompt).digest("hex"),
|
|
18867
18092
|
requestEncrypted: encryptEnvironmentValue2(encryptionKey, JSON.stringify(canonical)),
|
|
18868
18093
|
modelId: canonical.modelId,
|
|
18869
18094
|
sourceMode: canonical.sourceMode,
|
|
@@ -19059,18 +18284,27 @@ import {
|
|
|
19059
18284
|
var CODEX_IMAGE_MODEL = "gpt-image-2";
|
|
19060
18285
|
async function executeCodexImageGeneration(input) {
|
|
19061
18286
|
const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID2, input.credentialId);
|
|
18287
|
+
let providerDispatchAdmitted = false;
|
|
18288
|
+
const codexContext = {
|
|
18289
|
+
...input.codexContext,
|
|
18290
|
+
beforeProviderDispatch: async () => {
|
|
18291
|
+
await input.codexContext.beforeProviderDispatch?.();
|
|
18292
|
+
providerDispatchAdmitted = true;
|
|
18293
|
+
}
|
|
18294
|
+
};
|
|
19062
18295
|
return await executeImageGenerationOperation({
|
|
19063
18296
|
...input,
|
|
19064
18297
|
providerId: CODEX_PROVIDER_ID2,
|
|
19065
18298
|
providerBindingHash,
|
|
19066
18299
|
modelId: CODEX_IMAGE_MODEL,
|
|
19067
18300
|
...input.references ? { referenceDigests: input.references } : {},
|
|
18301
|
+
isProviderDispatchRejected: (error) => !providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
|
|
19068
18302
|
generate: async () => {
|
|
19069
18303
|
const generated = await generateCodexSubscriptionImage({
|
|
19070
18304
|
prompt: input.prompt,
|
|
19071
18305
|
...input.references ? { references: input.references } : {},
|
|
19072
18306
|
turnId: input.turnId,
|
|
19073
|
-
context:
|
|
18307
|
+
context: codexContext,
|
|
19074
18308
|
...input.abortSignal ? { abortSignal: input.abortSignal } : {}
|
|
19075
18309
|
});
|
|
19076
18310
|
return {
|
|
@@ -19326,7 +18560,8 @@ async function buildTurnAgent(deps) {
|
|
|
19326
18560
|
sandboxCodemodeToken,
|
|
19327
18561
|
fileResourceDownloads,
|
|
19328
18562
|
attemptConnectorActionBindings,
|
|
19329
|
-
|
|
18563
|
+
connectorActionPolicy,
|
|
18564
|
+
trigger,
|
|
19330
18565
|
preparationIndependentToolNames,
|
|
19331
18566
|
videoGenerationAcceptancesByCallId,
|
|
19332
18567
|
activeSandboxBackend,
|
|
@@ -19514,7 +18749,7 @@ async function buildTurnAgent(deps) {
|
|
|
19514
18749
|
};
|
|
19515
18750
|
}
|
|
19516
18751
|
} else if (videoGenerationPolicy.fundingSource === "supergrok_subscription") {
|
|
19517
|
-
const encryptionKey =
|
|
18752
|
+
const encryptionKey = environmentsEncryptionKeyBytes3(eventing.modelRunSettings);
|
|
19518
18753
|
const subjectId = leases.xai.subjectId ?? turn.initiatingHumanSubjectId;
|
|
19519
18754
|
if (encryptionKey && subjectId) {
|
|
19520
18755
|
const authoritySnapshot = providerTurn.xaiAuthoritySnapshot ?? await resolveXaiProviderAccountAuthoritySnapshotForAcceptance(db, {
|
|
@@ -19555,7 +18790,7 @@ async function buildTurnAgent(deps) {
|
|
|
19555
18790
|
throw error;
|
|
19556
18791
|
});
|
|
19557
18792
|
}
|
|
19558
|
-
const credential = await
|
|
18793
|
+
const credential = await materializeXaiCredentialForRun(db, {
|
|
19559
18794
|
workspaceId: input.workspaceId,
|
|
19560
18795
|
subjectId,
|
|
19561
18796
|
credentialId: selected.credentialId,
|
|
@@ -19645,17 +18880,7 @@ async function buildTurnAgent(deps) {
|
|
|
19645
18880
|
turnExecutionPolicy.providerId,
|
|
19646
18881
|
turnExecutionPolicy.latencyMode
|
|
19647
18882
|
);
|
|
19648
|
-
const
|
|
19649
|
-
prepare: async (call) => await prepareConnectorActionApproval3(db, connectorActionIdentity, call),
|
|
19650
|
-
begin: async (call) => await beginConnectorActionExecution3(db, connectorActionIdentity, call),
|
|
19651
|
-
complete: async ({ requestId, outcome }) => await completeConnectorActionExecution3(db, {
|
|
19652
|
-
accountId: input.accountId,
|
|
19653
|
-
workspaceId: input.workspaceId,
|
|
19654
|
-
requestId,
|
|
19655
|
-
attemptId: input.attemptId,
|
|
19656
|
-
outcome
|
|
19657
|
-
})
|
|
19658
|
-
};
|
|
18883
|
+
const approvedToolCallId = approvedConnectorActionCallId(trigger);
|
|
19659
18884
|
const runtimeSkillActivations = [
|
|
19660
18885
|
...installedSkillRuntime.activations,
|
|
19661
18886
|
...packRuntime.skillActivations,
|
|
@@ -19695,6 +18920,7 @@ async function buildTurnAgent(deps) {
|
|
|
19695
18920
|
let agentConstructionOutcome = "completed";
|
|
19696
18921
|
try {
|
|
19697
18922
|
return runtime.buildAgent(eventing.modelRunSettings, runtimeResources, {
|
|
18923
|
+
...preparedTools.inputWaitYield ? { inputWaitYield: preparedTools.inputWaitYield } : {},
|
|
19698
18924
|
reasoningEffort: turn.reasoningEffort,
|
|
19699
18925
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
19700
18926
|
...serviceTier ? { serviceTier } : {},
|
|
@@ -19732,6 +18958,7 @@ async function buildTurnAgent(deps) {
|
|
|
19732
18958
|
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
19733
18959
|
connectorActionPolicy,
|
|
19734
18960
|
attemptConnectorActionBindings,
|
|
18961
|
+
...approvedToolCallId ? { approvedToolCallId } : {},
|
|
19735
18962
|
// LIVE by-reference connector namespaces (fills during this turn's
|
|
19736
18963
|
// codex_apps tools/list): the codex tool_search description reads it per
|
|
19737
18964
|
// model call so the model sees the account's real connected sources.
|
|
@@ -19838,6 +19065,13 @@ async function buildTurnAgent(deps) {
|
|
|
19838
19065
|
postAgentPreparationStartedAt
|
|
19839
19066
|
};
|
|
19840
19067
|
}
|
|
19068
|
+
function approvedConnectorActionCallId(trigger) {
|
|
19069
|
+
if (trigger.type !== "user.approvalDecision") {
|
|
19070
|
+
return null;
|
|
19071
|
+
}
|
|
19072
|
+
const payload = trigger.payload;
|
|
19073
|
+
return payload.decision === "approve" && typeof payload.approvalId === "string" ? payload.approvalId : null;
|
|
19074
|
+
}
|
|
19841
19075
|
|
|
19842
19076
|
// src/activities/agent-turn/run.ts
|
|
19843
19077
|
function sessionTitleCodexRequestContext(context, nextRequestId) {
|
|
@@ -19848,6 +19082,7 @@ function sessionTitleCodexRequestContext(context, nextRequestId) {
|
|
|
19848
19082
|
refresh: context.refresh,
|
|
19849
19083
|
resolveModel: context.resolveModel,
|
|
19850
19084
|
...context.onUsageHeaders ? { onUsageHeaders: context.onUsageHeaders } : {},
|
|
19085
|
+
...context.beforeProviderDispatch ? { beforeProviderDispatch: context.beforeProviderDispatch } : {},
|
|
19851
19086
|
...context.responseTimeoutPolicy ? { responseTimeoutPolicy: context.responseTimeoutPolicy } : {},
|
|
19852
19087
|
nextRequestId,
|
|
19853
19088
|
turnMetadata: { request_kind: "session_title" }
|
|
@@ -20177,6 +19412,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
20177
19412
|
input.workspaceId,
|
|
20178
19413
|
providerTurn.effectiveCodexCredentialId ?? ""
|
|
20179
19414
|
);
|
|
19415
|
+
const resolveTrackedToken = async (resolve) => {
|
|
19416
|
+
const token = await resolve();
|
|
19417
|
+
providerTurn.effectiveCodexCredentialVersion = token.credentialVersion;
|
|
19418
|
+
return token;
|
|
19419
|
+
};
|
|
20180
19420
|
return {
|
|
20181
19421
|
clientVersion: CODEX_CLIENT_VERSION2,
|
|
20182
19422
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -20186,8 +19426,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
20186
19426
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
20187
19427
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
20188
19428
|
sessionId: input.sessionId,
|
|
20189
|
-
getToken: () => resolver.getToken
|
|
20190
|
-
refresh: () => resolver.refresh
|
|
19429
|
+
getToken: () => resolveTrackedToken(resolver.getToken),
|
|
19430
|
+
refresh: () => resolveTrackedToken(resolver.refresh),
|
|
20191
19431
|
resolveModel: buildModelResolver(
|
|
20192
19432
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
20193
19433
|
CODEX_FALLBACK_MODEL_SLUGS[0]
|
|
@@ -20196,6 +19436,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
20196
19436
|
providerTurn.latestCodexUsage = snapshot;
|
|
20197
19437
|
},
|
|
20198
19438
|
// latest wins; flushed once in finally
|
|
19439
|
+
beforeProviderDispatch: () => {
|
|
19440
|
+
leases.codex.assertUsable();
|
|
19441
|
+
},
|
|
20199
19442
|
onRequestPreparationDiagnostic: (phase) => {
|
|
20200
19443
|
if (eventing.firstModelRequestCheckpointAt === null || firstModelRequestCheckpoints.has(phase)) {
|
|
20201
19444
|
return;
|
|
@@ -20748,7 +19991,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
20748
19991
|
});
|
|
20749
19992
|
const {
|
|
20750
19993
|
attemptConnectorActionBindings,
|
|
20751
|
-
|
|
19994
|
+
connectorActionPolicy,
|
|
20752
19995
|
generateSessionTitleInParallel,
|
|
20753
19996
|
postToolPreparationStartedAt,
|
|
20754
19997
|
preparationIndependentToolNames
|
|
@@ -20802,7 +20045,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
20802
20045
|
sandboxCodemodeToken,
|
|
20803
20046
|
fileResourceDownloads,
|
|
20804
20047
|
attemptConnectorActionBindings,
|
|
20805
|
-
|
|
20048
|
+
connectorActionPolicy,
|
|
20049
|
+
trigger,
|
|
20806
20050
|
preparationIndependentToolNames,
|
|
20807
20051
|
videoGenerationAcceptancesByCallId,
|
|
20808
20052
|
activeSandboxBackend,
|
|
@@ -21111,6 +20355,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
21111
20355
|
videoGenerationAcceptancesByCallId
|
|
21112
20356
|
});
|
|
21113
20357
|
} catch (error) {
|
|
20358
|
+
eventing.preparedTools?.inputWaitYield?.closeAdmission();
|
|
21114
20359
|
return await settleTurnFailure({
|
|
21115
20360
|
error,
|
|
21116
20361
|
input,
|
|
@@ -21119,7 +20364,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
21119
20364
|
bus,
|
|
21120
20365
|
observability,
|
|
21121
20366
|
wakeSessionWorkflow,
|
|
21122
|
-
signalCodexCapacityWorkflow,
|
|
21123
20367
|
cancellationSignal,
|
|
21124
20368
|
sandboxRotationController,
|
|
21125
20369
|
noteCancellationRequested,
|
|
@@ -21173,7 +20417,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
21173
20417
|
}
|
|
21174
20418
|
|
|
21175
20419
|
// src/activities/video-generation-reconciliation.ts
|
|
21176
|
-
import { environmentsEncryptionKeyBytes as
|
|
20420
|
+
import { environmentsEncryptionKeyBytes as environmentsEncryptionKeyBytes4 } from "@opengeni/config";
|
|
21177
20421
|
import { videoGenerationRequestDigest as videoGenerationRequestDigest2 } from "@opengeni/core";
|
|
21178
20422
|
import {
|
|
21179
20423
|
addSessionSystemUpdateWithSourceMutation,
|
|
@@ -21185,8 +20429,8 @@ import {
|
|
|
21185
20429
|
markVideoGenerationRetaining,
|
|
21186
20430
|
markVideoGenerationSubmissionIntent,
|
|
21187
20431
|
markVideoGenerationSubmissionUncertain,
|
|
21188
|
-
materializeXaiCredentialForRun as
|
|
21189
|
-
refreshXaiSubscriptionCredentialSerialized
|
|
20432
|
+
materializeXaiCredentialForRun as materializeXaiCredentialForRun2,
|
|
20433
|
+
refreshXaiSubscriptionCredentialSerialized,
|
|
21190
20434
|
rotateVideoGenerationCredential,
|
|
21191
20435
|
markVideoGenerationTerminalUpdate,
|
|
21192
20436
|
markVideoGenerationTerminalUpdateInTransaction,
|
|
@@ -21399,7 +20643,7 @@ function publicProviderReason(error, cancelled) {
|
|
|
21399
20643
|
// src/activities/video-output-retention.ts
|
|
21400
20644
|
import { GENERATED_VIDEO_MAX_BYTES as GENERATED_VIDEO_MAX_BYTES2 } from "@opengeni/contracts";
|
|
21401
20645
|
import { pinnedFetch } from "@opengeni/network";
|
|
21402
|
-
import { createHash as
|
|
20646
|
+
import { createHash as createHash14, randomUUID as randomUUID7 } from "crypto";
|
|
21403
20647
|
import { createReadStream } from "fs";
|
|
21404
20648
|
import { mkdir as mkdir2, open as open2, rm as rm2 } from "fs/promises";
|
|
21405
20649
|
import { join as join2 } from "path";
|
|
@@ -21437,7 +20681,7 @@ async function downloadGeneratedVideoToVerifiedTemp(input) {
|
|
|
21437
20681
|
await mkdir2(directory, { recursive: false, mode: 448 });
|
|
21438
20682
|
const handle = await open2(path, "wx", 384);
|
|
21439
20683
|
const reader = response.body.getReader();
|
|
21440
|
-
const digest2 =
|
|
20684
|
+
const digest2 = createHash14("sha256");
|
|
21441
20685
|
let sizeBytes = 0;
|
|
21442
20686
|
try {
|
|
21443
20687
|
for (; ; ) {
|
|
@@ -21606,9 +20850,9 @@ function throwIfAborted3(signal) {
|
|
|
21606
20850
|
|
|
21607
20851
|
// src/activities/video-generation-reconciliation.ts
|
|
21608
20852
|
import {
|
|
21609
|
-
refreshXaiToken
|
|
20853
|
+
refreshXaiToken,
|
|
21610
20854
|
XaiSubscriptionError,
|
|
21611
|
-
xaiAccessTokenExpiry
|
|
20855
|
+
xaiAccessTokenExpiry
|
|
21612
20856
|
} from "@opengeni/xai-subscription";
|
|
21613
20857
|
|
|
21614
20858
|
// src/activities/xai-video-generation.ts
|
|
@@ -21677,7 +20921,7 @@ async function reconcileVideoGenerationOperation(service, input) {
|
|
|
21677
20921
|
await deliverTerminalResult(service, operation);
|
|
21678
20922
|
return { action: "terminal", status: operation.status };
|
|
21679
20923
|
}
|
|
21680
|
-
const key =
|
|
20924
|
+
const key = environmentsEncryptionKeyBytes4(service.settings);
|
|
21681
20925
|
if (!key) throw new Error("Video generation recovery encryption key is unavailable");
|
|
21682
20926
|
if (!operation.credentialEncrypted) {
|
|
21683
20927
|
throw new Error("Video provider credential lease was erased too early");
|
|
@@ -22122,7 +21366,7 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
|
|
|
22122
21366
|
let currentEncrypted = frozenEncrypted;
|
|
22123
21367
|
let connectedAccount = null;
|
|
22124
21368
|
try {
|
|
22125
|
-
connectedAccount = await
|
|
21369
|
+
connectedAccount = await materializeXaiCredentialForRun2(service.db, {
|
|
22126
21370
|
workspaceId: operation.workspaceId,
|
|
22127
21371
|
subjectId: current.subjectId,
|
|
22128
21372
|
credentialId: current.credentialId,
|
|
@@ -22140,7 +21384,7 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
|
|
|
22140
21384
|
let next = null;
|
|
22141
21385
|
if (connectedAccount) {
|
|
22142
21386
|
try {
|
|
22143
|
-
const result = await
|
|
21387
|
+
const result = await refreshXaiSubscriptionCredentialSerialized(service.db, {
|
|
22144
21388
|
accountId: operation.accountId,
|
|
22145
21389
|
workspaceId: operation.workspaceId,
|
|
22146
21390
|
subjectId: current.subjectId,
|
|
@@ -22154,14 +21398,14 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
|
|
|
22154
21398
|
if (!refreshToken) {
|
|
22155
21399
|
throw new Error("The connected SuperGrok account cannot be refreshed");
|
|
22156
21400
|
}
|
|
22157
|
-
const tokens = await
|
|
21401
|
+
const tokens = await refreshXaiToken(refreshToken);
|
|
22158
21402
|
return {
|
|
22159
21403
|
secret: {
|
|
22160
21404
|
version: 1,
|
|
22161
21405
|
accessToken: tokens.accessToken,
|
|
22162
21406
|
refreshToken: tokens.refreshToken
|
|
22163
21407
|
},
|
|
22164
|
-
expiresAt:
|
|
21408
|
+
expiresAt: xaiAccessTokenExpiry(tokens.accessToken) ?? new Date(Date.now() + tokens.expiresInSeconds * 1e3)
|
|
22165
21409
|
};
|
|
22166
21410
|
}
|
|
22167
21411
|
});
|
|
@@ -22179,7 +21423,7 @@ async function durableXaiVideoAuth(service, operation, key, credential) {
|
|
|
22179
21423
|
}
|
|
22180
21424
|
}
|
|
22181
21425
|
if (!next) {
|
|
22182
|
-
const tokens = await
|
|
21426
|
+
const tokens = await refreshXaiToken(current.refreshToken);
|
|
22183
21427
|
next = Object.freeze({
|
|
22184
21428
|
...current,
|
|
22185
21429
|
accessToken: tokens.accessToken,
|
|
@@ -22262,4 +21506,4 @@ export {
|
|
|
22262
21506
|
createTurnActivities,
|
|
22263
21507
|
createTurnActivitiesFromServices
|
|
22264
21508
|
};
|
|
22265
|
-
//# sourceMappingURL=activities-turn-
|
|
21509
|
+
//# sourceMappingURL=activities-turn-GHSFZB64.js.map
|