@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +3 -1
- package/dist/activities/agent-turn/admission.d.ts +24 -3
- package/dist/activities/agent-turn/agent-build.d.ts +4 -0
- package/dist/activities/agent-turn/claude-usage-observer.d.ts +58 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +13 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex.d.ts +3 -2
- package/dist/activities/agent-turn/compaction-prep.d.ts +21 -5
- package/dist/activities/agent-turn/credential-leases.d.ts +19 -33
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +82 -11
- package/dist/activities/agent-turn/failure-settlement.d.ts +6 -3
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +17 -0
- package/dist/activities/agent-turn/finalization.d.ts +4 -2
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +9 -2
- package/dist/activities/agent-turn/history-sink.d.ts +2 -0
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +41 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +6 -0
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/quiescence.d.ts +7 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +9 -0
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +8 -3
- package/dist/activities/agent-turn/sandbox-establish.d.ts +2 -1
- package/dist/activities/agent-turn/sandbox-provision.d.ts +3 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -20
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +2 -2
- package/dist/activities/agent-turn/session-title.d.ts +34 -2
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-remove.d.ts +14 -0
- package/dist/activities/agent-turn/skill-search.d.ts +3 -1
- package/dist/activities/agent-turn/skill-tools.d.ts +15 -2
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +7 -3
- package/dist/activities/agent-turn/subscription-lease.d.ts +38 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +56 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +43 -0
- package/dist/activities/agent-turn/turn-context.d.ts +29 -0
- package/dist/activities/agent-turn/turn-sandbox-access.d.ts +3 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -1
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +29 -0
- package/dist/activities/codemode-dispatcher.d.ts +9 -2
- package/dist/activities/codex-capacity.d.ts +18 -2
- package/dist/activities/codex-rotation.d.ts +7 -1
- package/dist/activities/context-compaction.d.ts +16 -2
- package/dist/activities/gmail-files.d.ts +32 -0
- package/dist/activities/goals.d.ts +8 -0
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +3 -0
- package/dist/activities/mcp-credentials.d.ts +12 -13
- package/dist/activities/mcp-operation-authority.d.ts +0 -5
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +13 -0
- package/dist/activities/parent-wake.d.ts +6 -12
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/rig-verification.d.ts +2 -3
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +15 -3
- package/dist/activities/sandbox-images.d.ts +1 -1
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-state.d.ts +70 -3
- package/dist/activities/types.d.ts +55 -6
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +46 -0
- package/dist/{activities-control-HMFGSKC5.js → activities-control-BNPTRTOE.js} +1896 -418
- package/dist/activities-control-BNPTRTOE.js.map +1 -0
- package/dist/activities-control.d.ts +182 -4
- package/dist/{activities-turn-AP67JADS.js → activities-turn-5LGZOSVU.js} +4476 -1327
- package/dist/activities-turn-5LGZOSVU.js.map +1 -0
- package/dist/activities.d.ts +182 -4
- package/dist/artifact-materializer-entry.js +5 -2
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +5 -2
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-A4XQWVZK.js → chunk-3ZTRWDMN.js} +307 -4
- package/dist/chunk-3ZTRWDMN.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-CYITU5UH.js → chunk-LJX7Z2KP.js} +1445 -343
- package/dist/chunk-LJX7Z2KP.js.map +1 -0
- package/dist/editable-artifact-materializer-service.d.ts +2 -2
- package/dist/editable-artifact-outbox-service.d.ts +2 -2
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +1 -1
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +95 -3
- package/dist/index.js +137 -43
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +118 -8
- package/dist/retained-process-diagnostics.d.ts +4 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-resume.d.ts +55 -13
- package/dist/sandbox-routing.d.ts +5 -3
- package/dist/sandbox-snapshot-diagnostics.d.ts +20 -0
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/warm-capture-holder.d.ts +3 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +273 -41
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +9 -0
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +21 -19
- package/src/activities/agent-run-admission.ts +17 -1
- package/src/activities/agent-turn/admission.ts +66 -3
- package/src/activities/agent-turn/agent-build.ts +84 -10
- package/src/activities/agent-turn/claim.ts +186 -43
- package/src/activities/agent-turn/claude-usage-observer.ts +221 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +172 -8
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex.ts +13 -5
- package/src/activities/agent-turn/compaction-prep.ts +170 -102
- package/src/activities/agent-turn/credential-leases.ts +122 -216
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +647 -48
- package/src/activities/agent-turn/failure-settlement.ts +682 -130
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +120 -0
- package/src/activities/agent-turn/finalization.ts +146 -43
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +69 -28
- package/src/activities/agent-turn/history-sink.ts +16 -2
- package/src/activities/agent-turn/history.ts +38 -11
- package/src/activities/agent-turn/input-wait-reply.ts +103 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/media-artifacts.ts +2 -1
- package/src/activities/agent-turn/model-call-admission.ts +94 -0
- package/src/activities/agent-turn/model-usage.ts +64 -3
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/quiescence.ts +10 -7
- package/src/activities/agent-turn/recovery-warning.ts +16 -0
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +116 -55
- package/src/activities/agent-turn/run.ts +320 -93
- package/src/activities/agent-turn/sandbox-establish.ts +84 -8
- package/src/activities/agent-turn/sandbox-provision.ts +3 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -21
- package/src/activities/agent-turn/sandbox-runtime.ts +343 -18
- package/src/activities/agent-turn/session-title.ts +81 -5
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-remove.ts +46 -0
- package/src/activities/agent-turn/skill-save.ts +1 -1
- package/src/activities/agent-turn/skill-search.ts +10 -1
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +112 -18
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +313 -77
- package/src/activities/agent-turn/subscription-lease.ts +115 -0
- package/src/activities/agent-turn/tool-environment.ts +316 -167
- package/src/activities/agent-turn/tool-policy.ts +80 -1
- package/src/activities/agent-turn/turn-context.ts +35 -0
- package/src/activities/agent-turn/turn-sandbox-access.ts +7 -1
- package/src/activities/agent-turn/xai-capacity.ts +94 -51
- package/src/activities/automations.ts +76 -4
- package/src/activities/browser-deadline-checkpoint.ts +279 -0
- package/src/activities/capabilities.ts +108 -11
- package/src/activities/codemode-dispatcher.ts +36 -8
- package/src/activities/codex-capacity.ts +60 -25
- package/src/activities/codex-rotation.ts +39 -17
- package/src/activities/context-compaction.ts +75 -18
- package/src/activities/gmail-files.ts +88 -0
- package/src/activities/goals.ts +116 -121
- package/src/activities/google-drive-publication.ts +1 -3
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/knowledge-indexing.ts +269 -15
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +17 -0
- package/src/activities/mcp-credentials.ts +83 -194
- package/src/activities/mcp-operation-authority.ts +16 -41
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +15 -2
- package/src/activities/parent-wake.ts +46 -21
- package/src/activities/retained-screenshots.ts +82 -11
- package/src/activities/rig-verification.ts +38 -37
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +59 -17
- package/src/activities/run-input.ts +25 -16
- package/src/activities/sandbox-images.ts +0 -1
- package/src/activities/sandbox-lease.ts +798 -82
- package/src/activities/scheduled-tasks.ts +550 -178
- package/src/activities/session-state.ts +245 -6
- package/src/activities/site-auth-maintenance.ts +3 -0
- package/src/activities/types.ts +72 -5
- package/src/activities/workflow-wake.ts +2 -17
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +283 -0
- package/src/activities-control.ts +4 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +13 -0
- package/src/activity-services.ts +16 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +7 -3
- package/src/editable-artifact-outbox-service.ts +4 -4
- package/src/github-rest-mcp.ts +1 -2
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +3 -0
- package/src/http.ts +4 -2
- package/src/index.ts +99 -37
- package/src/observability-metrics.ts +510 -5
- package/src/personal-github-git-credentials.ts +2 -1
- package/src/pr-review-credentials.ts +1 -3
- package/src/retained-process-diagnostics.ts +50 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-resume.ts +388 -101
- package/src/sandbox-routing.ts +150 -25
- package/src/sandbox-snapshot-diagnostics.ts +112 -0
- package/src/session-event-fanout.ts +23 -0
- package/src/warm-capture-holder.ts +15 -0
- package/src/worker-service-lifecycle.ts +53 -0
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +50 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session.ts +242 -22
- package/src/workflows.ts +4 -0
- package/dist/activities/agent-turn/knowledge-sources.d.ts +0 -20
- package/dist/activities/packs.d.ts +0 -55
- package/dist/activities-control-HMFGSKC5.js.map +0 -1
- package/dist/activities-turn-AP67JADS.js.map +0 -1
- package/dist/chunk-A4XQWVZK.js.map +0 -1
- package/dist/chunk-CYITU5UH.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/knowledge-sources.ts +0 -86
- package/src/activities/packs.ts +0 -166
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
} from "./chunk-
|
|
2
|
+
observabilityEventBusOptions
|
|
3
|
+
} from "./chunk-3ZTRWDMN.js";
|
|
4
4
|
|
|
5
5
|
// src/activity-services.ts
|
|
6
|
-
import {
|
|
7
|
-
|
|
6
|
+
import {
|
|
7
|
+
codeSearchDeploymentPolicy,
|
|
8
|
+
dbSearchPath,
|
|
9
|
+
getSettings,
|
|
10
|
+
resolveNatsControlPlaneAuth
|
|
11
|
+
} from "@opengeni/config";
|
|
12
|
+
import {
|
|
13
|
+
configureChildLifecycleNotices,
|
|
14
|
+
configureCodeSearchDeploymentPolicy,
|
|
15
|
+
createDb
|
|
16
|
+
} from "@opengeni/db";
|
|
8
17
|
import { createNatsEventBus } from "@opengeni/events";
|
|
9
18
|
import { createObservability } from "@opengeni/observability";
|
|
10
19
|
import { resolveCatalogSettings } from "@opengeni/core";
|
|
@@ -220,7 +229,7 @@ function exactPersonalGitHubDelegation(delegations, credentialBindingId) {
|
|
|
220
229
|
(delegation2) => delegation2.serverId === PERSONAL_GITHUB_SERVER_ID && delegation2.connectionType === "github_personal" && delegation2.personalGitHubRepositorySelection?.credentialBindingId === credentialBindingId
|
|
221
230
|
);
|
|
222
231
|
const delegation = matches[0];
|
|
223
|
-
if (matches.length !== 1 || !delegation || delegation.providerDomain !== "github.com" || delegation.kind !== "oauth2" || !delegation.originWorkspaceId || !delegation.
|
|
232
|
+
if (matches.length !== 1 || !delegation || delegation.providerDomain !== "github.com" || delegation.kind !== "oauth2" || !delegation.originWorkspaceId || !delegation.personalGitHubRepositorySelection) {
|
|
224
233
|
throw new Error("personal GitHub Git delegation is unavailable");
|
|
225
234
|
}
|
|
226
235
|
return delegation;
|
|
@@ -246,7 +255,6 @@ function exactRequestedRepositories(refs, credentialBindingId, snapshots) {
|
|
|
246
255
|
// src/pr-review-credentials.ts
|
|
247
256
|
import { environmentsEncryptionKeyBytes } from "@opengeni/config";
|
|
248
257
|
import { prReviewRegistrationIdFromCredentialBinding } from "@opengeni/core";
|
|
249
|
-
import { createNativeRemoteMcpCredentialsPort } from "@opengeni/core/remote-mcp-credentials";
|
|
250
258
|
import { decryptVariableSetValue, resolvePrReviewGitCredential } from "@opengeni/db";
|
|
251
259
|
import {
|
|
252
260
|
createGitHubAppInstallationTokenWithExpiry,
|
|
@@ -254,7 +262,6 @@ import {
|
|
|
254
262
|
} from "@opengeni/github";
|
|
255
263
|
function createStandaloneConnectionCredentialsPort(settings, db) {
|
|
256
264
|
return {
|
|
257
|
-
...createNativeRemoteMcpCredentialsPort(settings, db),
|
|
258
265
|
gitCredentials: async (request) => await resolveStandaloneGitCredentials(settings, db, request)
|
|
259
266
|
};
|
|
260
267
|
}
|
|
@@ -324,7 +331,7 @@ async function resolveStandaloneGitCredentials(settings, db, request) {
|
|
|
324
331
|
credentialEncrypted: authority.credentialEncrypted
|
|
325
332
|
});
|
|
326
333
|
if (authority.credentialKind === "managed_github_app" && settings.prReviewGithubAppId !== authority.appId) {
|
|
327
|
-
throw new Error("
|
|
334
|
+
throw new Error("Opengeni Lens App identity no longer matches this registration");
|
|
328
335
|
}
|
|
329
336
|
const minted = await createGitHubAppInstallationTokenWithSigningSettings(signingSettings, {
|
|
330
337
|
installationId: request.installationId,
|
|
@@ -405,6 +412,7 @@ function createSharedActivityServices(dependencies) {
|
|
|
405
412
|
const db = dependencies.db ?? dbClient.db;
|
|
406
413
|
const resolvedSettings = (await resolveCatalogSettings(db, settings)).settings;
|
|
407
414
|
configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
|
|
415
|
+
configureCodeSearchDeploymentPolicy(codeSearchDeploymentPolicy(settings));
|
|
408
416
|
return {
|
|
409
417
|
settings: resolvedSettings,
|
|
410
418
|
catalogSourceSettings: settings,
|
|
@@ -412,7 +420,7 @@ function createSharedActivityServices(dependencies) {
|
|
|
412
420
|
bus: dependencies.bus ?? await createNatsEventBus(
|
|
413
421
|
settings.natsUrl,
|
|
414
422
|
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
415
|
-
|
|
423
|
+
observabilityEventBusOptions(observability)
|
|
416
424
|
),
|
|
417
425
|
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
418
426
|
observability,
|
|
@@ -437,6 +445,9 @@ import {
|
|
|
437
445
|
fetchCodexUsageForAccount,
|
|
438
446
|
getCodexCapacityWaitForSession,
|
|
439
447
|
getXaiCapacityWaitForSession,
|
|
448
|
+
getClaudeCapacityWaitForSession,
|
|
449
|
+
resolveClaudeWaiterSubject,
|
|
450
|
+
reconcileClaudeCapacityWait as reconcileClaudeCapacityWaitDb,
|
|
440
451
|
resolveXaiWaiterSubject,
|
|
441
452
|
listCodexAccountStatuses,
|
|
442
453
|
listPendingCodexCapacityWakeTargets,
|
|
@@ -571,8 +582,13 @@ async function refreshExhaustedXaiQuota(input) {
|
|
|
571
582
|
}
|
|
572
583
|
|
|
573
584
|
// src/activities/codex-rotation.ts
|
|
574
|
-
|
|
575
|
-
|
|
585
|
+
import { subscriptionAccountShardIndex } from "@opengeni/config";
|
|
586
|
+
import { codexPlanExcludesModel, connectionModelAllowed } from "@opengeni/db";
|
|
587
|
+
function codexAccountServesModel(account, modelId, now) {
|
|
588
|
+
return connectionModelAllowed(account.allowedModelIds, modelId) && !codexPlanExcludesModel(account, modelId, now);
|
|
589
|
+
}
|
|
590
|
+
function codexAccountNeedsLiveCapacityRefresh(account, _now) {
|
|
591
|
+
return (account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || (account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || account.exhaustedKind === "quota" && account.exhaustedUntil !== null;
|
|
576
592
|
}
|
|
577
593
|
function effectiveRotationStrategy(_stored) {
|
|
578
594
|
return "sharded";
|
|
@@ -618,21 +634,13 @@ function isCodexCredentialHealthy(acct, now) {
|
|
|
618
634
|
function isCodexCredentialEligible(acct, now) {
|
|
619
635
|
return acct.allocatorEnabled && isCodexCredentialHealthy(acct, now);
|
|
620
636
|
}
|
|
621
|
-
function fnv1a32(input) {
|
|
622
|
-
let hash = 2166136261;
|
|
623
|
-
for (let i = 0; i < input.length; i++) {
|
|
624
|
-
hash ^= input.charCodeAt(i);
|
|
625
|
-
hash = Math.imul(hash, 16777619);
|
|
626
|
-
}
|
|
627
|
-
return hash >>> 0;
|
|
628
|
-
}
|
|
629
637
|
function shardCredentialForSession(args) {
|
|
630
638
|
const { sessionId, accounts, now } = args;
|
|
631
639
|
const eligibles = accounts.filter((acct) => isCodexCredentialEligible(acct, now));
|
|
632
640
|
if (eligibles.length === 0) {
|
|
633
641
|
return null;
|
|
634
642
|
}
|
|
635
|
-
const index =
|
|
643
|
+
const index = subscriptionAccountShardIndex(sessionId, eligibles.length);
|
|
636
644
|
return eligibles[index].id;
|
|
637
645
|
}
|
|
638
646
|
function chooseShardedHome(args) {
|
|
@@ -761,7 +769,13 @@ function chooseRotationActive(args) {
|
|
|
761
769
|
return decide(chosen);
|
|
762
770
|
}
|
|
763
771
|
function selectCodexCredentialLeaseForTurn(args) {
|
|
764
|
-
const {
|
|
772
|
+
const { activeCredentialId, rotationEnabled, rotationStrategy, existingCredentialId } = args.context;
|
|
773
|
+
if (args.context.failoverExhausted) {
|
|
774
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
775
|
+
}
|
|
776
|
+
const accounts = args.context.modelId ? args.context.accounts.filter(
|
|
777
|
+
(account) => codexAccountServesModel(account, args.context.modelId, args.now)
|
|
778
|
+
) : args.context.accounts;
|
|
765
779
|
const failedCredentialIds = new Set(args.context.failedCredentialIds ?? []);
|
|
766
780
|
const strategy = effectiveRotationStrategy(rotationStrategy);
|
|
767
781
|
const existing = existingCredentialId ? accounts.find((account) => account.id === existingCredentialId) : void 0;
|
|
@@ -787,6 +801,9 @@ function selectCodexCredentialLeaseForTurn(args) {
|
|
|
787
801
|
if (!pinned) {
|
|
788
802
|
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
789
803
|
}
|
|
804
|
+
if (failedCredentialIds.has(pinned.id)) {
|
|
805
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
806
|
+
}
|
|
790
807
|
if (!pinned.allocatorEnabled) {
|
|
791
808
|
return {
|
|
792
809
|
credentialId: null,
|
|
@@ -843,6 +860,9 @@ function selectCodexCredentialLeaseForTurn(args) {
|
|
|
843
860
|
if (!active) {
|
|
844
861
|
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
845
862
|
}
|
|
863
|
+
if (failedCredentialIds.has(active.id)) {
|
|
864
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
865
|
+
}
|
|
846
866
|
if (!active.allocatorEnabled) {
|
|
847
867
|
return {
|
|
848
868
|
credentialId: null,
|
|
@@ -910,8 +930,13 @@ async function signalPendingCodexCapacityWakeTargets(services, workspaceId) {
|
|
|
910
930
|
);
|
|
911
931
|
await signalCodexCapacityWakeTargets(services, targets);
|
|
912
932
|
}
|
|
913
|
-
function codexCapacityDecision(context) {
|
|
914
|
-
|
|
933
|
+
function codexCapacityDecision(context, now = /* @__PURE__ */ new Date()) {
|
|
934
|
+
context = {
|
|
935
|
+
...context,
|
|
936
|
+
accounts: context.accounts.filter(
|
|
937
|
+
(account) => !context.modelId || codexAccountServesModel(account, context.modelId, now)
|
|
938
|
+
)
|
|
939
|
+
};
|
|
915
940
|
const selected = selectCodexCredentialLeaseForTurn({
|
|
916
941
|
context,
|
|
917
942
|
sessionId: context.sessionId,
|
|
@@ -928,7 +953,9 @@ function codexCapacityDecision(context) {
|
|
|
928
953
|
credentialId: selected.credentialId,
|
|
929
954
|
diagnostic: {
|
|
930
955
|
connectedCount: context.accounts.length,
|
|
931
|
-
eligibleCount: context.accounts.filter(
|
|
956
|
+
eligibleCount: context.accounts.filter(
|
|
957
|
+
(account) => (account.id === selected.credentialId || !context.failedCredentialIds?.includes(account.id)) && isCodexCredentialEligible(account, now)
|
|
958
|
+
).length
|
|
932
959
|
}
|
|
933
960
|
};
|
|
934
961
|
}
|
|
@@ -936,7 +963,7 @@ function codexCapacityDecision(context) {
|
|
|
936
963
|
const capacityAccounts = policyCredentialId ? context.accounts.filter((account) => account.id === policyCredentialId) : context.accounts;
|
|
937
964
|
const authoritativeReset = authoritativeCodexCapacityResetAt(capacityAccounts, now);
|
|
938
965
|
const hasReconcilableQuotaCooldown = capacityAccounts.some(
|
|
939
|
-
(account) => account.status === "active" && account.allocatorEnabled && account.exhaustedKind === "quota" && account.exhaustedUntil !== null
|
|
966
|
+
(account) => account.status === "active" && account.allocatorEnabled && account.exhaustedKind === "quota" && account.exhaustedUntil !== null
|
|
940
967
|
);
|
|
941
968
|
const policyAccount = capacityAccounts[0] ?? null;
|
|
942
969
|
const mutationOnlyStatusBlock = policyAccount != null && (!policyAccount.allocatorEnabled || policyAccount.status !== "active") || authoritativeReset === null && capacityAccounts.length > 0 && capacityAccounts.every(
|
|
@@ -946,7 +973,7 @@ function codexCapacityDecision(context) {
|
|
|
946
973
|
return {
|
|
947
974
|
kind: "unavailable",
|
|
948
975
|
earliestResetAt: authoritativeReset,
|
|
949
|
-
resetKind: selected.decision.kind === "none" || selected.decision.kind === "allocatorDisabled" || mutationOnlyStatusBlock ? "mutation_only" : authoritativeReset && !hasReconcilableQuotaCooldown ? "authoritative" : "bounded_refresh",
|
|
976
|
+
resetKind: selected.decision.kind === "none" && !hasReconcilableQuotaCooldown && !authoritativeReset || selected.decision.kind === "allocatorDisabled" || mutationOnlyStatusBlock ? "mutation_only" : authoritativeReset && !hasReconcilableQuotaCooldown ? "authoritative" : "bounded_refresh",
|
|
950
977
|
diagnostic: {
|
|
951
978
|
connectedCount: context.accounts.length,
|
|
952
979
|
allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
|
|
@@ -957,6 +984,16 @@ function codexCapacityDecision(context) {
|
|
|
957
984
|
}
|
|
958
985
|
async function armAndReconcileCodexCapacityWait(services, input, options = {}) {
|
|
959
986
|
const armed = await armCodexCapacityWait(services.db, input);
|
|
987
|
+
if (armed.action === "stopped") {
|
|
988
|
+
options.onArmed?.();
|
|
989
|
+
await publishDurableSessionEvents(
|
|
990
|
+
services.bus,
|
|
991
|
+
input.workspaceId,
|
|
992
|
+
input.sessionId,
|
|
993
|
+
armed.events
|
|
994
|
+
);
|
|
995
|
+
return armed;
|
|
996
|
+
}
|
|
960
997
|
if (armed.action !== "waiting") return armed;
|
|
961
998
|
options.onArmed?.();
|
|
962
999
|
await publishDurableSessionEvents(services.bus, input.workspaceId, input.sessionId, armed.events);
|
|
@@ -980,15 +1017,22 @@ async function armAndReconcileCodexCapacityWait(services, input, options = {}) {
|
|
|
980
1017
|
);
|
|
981
1018
|
return evaluated;
|
|
982
1019
|
}
|
|
983
|
-
async function refreshCapacityMetadata(services, workspaceId) {
|
|
984
|
-
const accounts = await listCodexAccountStatuses(services.db, workspaceId).catch(() => []);
|
|
1020
|
+
async function refreshCapacityMetadata(services, workspaceId, turnId) {
|
|
1021
|
+
const accounts = await listCodexAccountStatuses(services.db, workspaceId, turnId).catch(() => []);
|
|
985
1022
|
const now = /* @__PURE__ */ new Date();
|
|
986
1023
|
const stale = accounts.filter(
|
|
987
1024
|
(account) => account.allocatorEnabled && account.status === "active" && (codexAccountNeedsLiveCapacityRefresh(account, now) || account.usageCheckedAt === null)
|
|
988
1025
|
);
|
|
989
1026
|
await refreshCodexUsageAndRepairCapacityWaiters(
|
|
990
1027
|
stale.map(
|
|
991
|
-
(account) => () => fetchCodexUsageForAccount(
|
|
1028
|
+
(account) => () => fetchCodexUsageForAccount(
|
|
1029
|
+
services.db,
|
|
1030
|
+
services.settings,
|
|
1031
|
+
workspaceId,
|
|
1032
|
+
account.id,
|
|
1033
|
+
void 0,
|
|
1034
|
+
turnId
|
|
1035
|
+
)
|
|
992
1036
|
),
|
|
993
1037
|
() => signalPendingCodexCapacityWakeTargets(services, workspaceId)
|
|
994
1038
|
);
|
|
@@ -1002,9 +1046,10 @@ function createCodexCapacityActivities(services) {
|
|
|
1002
1046
|
input.sessionId
|
|
1003
1047
|
);
|
|
1004
1048
|
const xaiWaiter = codexWaiter ? null : await getXaiCapacityWaitForSession(db, input.workspaceId, input.sessionId);
|
|
1005
|
-
const
|
|
1049
|
+
const claudeWaiter = codexWaiter || xaiWaiter ? null : await getClaudeCapacityWaitForSession(db, input.workspaceId, input.sessionId);
|
|
1050
|
+
const waiter = codexWaiter ?? xaiWaiter ?? claudeWaiter;
|
|
1006
1051
|
return waiter ? {
|
|
1007
|
-
...xaiWaiter ? { provider: "xai" } : {},
|
|
1052
|
+
...xaiWaiter ? { provider: "xai" } : claudeWaiter ? { provider: "claude" } : {},
|
|
1008
1053
|
waiterId: waiter.id,
|
|
1009
1054
|
generation: waiter.generation,
|
|
1010
1055
|
// A capacity mutation may have committed while its Temporal signal
|
|
@@ -1017,21 +1062,17 @@ function createCodexCapacityActivities(services) {
|
|
|
1017
1062
|
}
|
|
1018
1063
|
async function reconcileCodexCapacityWait(input) {
|
|
1019
1064
|
const resolved = await services();
|
|
1020
|
-
if (input.provider === "xai") {
|
|
1021
|
-
const
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
);
|
|
1065
|
+
if (input.provider === "xai" || input.provider === "claude") {
|
|
1066
|
+
const claude = input.provider === "claude";
|
|
1067
|
+
const getWaiter = claude ? getClaudeCapacityWaitForSession : getXaiCapacityWaitForSession;
|
|
1068
|
+
const resolveAuthority = claude ? resolveClaudeWaiterSubject : resolveXaiWaiterSubject;
|
|
1069
|
+
const reconcile = claude ? reconcileClaudeCapacityWaitDb : reconcileXaiCapacityWaitDb;
|
|
1070
|
+
const current2 = await getWaiter(resolved.db, input.workspaceId, input.sessionId);
|
|
1026
1071
|
if (!current2 || current2.id !== input.waiterId || current2.generation !== input.generation) {
|
|
1027
1072
|
return { action: "stale" };
|
|
1028
1073
|
}
|
|
1029
|
-
const authority = await
|
|
1030
|
-
|
|
1031
|
-
input.workspaceId,
|
|
1032
|
-
input.sessionId
|
|
1033
|
-
);
|
|
1034
|
-
if (authority)
|
|
1074
|
+
const authority = await resolveAuthority(resolved.db, input.workspaceId, input.sessionId);
|
|
1075
|
+
if (authority && !claude)
|
|
1035
1076
|
await refreshExhaustedXaiQuota({
|
|
1036
1077
|
db: resolved.db,
|
|
1037
1078
|
settings: resolved.settings,
|
|
@@ -1042,7 +1083,7 @@ function createCodexCapacityActivities(services) {
|
|
|
1042
1083
|
subjectId: authority.subjectId,
|
|
1043
1084
|
authoritySnapshot: authority.snapshot
|
|
1044
1085
|
});
|
|
1045
|
-
const result2 = await
|
|
1086
|
+
const result2 = await reconcile(resolved.db, {
|
|
1046
1087
|
accountId: input.accountId,
|
|
1047
1088
|
workspaceId: input.workspaceId,
|
|
1048
1089
|
sessionId: input.sessionId,
|
|
@@ -1059,7 +1100,7 @@ function createCodexCapacityActivities(services) {
|
|
|
1059
1100
|
if (result2.action === "waiting") {
|
|
1060
1101
|
return {
|
|
1061
1102
|
action: "waiting",
|
|
1062
|
-
provider:
|
|
1103
|
+
provider: input.provider,
|
|
1063
1104
|
waiterId: result2.waiter.id,
|
|
1064
1105
|
generation: result2.waiter.generation,
|
|
1065
1106
|
nextCheckAt: result2.waiter.nextCheckAt.toISOString(),
|
|
@@ -1078,7 +1119,7 @@ function createCodexCapacityActivities(services) {
|
|
|
1078
1119
|
}
|
|
1079
1120
|
const boundedRefreshAttempted = current.resetKind === "bounded_refresh" && input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now();
|
|
1080
1121
|
if (boundedRefreshAttempted) {
|
|
1081
|
-
await refreshCapacityMetadata(resolved, input.workspaceId);
|
|
1122
|
+
await refreshCapacityMetadata(resolved, input.workspaceId, current.blockedTurnId);
|
|
1082
1123
|
}
|
|
1083
1124
|
const result = await reconcileCodexCapacityWaitDb(
|
|
1084
1125
|
resolved.db,
|
|
@@ -1194,7 +1235,8 @@ function createDocumentActivities(services, resolveDocumentServices) {
|
|
|
1194
1235
|
// src/activities/goals.ts
|
|
1195
1236
|
import {
|
|
1196
1237
|
allowedFirstPartyMcpToolsForSession,
|
|
1197
|
-
configuredStaticUsageLimits,
|
|
1238
|
+
configuredStaticUsageLimits as configuredStaticUsageLimits2,
|
|
1239
|
+
isModelAvailableForNewSelection,
|
|
1198
1240
|
policyProviderIdForModel,
|
|
1199
1241
|
resolveModelProvider,
|
|
1200
1242
|
resolveTurnExecutionPolicyV1,
|
|
@@ -1205,25 +1247,106 @@ import {
|
|
|
1205
1247
|
} from "@opengeni/config";
|
|
1206
1248
|
import {
|
|
1207
1249
|
evaluateWorkspaceModelPolicy,
|
|
1208
|
-
mergeToolRefs
|
|
1250
|
+
mergeToolRefs,
|
|
1251
|
+
readTurnExecutionPolicyV1 as readTurnExecutionPolicyV12
|
|
1209
1252
|
} from "@opengeni/contracts";
|
|
1210
1253
|
import { isCodexBilledModel } from "@opengeni/codex";
|
|
1211
1254
|
import {
|
|
1212
1255
|
enqueueSessionWorkflowWakeIfRunnable,
|
|
1213
|
-
getBillingBalance,
|
|
1214
|
-
getLatestStartedSessionTurn,
|
|
1215
1256
|
getWorkspaceModelPolicy,
|
|
1216
1257
|
getSessionGoal,
|
|
1217
|
-
|
|
1258
|
+
getSessionTurn,
|
|
1218
1259
|
materializeGoalContinuation,
|
|
1219
|
-
requireSession
|
|
1220
|
-
sumUsageQuantity
|
|
1260
|
+
requireSession
|
|
1221
1261
|
} from "@opengeni/db";
|
|
1222
1262
|
import {
|
|
1223
|
-
modelFundingForAdmission,
|
|
1263
|
+
modelFundingForAdmission as modelFundingForAdmission2,
|
|
1224
1264
|
resolveCatalogSettings as resolveCatalogSettings2,
|
|
1225
1265
|
resolveWorkspaceCatalogSettings
|
|
1226
1266
|
} from "@opengeni/core";
|
|
1267
|
+
|
|
1268
|
+
// src/activities/agent-run-admission.ts
|
|
1269
|
+
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
1270
|
+
import { modelFundingForAdmission } from "@opengeni/core";
|
|
1271
|
+
import {
|
|
1272
|
+
checkWorkspaceAllowance,
|
|
1273
|
+
getBillingBalance,
|
|
1274
|
+
isCodexBilledTurn,
|
|
1275
|
+
sumUsageQuantity
|
|
1276
|
+
} from "@opengeni/db";
|
|
1277
|
+
async function agentRunAdmissionDenial(services, input) {
|
|
1278
|
+
const codexBilled = await isCodexBilledTurn({
|
|
1279
|
+
db: services.db,
|
|
1280
|
+
settings: services.settings,
|
|
1281
|
+
workspaceId: input.workspaceId,
|
|
1282
|
+
model: input.model
|
|
1283
|
+
});
|
|
1284
|
+
const externallyBilled = modelFundingForAdmission(
|
|
1285
|
+
services.settings,
|
|
1286
|
+
input.model,
|
|
1287
|
+
codexBilled
|
|
1288
|
+
).fundedWithoutCredits;
|
|
1289
|
+
if (!externallyBilled && (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")) {
|
|
1290
|
+
if (services.entitlements) {
|
|
1291
|
+
const decision = await services.entitlements.admitRun({
|
|
1292
|
+
accountId: input.accountId,
|
|
1293
|
+
workspaceId: input.workspaceId,
|
|
1294
|
+
action: "agent_run:create",
|
|
1295
|
+
quantity: input.requestedAgentRuns
|
|
1296
|
+
});
|
|
1297
|
+
if (!decision.allowed) return "insufficient_credits";
|
|
1298
|
+
} else {
|
|
1299
|
+
const balance = await getBillingBalance(services.db, input.accountId);
|
|
1300
|
+
if (balance.balanceMicros <= 0) return "insufficient_credits";
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
if (!externallyBilled) {
|
|
1304
|
+
const refusal = await checkWorkspaceAllowance(services.db, {
|
|
1305
|
+
accountId: input.accountId,
|
|
1306
|
+
workspaceId: input.workspaceId,
|
|
1307
|
+
subjectId: input.initiatingHumanSubjectId ?? null
|
|
1308
|
+
});
|
|
1309
|
+
if (refusal) return refusal.code;
|
|
1310
|
+
}
|
|
1311
|
+
if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
|
|
1312
|
+
return null;
|
|
1313
|
+
}
|
|
1314
|
+
const limits = configuredStaticUsageLimits(services.settings);
|
|
1315
|
+
if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
|
|
1316
|
+
const used = await sumUsageQuantity(services.db, {
|
|
1317
|
+
accountId: input.accountId,
|
|
1318
|
+
eventType: "model.cost",
|
|
1319
|
+
since: startOfUtcMonth()
|
|
1320
|
+
});
|
|
1321
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
1322
|
+
return "monthly_model_cost_limit";
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
1326
|
+
const used = await sumUsageQuantity(services.db, {
|
|
1327
|
+
workspaceId: input.workspaceId,
|
|
1328
|
+
eventType: "agent_run.created",
|
|
1329
|
+
since: startOfUtcMonth()
|
|
1330
|
+
});
|
|
1331
|
+
if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
1332
|
+
return "monthly_agent_run_limit";
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
return null;
|
|
1336
|
+
}
|
|
1337
|
+
function startOfUtcMonth() {
|
|
1338
|
+
const now = /* @__PURE__ */ new Date();
|
|
1339
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
// src/activities/agent-turn/credential-restriction.ts
|
|
1343
|
+
import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
1344
|
+
function turnCredentialRestriction(acceptedPolicy, initialSessionMetadata) {
|
|
1345
|
+
const initialPolicy = readTurnExecutionPolicyV1(initialSessionMetadata);
|
|
1346
|
+
return acceptedPolicy.credentialRestriction === "developer_setup" || initialPolicy.kind === "valid" && initialPolicy.policy.credentialRestriction === "developer_setup" ? "developer_setup" : void 0;
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
// src/activities/goals.ts
|
|
1227
1350
|
function createGoalActivities(services) {
|
|
1228
1351
|
async function enqueueGoalRetryWake(input) {
|
|
1229
1352
|
const { db } = await services();
|
|
@@ -1246,17 +1369,15 @@ function createGoalActivities(services) {
|
|
|
1246
1369
|
if (!existingGoal || existingGoal.status !== "active") {
|
|
1247
1370
|
return { action: "none" };
|
|
1248
1371
|
}
|
|
1249
|
-
let settings = (await resolveCatalogSettings2(db, catalogSourceSettings)).settings;
|
|
1250
1372
|
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
const inheritedContinuationModel = latestStartedTurn?.model ?? session.model;
|
|
1373
|
+
if (session.status === "failed" || session.status === "cancelled") {
|
|
1374
|
+
return { action: "none" };
|
|
1375
|
+
}
|
|
1376
|
+
let settings = (await resolveCatalogSettings2(db, catalogSourceSettings)).settings;
|
|
1377
|
+
const inheritedContinuationModel = session.model;
|
|
1257
1378
|
let continuationModel = inheritedContinuationModel;
|
|
1258
|
-
const continuationReasoningEffort =
|
|
1259
|
-
const continuationLatencyMode =
|
|
1379
|
+
const continuationReasoningEffort = session.reasoningEffort;
|
|
1380
|
+
const continuationLatencyMode = session.latencyMode;
|
|
1260
1381
|
const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
|
|
1261
1382
|
if (inheritedContinuationModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || inheritedContinuationModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || session.model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || session.model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX)) {
|
|
1262
1383
|
settings = (await resolveWorkspaceCatalogSettings(db, catalogSourceSettings, {
|
|
@@ -1275,25 +1396,7 @@ function createGoalActivities(services) {
|
|
|
1275
1396
|
if (!modelPolicyBlocked && session.codexCompactionMode === "remote_v2" && !isCodexBilledModel(continuationModel)) {
|
|
1276
1397
|
modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
|
|
1277
1398
|
}
|
|
1278
|
-
const
|
|
1279
|
-
db,
|
|
1280
|
-
settings,
|
|
1281
|
-
workspaceId: input.workspaceId,
|
|
1282
|
-
model: continuationModel
|
|
1283
|
-
});
|
|
1284
|
-
const fundedWithoutCredits = goalContinuationFundedWithoutCredits(
|
|
1285
|
-
settings,
|
|
1286
|
-
continuationModel,
|
|
1287
|
-
isCodexRun
|
|
1288
|
-
);
|
|
1289
|
-
const budgetBlocked = await goalRunBudgetBlocked(
|
|
1290
|
-
settings,
|
|
1291
|
-
db,
|
|
1292
|
-
input.accountId,
|
|
1293
|
-
input.workspaceId,
|
|
1294
|
-
fundedWithoutCredits
|
|
1295
|
-
);
|
|
1296
|
-
const turnExecutionPolicy = resolveTurnExecutionPolicyV1(settings, {
|
|
1399
|
+
const turnExecutionPolicy = modelPolicyBlocked ? void 0 : resolveTurnExecutionPolicyV1(settings, {
|
|
1297
1400
|
modelId: continuationModel,
|
|
1298
1401
|
requestedModelId: null,
|
|
1299
1402
|
modelSource: "continuation",
|
|
@@ -1302,6 +1405,14 @@ function createGoalActivities(services) {
|
|
|
1302
1405
|
latencyMode: continuationLatencyMode,
|
|
1303
1406
|
latencyModeSource: "continuation"
|
|
1304
1407
|
});
|
|
1408
|
+
const continuationPolicy = {
|
|
1409
|
+
model: continuationModel,
|
|
1410
|
+
reasoningEffort: continuationReasoningEffort,
|
|
1411
|
+
latencyMode: continuationLatencyMode,
|
|
1412
|
+
...turnExecutionPolicy ? { turnExecutionPolicy } : {},
|
|
1413
|
+
tools: withFirstPartyTools(settings, session.tools),
|
|
1414
|
+
sandboxBackend: session.sandboxBackend
|
|
1415
|
+
};
|
|
1305
1416
|
const decision = await materializeGoalContinuation(db, {
|
|
1306
1417
|
accountId: input.accountId,
|
|
1307
1418
|
workspaceId: input.workspaceId,
|
|
@@ -1317,15 +1428,34 @@ function createGoalActivities(services) {
|
|
|
1317
1428
|
// A model-policy block takes precedence: it is deterministic (a budget
|
|
1318
1429
|
// pause can clear on its own; a policy pause needs a model/policy change)
|
|
1319
1430
|
// and rides the same visible-pause channel.
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1431
|
+
admission: async (tx, causalTurn) => {
|
|
1432
|
+
const sourceTurn = causalTurn ? await getSessionTurn(tx, input.workspaceId, causalTurn.id) : null;
|
|
1433
|
+
if (causalTurn && (!sourceTurn || sourceTurn.sessionId !== input.sessionId)) {
|
|
1434
|
+
throw new Error("Goal continuation source turn is unavailable");
|
|
1435
|
+
}
|
|
1436
|
+
const sourcePolicy = readTurnExecutionPolicyV12(sourceTurn?.metadata);
|
|
1437
|
+
if (turnExecutionPolicy) {
|
|
1438
|
+
const credentialRestriction = turnCredentialRestriction(
|
|
1439
|
+
sourcePolicy.kind === "valid" ? sourcePolicy.policy : turnExecutionPolicy,
|
|
1440
|
+
session.metadata
|
|
1441
|
+
);
|
|
1442
|
+
continuationPolicy.turnExecutionPolicy = credentialRestriction ? { ...turnExecutionPolicy, credentialRestriction } : turnExecutionPolicy;
|
|
1443
|
+
}
|
|
1444
|
+
const budgetBlocked = modelPolicyBlocked ? null : await goalRunBudgetBlocked(
|
|
1445
|
+
{ ...service, settings, db: tx },
|
|
1446
|
+
{
|
|
1447
|
+
accountId: input.accountId,
|
|
1448
|
+
workspaceId: input.workspaceId,
|
|
1449
|
+
model: continuationModel,
|
|
1450
|
+
initiatingHumanSubjectId: causalTurn?.initiatingHumanSubjectId ?? null
|
|
1451
|
+
}
|
|
1452
|
+
);
|
|
1453
|
+
return {
|
|
1454
|
+
budgetBlocked: modelPolicyBlocked ?? budgetBlocked?.message ?? null,
|
|
1455
|
+
budgetPausedReason: modelPolicyBlocked ? "limits" : budgetBlocked?.pausedReason ?? "limits"
|
|
1456
|
+
};
|
|
1328
1457
|
},
|
|
1458
|
+
policy: continuationPolicy,
|
|
1329
1459
|
// Long-wait guidance is only given when `wait_for_input` is actually in this
|
|
1330
1460
|
// session's effective first-party selection (the same source the worker
|
|
1331
1461
|
// signs into the delegated token and the API uses to register tools), so
|
|
@@ -1361,46 +1491,52 @@ function goalContinuationModelDecision(input) {
|
|
|
1361
1491
|
providerId: policyProviderIdForModel(catalogSettings, modelId),
|
|
1362
1492
|
modelId
|
|
1363
1493
|
}).allowed;
|
|
1364
|
-
if (resolveModelProvider(catalogSettings, input.inheritedModel) && !policyBlocks(input.inheritedModel)) {
|
|
1365
|
-
return { model: input.inheritedModel, blocked: null };
|
|
1366
|
-
}
|
|
1367
1494
|
if (!resolveModelProvider(catalogSettings, input.inheritedModel)) {
|
|
1368
1495
|
return {
|
|
1369
1496
|
model: input.inheritedModel,
|
|
1370
1497
|
blocked: `model "${input.inheritedModel}" is no longer in the deployment or workspace catalog; choose an available model before resuming the goal`
|
|
1371
1498
|
};
|
|
1372
1499
|
}
|
|
1500
|
+
if (!isModelAvailableForNewSelection(catalogSettings, input.inheritedModel)) {
|
|
1501
|
+
return {
|
|
1502
|
+
model: input.inheritedModel,
|
|
1503
|
+
blocked: `model "${input.inheritedModel}" is retired from new selection; choose an available model before resuming the goal`
|
|
1504
|
+
};
|
|
1505
|
+
}
|
|
1506
|
+
if (!policyBlocks(input.inheritedModel)) {
|
|
1507
|
+
return { model: input.inheritedModel, blocked: null };
|
|
1508
|
+
}
|
|
1373
1509
|
return {
|
|
1374
1510
|
model: input.inheritedModel,
|
|
1375
1511
|
blocked: `workspace model policy blocks model "${input.inheritedModel}"; pick an allowed model or change the workspace model policy`
|
|
1376
1512
|
};
|
|
1377
1513
|
}
|
|
1378
|
-
function goalContinuationFundedWithoutCredits(settings, model, codexBilled) {
|
|
1379
|
-
return modelFundingForAdmission(settings, model, codexBilled).fundedWithoutCredits;
|
|
1380
|
-
}
|
|
1381
1514
|
function goalContinuationPrompt(_goal, _autoContinuation, _cap, options = {}) {
|
|
1382
1515
|
const waitingGuidance = options.inputWaitAvailable ? [
|
|
1383
1516
|
"Waiting on child sessions, background commands, or external events:",
|
|
1384
1517
|
"- When further progress depends on work already in flight, do not sleep, loop, or poll session or command state repeatedly.",
|
|
1385
|
-
"-
|
|
1386
|
-
"-
|
|
1518
|
+
"- If progress depends on work already in flight and the wait is long or uncertain, call opengeni__wait_for_input with a concrete reason and timeoutSeconds, then end your turn immediately. A preliminary status check is not required. Relevant session input or the safety deadline will start a new turn, and this goal stays active.",
|
|
1519
|
+
"- No short execution wait is required first. Choose the safety deadline for the dependency or a meaningful monitoring cadence; an out-of-turn wait may span hours or days within the tool's limits. Do not schedule model wakeups merely to repeat reassurance. Honor explicit user/task/Skill check or update cadences within existing authority.",
|
|
1520
|
+
"- If the immediately preceding user-facing update already reported this same unchanged wait, do not restate it or produce another equivalent final answer. Call opengeni__wait_for_input and end the turn. Report only material new state or a newly discovered blocker, unless an explicit user/task/Skill update cadence calls for an update.",
|
|
1387
1521
|
"- If you are blocked on a human decision, use opengeni__goal_pause under the blocked audit below instead.",
|
|
1388
1522
|
""
|
|
1389
1523
|
] : [];
|
|
1390
1524
|
const childNoticeGuidance = [
|
|
1391
1525
|
"Child lifecycle notices:",
|
|
1392
|
-
"- A `child_requires_action` update means a worker you spawned is blocked on a question or a tool approval and will not progress until it is answered. " + (options.humanInputRespondAvailable ? "If you know the answer to its question, answer it with opengeni__session_human_input_respond (pass the worker's sessionId, the requestId from the notice, and a response). " : "") + "Tool approvals can only be decided by a human. If you cannot resolve the blocker yourself, report the exact blocker (worker session id, the question)
|
|
1526
|
+
"- A `child_requires_action` update means a worker you spawned is blocked on a question or a tool approval and will not progress until it is answered. " + (options.humanInputRespondAvailable ? "If you know the answer to its question, answer it with opengeni__session_human_input_respond (pass the worker's sessionId, the requestId from the notice, and a response). " : "") + "Tool approvals can only be decided by a human. If you cannot resolve the blocker yourself, report the exact blocker (worker session id, the question); if it prevents further goal progress, pause under the blocked audit instead of retrying the worker or treating a human decision as an in-flight wait.",
|
|
1393
1527
|
"- `child_requires_action_resolved`, `child_paused`, `child_waiting_capacity`, and `child_progress` updates are informational: a resolved notice means the worker is moving again; a paused worker needs a human or you to resume it; a capacity wait resumes by itself; a progress note needs no action.",
|
|
1394
1528
|
""
|
|
1395
1529
|
];
|
|
1396
1530
|
return [
|
|
1397
|
-
"
|
|
1531
|
+
"Automatic goal continuation (generated input, not a new user request or grant of authority). Resume established work toward the full applied objective and verify its requested end state.",
|
|
1398
1532
|
"",
|
|
1399
1533
|
"Goal recovery:",
|
|
1534
|
+
"- Use the current applied goal frozen for this turn, including its success criteria, root constraints, and report requirements, as the authoritative objective. Pending proposals and older goal revisions do not replace it; change it only through the authorized goal-mutation controls, without widening authority.",
|
|
1400
1535
|
"- Treat this continuation as re-entry into the full objective, not as a request to perform one step and stop.",
|
|
1536
|
+
"- Resume from established work and relevant evidence rather than restarting discovery or a full reconciliation on every turn. Perform full reconciliation or a comprehensive audit when the goal, user, or applicable Skill calls for it, when uncertainty or recovery warrants it, or when required by risk or a gate.",
|
|
1401
1537
|
"- Do not rely on previous assistant claims of progress or completion; use them only to locate authoritative evidence.",
|
|
1402
1538
|
"- If authoritative evidence already proves the full objective, call opengeni__goal_complete instead of manufacturing more work.",
|
|
1403
|
-
"- Before repeating a state-setting action, verify whether its desired state already holds. If it does, do not repeat it; continue
|
|
1539
|
+
"- Before repeating a state-setting action, verify whether its desired state already holds, using relevant evidence that remains valid or a fresh check when needed. If it does, do not repeat it; continue the overall goal.",
|
|
1404
1540
|
"",
|
|
1405
1541
|
"Continuation behavior:",
|
|
1406
1542
|
"- This goal persists across turns. A runtime boundary can end one turn without shrinking the objective; the next continuation resumes the same full objective.",
|
|
@@ -1410,7 +1546,7 @@ function goalContinuationPrompt(_goal, _autoContinuation, _cap, options = {}) {
|
|
|
1410
1546
|
"- Temporary rough edges are acceptable while the work is moving in the right direction. Completion still requires the requested end state to be true and verified.",
|
|
1411
1547
|
"",
|
|
1412
1548
|
"Work from evidence:",
|
|
1413
|
-
"Use
|
|
1549
|
+
"Use authoritative workspace and external evidence, including prior tool results that remain relevant and valid. Reuse evidence only for the requirement, scope, version, and state it actually establishes; recheck changed, stale, uncertain, or insufficient evidence. Conversation summaries and assistant claims are pointers, not proof. Do not infer progress or evidence validity merely from this continuation. Improve, replace, or remove existing work as needed within authority to satisfy the actual objective.",
|
|
1414
1550
|
"",
|
|
1415
1551
|
"Progress visibility:",
|
|
1416
1552
|
"If a planning tool is available and the next work is meaningfully multi-step, use it to show a concise plan tied to the real objective. Keep the plan current as steps complete or the next best action changes. Skip planning overhead for trivial one-step progress, and do not treat a plan update as a substitute for doing the work.",
|
|
@@ -1421,22 +1557,26 @@ function goalContinuationPrompt(_goal, _autoContinuation, _cap, options = {}) {
|
|
|
1421
1557
|
"- Treat alignment as movement toward the requested end state. An edit is aligned only if it makes the requested final state more true; useful-looking behavior that preserves a different end state is misaligned.",
|
|
1422
1558
|
"",
|
|
1423
1559
|
"Completion audit:",
|
|
1424
|
-
"Before deciding that the goal is achieved, treat completion as unproven and verify it against the actual current state:",
|
|
1560
|
+
"Before deciding that the goal is achieved, treat completion as unproven and verify it against the actual current state. This full completion audit is required even when ordinary continuation did not need full reconciliation; reuse still-valid authoritative evidence, but refresh it when the requirement or gate requires a fresh check:",
|
|
1425
1561
|
"- Derive concrete requirements from the objective and any referenced files, plans, specifications, issues, or user instructions.",
|
|
1426
|
-
"- Preserve the
|
|
1562
|
+
"- Preserve the applied objective's scope; do not redefine success around the work that already exists.",
|
|
1427
1563
|
"- For every explicit requirement, named artifact, command, test, gate, invariant, and deliverable, identify and inspect the authoritative evidence that would prove it.",
|
|
1428
1564
|
"- Match verification scope to requirement scope. Treat uncertain, indirect, incomplete, or missing evidence as not achieved and continue working.",
|
|
1429
1565
|
"- The audit must prove completion, not merely fail to find obvious remaining work.",
|
|
1566
|
+
"- For document report deliverables (a document the user asked for, or a large report meant to be kept or shared), follow the Documents Skill: create the durable native document first, inspect its relevant final head after the last edit, and provide the returned artifact reference. Declare report requirements through the available goal tools before authoring and satisfy every persisted report requirement with verified artifact delivery evidence before completion. Sandbox paths and raw file IDs do not prove report delivery. If artifact tooling or access is unavailable, keep that deliverable incomplete and state the blocker; never invent proof or silently substitute a local report. Ordinary chat answers, short progress updates, source-code links, and explicitly requested local-file work remain outside this report contract.",
|
|
1430
1567
|
"",
|
|
1431
1568
|
"Do not rely on intent, partial progress, memory of earlier work, or a plausible final answer as proof of completion. Call opengeni__goal_complete with concrete evidence only when the full objective is actually achieved and no required work remains.",
|
|
1569
|
+
"Goal evidence is a short proof for the ledger, not the deliverable. After goal_complete succeeds, finish this same turn with the requested user-facing answer, or a concise summary and retained artifact link. Goal completion stops future automatic continuations; it does not send the answer or end this turn. Never compress a report into evidence or omit the final reply.",
|
|
1570
|
+
"Goal progress notes are short human-readable milestone statuses, not raw transcripts or continuation instructions. Keep normal spaces and summarize detail instead of squeezing words into a ledger field. The text and successCriteria fields each allow 8192 UTF-8 bytes, progressNote allows 8192 UTF-8 bytes, rationale allows 2048 UTF-8 bytes, and evidence allows 8192 characters.",
|
|
1432
1571
|
"",
|
|
1433
1572
|
...waitingGuidance,
|
|
1434
1573
|
...childNoticeGuidance,
|
|
1435
1574
|
"Blocked audit:",
|
|
1436
|
-
"-
|
|
1437
|
-
"-
|
|
1575
|
+
"- Base persistence on evidence, not a fixed number of turns or retries. Investigate recoverable failures and try plausible safe alternatives within current authority when they could materially advance the goal; do not exhaust every imaginable alternative or repeat an unchanged failure without a reason to expect progress.",
|
|
1576
|
+
"- A definitive missing permission, required human decision, or external prerequisite with no actionable authorized path can justify pausing immediately. State the concrete blocker, relevant evidence or attempted alternatives, and what must change to resume. Tool approvals remain human-only.",
|
|
1577
|
+
"- If progress depends on work already in flight or a meaningful timed recheck, use the available waiting mechanism rather than pausing the goal; continue any independent authorized work first.",
|
|
1438
1578
|
"- Do not pause merely because the work is hard, slow, uncertain, incomplete, or would benefit from clarification.",
|
|
1439
|
-
"-
|
|
1579
|
+
"- When no meaningful authorized progress remains and the blocker requires human input or an external change that cannot be handled by an available wait or monitoring mechanism, call opengeni__goal_pause with the concrete blocker instead of repeatedly reporting it while leaving the goal active.",
|
|
1440
1580
|
"",
|
|
1441
1581
|
"Do not call opengeni__goal_complete or opengeni__goal_pause unless the corresponding audit above is satisfied."
|
|
1442
1582
|
].join("\n");
|
|
@@ -1447,44 +1587,26 @@ function withFirstPartyTools(settings, tools) {
|
|
|
1447
1587
|
}
|
|
1448
1588
|
return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
|
|
1449
1589
|
}
|
|
1450
|
-
async function goalRunBudgetBlocked(
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
since: startOfUtcMonth()
|
|
1464
|
-
});
|
|
1465
|
-
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
1466
|
-
return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
1470
|
-
const used = await sumUsageQuantity(db, {
|
|
1471
|
-
workspaceId,
|
|
1472
|
-
eventType: "agent_run.created",
|
|
1473
|
-
since: startOfUtcMonth()
|
|
1474
|
-
});
|
|
1475
|
-
if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
1476
|
-
return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
|
|
1477
|
-
}
|
|
1478
|
-
}
|
|
1479
|
-
}
|
|
1480
|
-
return null;
|
|
1481
|
-
}
|
|
1482
|
-
function startOfUtcMonth() {
|
|
1483
|
-
const now = /* @__PURE__ */ new Date();
|
|
1484
|
-
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
1590
|
+
async function goalRunBudgetBlocked(services, input) {
|
|
1591
|
+
const denial = await agentRunAdmissionDenial(services, { ...input, requestedAgentRuns: 1 });
|
|
1592
|
+
if (denial === null) return null;
|
|
1593
|
+
if (denial === "allowance_exhausted") {
|
|
1594
|
+
return { pausedReason: "allowance", message: "Opengeni usage allowance exhausted" };
|
|
1595
|
+
}
|
|
1596
|
+
const limits = configuredStaticUsageLimits2(services.settings);
|
|
1597
|
+
const messages = {
|
|
1598
|
+
insufficient_credits: "insufficient Opengeni credits",
|
|
1599
|
+
monthly_model_cost_limit: `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`,
|
|
1600
|
+
monthly_agent_run_limit: `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`
|
|
1601
|
+
};
|
|
1602
|
+
return { pausedReason: "limits", message: messages[denial] };
|
|
1485
1603
|
}
|
|
1486
1604
|
|
|
1487
1605
|
// src/activities/knowledge-source-sync.ts
|
|
1606
|
+
import {
|
|
1607
|
+
ATLASSIAN_NATIVE_RETIRED_REASON,
|
|
1608
|
+
isRetiredNativeAtlassianSource
|
|
1609
|
+
} from "@opengeni/contracts/atlassian-native-retirement";
|
|
1488
1610
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
|
|
1489
1611
|
import { AsyncLocalStorage } from "async_hooks";
|
|
1490
1612
|
import { scheduledTaskKnowledgeSource } from "@opengeni/contracts";
|
|
@@ -2018,6 +2140,12 @@ function createKnowledgeSourceSyncActivities(services, resolveDocumentServices)
|
|
|
2018
2140
|
if (!task || !action || !liveTask || liveTask.status !== "active" || task.id !== input.taskId || task.accountId !== input.accountId || task.action.kind !== "agent_turn" || action.sourceId !== input.sourceId || accepted.causalHumanSubjectId !== action.initiatingSubjectId) {
|
|
2019
2141
|
return { action: "failed", bufferedWake: false };
|
|
2020
2142
|
}
|
|
2143
|
+
if (isRetiredNativeAtlassianSource(action))
|
|
2144
|
+
return {
|
|
2145
|
+
action: "failed",
|
|
2146
|
+
bufferedWake: false,
|
|
2147
|
+
errorCode: ATLASSIAN_NATIVE_RETIRED_REASON
|
|
2148
|
+
};
|
|
2021
2149
|
boundSource = action;
|
|
2022
2150
|
await authorize();
|
|
2023
2151
|
await ensureKnowledgeSourceSyncState(db, task);
|
|
@@ -4583,65 +4711,10 @@ var SyncFailure = class extends Error {
|
|
|
4583
4711
|
import {
|
|
4584
4712
|
resolveRigProviderImageSelection,
|
|
4585
4713
|
rigProviderImageSourceImage,
|
|
4586
|
-
settingsWithPackSandboxImage,
|
|
4587
4714
|
settingsWithRigImage,
|
|
4588
4715
|
settingsWithRigProviderImage
|
|
4589
4716
|
} from "@opengeni/core";
|
|
4590
4717
|
|
|
4591
|
-
// src/activities/packs.ts
|
|
4592
|
-
import { getWorkspace, listInstalledPortableSkills } from "@opengeni/db";
|
|
4593
|
-
import {
|
|
4594
|
-
legacySandboxRuntimeFromPacks,
|
|
4595
|
-
packInstallationUsesLegacyRuntime,
|
|
4596
|
-
resolveWorkspaceLegacyRuntimePacks
|
|
4597
|
-
} from "@opengeni/core";
|
|
4598
|
-
import {
|
|
4599
|
-
buildPortableSkillArtifact
|
|
4600
|
-
} from "@opengeni/runtime";
|
|
4601
|
-
var emptyPackRuntime = {
|
|
4602
|
-
sandboxImage: null,
|
|
4603
|
-
sandboxProviderImages: null,
|
|
4604
|
-
skillActivations: []
|
|
4605
|
-
};
|
|
4606
|
-
async function resolveWorkspacePackRuntime(db, workspaceId) {
|
|
4607
|
-
const packs = await resolveWorkspaceLegacyRuntimePacks(db, workspaceId);
|
|
4608
|
-
if (packs.length === 0) return emptyPackRuntime;
|
|
4609
|
-
return workspacePackRuntimeFromPacks(packs);
|
|
4610
|
-
}
|
|
4611
|
-
function workspacePackRuntimeFromPacks(packs) {
|
|
4612
|
-
const sandboxRuntime = legacySandboxRuntimeFromPacks(packs);
|
|
4613
|
-
const skillActivations = [];
|
|
4614
|
-
const skillOwners = /* @__PURE__ */ new Map();
|
|
4615
|
-
for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
4616
|
-
for (const skill of pack.skills) {
|
|
4617
|
-
const key = skill.name.toLowerCase();
|
|
4618
|
-
const existingOwner = skillOwners.get(key);
|
|
4619
|
-
if (existingOwner !== void 0 && existingOwner !== pack.id) {
|
|
4620
|
-
throw new Error(
|
|
4621
|
-
`Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
|
|
4622
|
-
);
|
|
4623
|
-
}
|
|
4624
|
-
skillOwners.set(key, pack.id);
|
|
4625
|
-
const artifact = {
|
|
4626
|
-
name: skill.name,
|
|
4627
|
-
description: skill.description ?? null,
|
|
4628
|
-
files: skill.files.map((file) => ({ path: file.path, content: file.content }))
|
|
4629
|
-
};
|
|
4630
|
-
skillActivations.push({
|
|
4631
|
-
source: "pack",
|
|
4632
|
-
id: `pack:${pack.id}:${skill.name}`,
|
|
4633
|
-
artifact,
|
|
4634
|
-
reason: `active legacy Pack ${pack.id}`
|
|
4635
|
-
});
|
|
4636
|
-
}
|
|
4637
|
-
}
|
|
4638
|
-
return {
|
|
4639
|
-
sandboxImage: sandboxRuntime.sandboxImage,
|
|
4640
|
-
sandboxProviderImages: sandboxRuntime.sandboxProviderImages,
|
|
4641
|
-
skillActivations
|
|
4642
|
-
};
|
|
4643
|
-
}
|
|
4644
|
-
|
|
4645
4718
|
// src/activities/streaming.ts
|
|
4646
4719
|
import { TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS } from "@opengeni/core";
|
|
4647
4720
|
import { Context } from "@temporalio/activity";
|
|
@@ -4816,6 +4889,85 @@ async function nextStreamEvent(iterator, context) {
|
|
|
4816
4889
|
});
|
|
4817
4890
|
}
|
|
4818
4891
|
|
|
4892
|
+
// src/sandbox-snapshot-diagnostics.ts
|
|
4893
|
+
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
4894
|
+
function safeSnapshotError(error, origin = "sandbox-resume") {
|
|
4895
|
+
const fields = {
|
|
4896
|
+
errorClass: "SnapshotOperationError",
|
|
4897
|
+
errorCode: "snapshot_operation_failed",
|
|
4898
|
+
origin
|
|
4899
|
+
};
|
|
4900
|
+
try {
|
|
4901
|
+
if (error && typeof error === "object") {
|
|
4902
|
+
const candidate = error;
|
|
4903
|
+
if (typeof candidate.name === "string" && /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)) {
|
|
4904
|
+
fields.causeName = candidate.name;
|
|
4905
|
+
}
|
|
4906
|
+
if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
|
|
4907
|
+
fields.integrityCode = candidate.code;
|
|
4908
|
+
}
|
|
4909
|
+
const status = Number(candidate.status ?? candidate.statusCode);
|
|
4910
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
4911
|
+
}
|
|
4912
|
+
} catch {
|
|
4913
|
+
}
|
|
4914
|
+
const read = (value, key) => {
|
|
4915
|
+
try {
|
|
4916
|
+
return value && typeof value === "object" ? Reflect.get(value, key) : void 0;
|
|
4917
|
+
} catch {
|
|
4918
|
+
return void 0;
|
|
4919
|
+
}
|
|
4920
|
+
};
|
|
4921
|
+
const details = read(error, "details");
|
|
4922
|
+
const providerName = read(details, "errorName");
|
|
4923
|
+
if (typeof providerName === "string" && [
|
|
4924
|
+
"ClientError",
|
|
4925
|
+
"TimeoutError",
|
|
4926
|
+
"ConnectionError",
|
|
4927
|
+
"AuthError",
|
|
4928
|
+
"NotFoundError",
|
|
4929
|
+
"InvalidError",
|
|
4930
|
+
"RemoteError",
|
|
4931
|
+
"AbortError"
|
|
4932
|
+
].includes(providerName))
|
|
4933
|
+
fields.providerErrorName = providerName;
|
|
4934
|
+
const grpcCode = read(details, "errorCode");
|
|
4935
|
+
if (providerName === "ClientError" && typeof grpcCode === "number" && Number.isInteger(grpcCode) && grpcCode >= 0 && grpcCode <= 16)
|
|
4936
|
+
fields.providerGrpcCode = grpcCode;
|
|
4937
|
+
for (const key of ["status", "httpStatus", "responseStatus"]) {
|
|
4938
|
+
const status = read(details, key);
|
|
4939
|
+
if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
4940
|
+
fields.providerHttpStatus = status;
|
|
4941
|
+
break;
|
|
4942
|
+
}
|
|
4943
|
+
}
|
|
4944
|
+
const retryable = read(error, "retryable");
|
|
4945
|
+
if (typeof retryable === "boolean") fields.providerRetryable = retryable;
|
|
4946
|
+
return fields;
|
|
4947
|
+
}
|
|
4948
|
+
function warnDrainSnapshotFailure(observability, message, error, lease) {
|
|
4949
|
+
try {
|
|
4950
|
+
observability.warn(message, {
|
|
4951
|
+
...safeSnapshotError(error, "worker-lifecycle"),
|
|
4952
|
+
...lease.workspaceId ? { sandboxLeaseKey: sandboxLeaseTelemetryKey(lease.workspaceId, lease.sandboxGroupId) } : {},
|
|
4953
|
+
leaseEpoch: lease.leaseEpoch
|
|
4954
|
+
});
|
|
4955
|
+
} catch {
|
|
4956
|
+
}
|
|
4957
|
+
}
|
|
4958
|
+
|
|
4959
|
+
// src/session-event-fanout.ts
|
|
4960
|
+
async function publishDurableSessionEvents2(bus, workspaceId, events, onError = () => void 0) {
|
|
4961
|
+
if (!bus || !events?.length) return;
|
|
4962
|
+
for (const sessionId of new Set(events.map((event) => event.sessionId))) {
|
|
4963
|
+
await bus.publish(
|
|
4964
|
+
workspaceId,
|
|
4965
|
+
sessionId,
|
|
4966
|
+
events.filter((event) => event.sessionId === sessionId)
|
|
4967
|
+
).catch(onError);
|
|
4968
|
+
}
|
|
4969
|
+
}
|
|
4970
|
+
|
|
4819
4971
|
// src/sandbox-archive-storage.ts
|
|
4820
4972
|
import { uploadWorkspaceArchiveSpool } from "@opengeni/storage";
|
|
4821
4973
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
@@ -4949,15 +5101,17 @@ async function deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref, metri
|
|
|
4949
5101
|
}
|
|
4950
5102
|
|
|
4951
5103
|
// src/sandbox-routing.ts
|
|
5104
|
+
import { isDeepStrictEqual as isDeepStrictEqual3 } from "util";
|
|
4952
5105
|
import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
|
|
4953
5106
|
import {
|
|
4954
5107
|
createProviderCommandRetainer,
|
|
4955
|
-
|
|
5108
|
+
supervisedCommandProtocolReady
|
|
4956
5109
|
} from "@opengeni/db/retained-provider-commands";
|
|
4957
5110
|
import {
|
|
4958
5111
|
adoptConnectedMachineSessionBackgroundCommand,
|
|
4959
5112
|
adoptManagedSessionBackgroundCommand,
|
|
4960
5113
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
5114
|
+
retainedProviderCommandPersistence,
|
|
4961
5115
|
advanceWorkspaceGeneration,
|
|
4962
5116
|
getRetainedProcess,
|
|
4963
5117
|
retainWorkspaceMutationProcess,
|
|
@@ -4974,7 +5128,10 @@ import {
|
|
|
4974
5128
|
resolvePersonalMachineConnectionForAttempt,
|
|
4975
5129
|
SandboxRetainedProcessPromotionFencedError
|
|
4976
5130
|
} from "@opengeni/db";
|
|
4977
|
-
import {
|
|
5131
|
+
import {
|
|
5132
|
+
observeSessionBackgroundCommandCompletion,
|
|
5133
|
+
recordConnectedCommandOutputConsumption
|
|
5134
|
+
} from "@opengeni/db/session-background-commands";
|
|
4978
5135
|
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
4979
5136
|
import {
|
|
4980
5137
|
buildSelfhostedBackendSession,
|
|
@@ -4999,17 +5156,29 @@ import {
|
|
|
4999
5156
|
import {
|
|
5000
5157
|
effectiveSandboxLifecycle,
|
|
5001
5158
|
sandboxArchiveCaptureTimeoutMs,
|
|
5002
|
-
sandboxLifecycleTransitionWaitMs
|
|
5159
|
+
sandboxLifecycleTransitionWaitMs,
|
|
5160
|
+
sandboxWarmRateMicrosPerSecond
|
|
5003
5161
|
} from "@opengeni/config";
|
|
5004
5162
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
5163
|
+
|
|
5164
|
+
// src/warm-capture-holder.ts
|
|
5165
|
+
function retainsHolderForWarmCapture(lease, expectedEpoch, now = Date.now()) {
|
|
5166
|
+
return Boolean(
|
|
5167
|
+
lease?.liveness === "warm" && lease.leaseEpoch === expectedEpoch && lease.archiveCapture && lease.archiveCapture.deadlineAt.getTime() > now
|
|
5168
|
+
);
|
|
5169
|
+
}
|
|
5170
|
+
|
|
5171
|
+
// src/sandbox-resume.ts
|
|
5005
5172
|
import {
|
|
5006
5173
|
acquireLease,
|
|
5007
5174
|
adoptLegacyModalCheckpointArtifact,
|
|
5008
5175
|
beginSandboxRematerialization,
|
|
5176
|
+
sessionHoldsFreshWorkspaceRecovery,
|
|
5009
5177
|
claimWorkspaceArchiveCapture,
|
|
5010
5178
|
commitWarmingToWarm,
|
|
5011
5179
|
failSandboxRematerialization,
|
|
5012
5180
|
failWarmingToCold,
|
|
5181
|
+
beginModalProviderCreate,
|
|
5013
5182
|
getSandboxSessionEnvelope,
|
|
5014
5183
|
heartbeatLeaseHolderStatus,
|
|
5015
5184
|
markSandboxRestoreVerifying,
|
|
@@ -5054,6 +5223,7 @@ import {
|
|
|
5054
5223
|
downloadWorkspaceArchiveSpool,
|
|
5055
5224
|
WorkspaceArchiveStorageError
|
|
5056
5225
|
} from "@opengeni/storage";
|
|
5226
|
+
import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni/observability";
|
|
5057
5227
|
import { parseWorkspaceArchiveObjectRef as parseWorkspaceArchiveObjectRef2 } from "@opengeni/contracts";
|
|
5058
5228
|
function sandboxLeaseHolderIdForAttempt(attemptId) {
|
|
5059
5229
|
const normalized = attemptId.trim();
|
|
@@ -5130,6 +5300,42 @@ var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError
|
|
|
5130
5300
|
this.name = "SandboxSiblingWarmingTimeoutError";
|
|
5131
5301
|
}
|
|
5132
5302
|
};
|
|
5303
|
+
var disposedUnreadyFreshSandboxErrors = /* @__PURE__ */ new WeakSet();
|
|
5304
|
+
function isReplaceableFreshSandboxReadinessTimeout(error) {
|
|
5305
|
+
return error instanceof SandboxExecReadinessTimeoutError && disposedUnreadyFreshSandboxErrors.has(error);
|
|
5306
|
+
}
|
|
5307
|
+
var FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2e3;
|
|
5308
|
+
var FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8e3;
|
|
5309
|
+
function freshSandboxReadinessReplacementDelayMs(random = Math.random) {
|
|
5310
|
+
const sample = random();
|
|
5311
|
+
const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
|
|
5312
|
+
return FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS + Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS);
|
|
5313
|
+
}
|
|
5314
|
+
var FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
|
|
5315
|
+
function createFreshSandboxReadinessReplacementBudget() {
|
|
5316
|
+
return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
|
|
5317
|
+
}
|
|
5318
|
+
function recordFreshSandboxReadinessReplacement(metrics, backend, outcome) {
|
|
5319
|
+
try {
|
|
5320
|
+
metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
|
|
5321
|
+
} catch {
|
|
5322
|
+
}
|
|
5323
|
+
}
|
|
5324
|
+
async function sleepUnlessCancelled(ms, signal) {
|
|
5325
|
+
if (signal?.aborted) return false;
|
|
5326
|
+
if (ms <= 0) return true;
|
|
5327
|
+
return await new Promise((resolve) => {
|
|
5328
|
+
const onAbort = () => {
|
|
5329
|
+
clearTimeout(timer);
|
|
5330
|
+
resolve(false);
|
|
5331
|
+
};
|
|
5332
|
+
const timer = setTimeout(() => {
|
|
5333
|
+
signal?.removeEventListener("abort", onAbort);
|
|
5334
|
+
resolve(true);
|
|
5335
|
+
}, ms);
|
|
5336
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
5337
|
+
});
|
|
5338
|
+
}
|
|
5133
5339
|
var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
|
|
5134
5340
|
constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
|
|
5135
5341
|
super(sandboxGroupId, leaseEpoch);
|
|
@@ -5141,9 +5347,9 @@ var WARMING_POLL_INTERVAL_MS = 250;
|
|
|
5141
5347
|
async function sleep(ms) {
|
|
5142
5348
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
5143
5349
|
}
|
|
5144
|
-
async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}) {
|
|
5350
|
+
async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}, signal) {
|
|
5145
5351
|
try {
|
|
5146
|
-
await verifySandboxExecReadiness(established, timeoutMs);
|
|
5352
|
+
await verifySandboxExecReadiness(established, timeoutMs, signal);
|
|
5147
5353
|
} catch (error) {
|
|
5148
5354
|
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
5149
5355
|
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
@@ -5160,29 +5366,6 @@ var SnapshotTimeoutError = class extends Error {
|
|
|
5160
5366
|
this.name = "SnapshotTimeoutError";
|
|
5161
5367
|
}
|
|
5162
5368
|
};
|
|
5163
|
-
function safeSnapshotError(error) {
|
|
5164
|
-
const fields = {
|
|
5165
|
-
errorClass: "SnapshotOperationError",
|
|
5166
|
-
errorCode: "snapshot_operation_failed",
|
|
5167
|
-
origin: "sandbox-resume"
|
|
5168
|
-
};
|
|
5169
|
-
try {
|
|
5170
|
-
if (error && typeof error === "object") {
|
|
5171
|
-
const candidate = error;
|
|
5172
|
-
if (typeof candidate.name === "string" && /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)) {
|
|
5173
|
-
fields.causeName = candidate.name;
|
|
5174
|
-
}
|
|
5175
|
-
if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
|
|
5176
|
-
fields.integrityCode = candidate.code;
|
|
5177
|
-
}
|
|
5178
|
-
const rawStatus = candidate.status ?? candidate.statusCode;
|
|
5179
|
-
const status = Number(rawStatus);
|
|
5180
|
-
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
5181
|
-
}
|
|
5182
|
-
} catch {
|
|
5183
|
-
}
|
|
5184
|
-
return fields;
|
|
5185
|
-
}
|
|
5186
5369
|
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
5187
5370
|
if (signal?.aborted) return false;
|
|
5188
5371
|
let timeout;
|
|
@@ -5305,8 +5488,9 @@ async function materializeSpawnEnvelopeArchive(envelope, objectStorage) {
|
|
|
5305
5488
|
}
|
|
5306
5489
|
if (!objectStorage) {
|
|
5307
5490
|
throw new WorkspaceArchiveIntegrityError(
|
|
5308
|
-
"
|
|
5309
|
-
"workspace archive object storage is not configured"
|
|
5491
|
+
"archive_storage_unavailable",
|
|
5492
|
+
"workspace archive object storage is not configured",
|
|
5493
|
+
{ retryable: true }
|
|
5310
5494
|
);
|
|
5311
5495
|
}
|
|
5312
5496
|
const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
@@ -5361,6 +5545,18 @@ function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, s
|
|
|
5361
5545
|
}
|
|
5362
5546
|
async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force, registerOwnedContinuation) {
|
|
5363
5547
|
const { db, settings } = services;
|
|
5548
|
+
const warnSnapshotFailure = (message, error) => {
|
|
5549
|
+
const fields = {
|
|
5550
|
+
...safeSnapshotError(error),
|
|
5551
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey2(ids.workspaceId, ids.sandboxGroupId),
|
|
5552
|
+
leaseEpoch
|
|
5553
|
+
};
|
|
5554
|
+
try {
|
|
5555
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
5556
|
+
else console.warn(message, fields);
|
|
5557
|
+
} catch {
|
|
5558
|
+
}
|
|
5559
|
+
};
|
|
5364
5560
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
5365
5561
|
if (intervalMs <= 0 && !force) {
|
|
5366
5562
|
return false;
|
|
@@ -5434,6 +5630,7 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
|
|
|
5434
5630
|
} : {}
|
|
5435
5631
|
});
|
|
5436
5632
|
if (claimed.status !== "claimed") {
|
|
5633
|
+
if (claimed.status === "throttled") return false;
|
|
5437
5634
|
console.error("mid-session workspace snapshot skipped (capture claim)", {
|
|
5438
5635
|
sandboxGroupId: ids.sandboxGroupId,
|
|
5439
5636
|
status: claimed.status
|
|
@@ -5472,6 +5669,8 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
|
|
|
5472
5669
|
}).catch(() => void 0);
|
|
5473
5670
|
};
|
|
5474
5671
|
const captureAndPublish = (async () => {
|
|
5672
|
+
const captureStarted = performance.now();
|
|
5673
|
+
let captureOutcome = "failed";
|
|
5475
5674
|
let archive;
|
|
5476
5675
|
let candidate = null;
|
|
5477
5676
|
let publicationAttempted = false;
|
|
@@ -5521,6 +5720,7 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
|
|
|
5521
5720
|
if (!wrote && candidate) {
|
|
5522
5721
|
await abandonCandidate(candidate.id, "snapshot_publication_fenced");
|
|
5523
5722
|
}
|
|
5723
|
+
if (wrote) captureOutcome = "completed";
|
|
5524
5724
|
return wrote;
|
|
5525
5725
|
} catch (error) {
|
|
5526
5726
|
if (candidate && !publicationAttempted)
|
|
@@ -5538,20 +5738,23 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
|
|
|
5538
5738
|
expectedEpoch: leaseEpoch,
|
|
5539
5739
|
expectedInstanceId: instanceId
|
|
5540
5740
|
}).catch((error) => {
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
safeSnapshotError(error)
|
|
5544
|
-
);
|
|
5741
|
+
captureOutcome = "failed";
|
|
5742
|
+
warnSnapshotFailure("mid-session workspace capture gate release failed", error);
|
|
5545
5743
|
});
|
|
5744
|
+
try {
|
|
5745
|
+
services.sandboxMetrics?.onWorkspaceCapture?.({
|
|
5746
|
+
backend: lease.backend,
|
|
5747
|
+
outcome: captureOutcome,
|
|
5748
|
+
durationSeconds: Math.max(0, performance.now() - captureStarted) / 1e3
|
|
5749
|
+
});
|
|
5750
|
+
} catch {
|
|
5751
|
+
}
|
|
5546
5752
|
}
|
|
5547
5753
|
})();
|
|
5548
5754
|
const settled = captureAndPublish.then(
|
|
5549
5755
|
(persisted) => ({ kind: "settled", persisted }),
|
|
5550
5756
|
(error) => {
|
|
5551
|
-
|
|
5552
|
-
"mid-session workspace snapshot failed (turn unaffected)",
|
|
5553
|
-
safeSnapshotError(error)
|
|
5554
|
-
);
|
|
5757
|
+
warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
|
|
5555
5758
|
return { kind: "settled", persisted: false };
|
|
5556
5759
|
}
|
|
5557
5760
|
);
|
|
@@ -5578,14 +5781,74 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
|
|
|
5578
5781
|
});
|
|
5579
5782
|
return outcome.kind === "settled" ? outcome.persisted : false;
|
|
5580
5783
|
} catch (error) {
|
|
5581
|
-
|
|
5582
|
-
"mid-session workspace snapshot failed (turn unaffected)",
|
|
5583
|
-
safeSnapshotError(error)
|
|
5584
|
-
);
|
|
5784
|
+
warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
|
|
5585
5785
|
return false;
|
|
5586
5786
|
}
|
|
5587
5787
|
}
|
|
5588
5788
|
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
5789
|
+
try {
|
|
5790
|
+
return await resumeBoxForTurnOnce(services, ids, kind, holderId);
|
|
5791
|
+
} catch (error) {
|
|
5792
|
+
if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
|
|
5793
|
+
const readiness = error;
|
|
5794
|
+
const warn = (message, fields) => {
|
|
5795
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
5796
|
+
else console.warn(message, fields);
|
|
5797
|
+
};
|
|
5798
|
+
const identity = {
|
|
5799
|
+
workspaceId: ids.workspaceId,
|
|
5800
|
+
sessionId: ids.sessionId,
|
|
5801
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
5802
|
+
backend: readiness.backend,
|
|
5803
|
+
instanceId: readiness.instanceId,
|
|
5804
|
+
readinessTimeoutMs: readiness.timeoutMs
|
|
5805
|
+
};
|
|
5806
|
+
const budget = services.freshSandboxReadinessReplacementBudget ?? createFreshSandboxReadinessReplacementBudget();
|
|
5807
|
+
if (budget.remaining <= 0) {
|
|
5808
|
+
recordFreshSandboxReadinessReplacement(
|
|
5809
|
+
services.sandboxMetrics,
|
|
5810
|
+
readiness.backend,
|
|
5811
|
+
"budget_spent"
|
|
5812
|
+
);
|
|
5813
|
+
warn(
|
|
5814
|
+
"sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
|
|
5815
|
+
identity
|
|
5816
|
+
);
|
|
5817
|
+
throw error;
|
|
5818
|
+
}
|
|
5819
|
+
budget.remaining -= 1;
|
|
5820
|
+
const delayMs = await (services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs)();
|
|
5821
|
+
warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
|
|
5822
|
+
...identity,
|
|
5823
|
+
replacementDelayMs: delayMs
|
|
5824
|
+
});
|
|
5825
|
+
if (!await sleepUnlessCancelled(delayMs, services.cancellationSignal)) {
|
|
5826
|
+
recordFreshSandboxReadinessReplacement(
|
|
5827
|
+
services.sandboxMetrics,
|
|
5828
|
+
readiness.backend,
|
|
5829
|
+
"cancelled"
|
|
5830
|
+
);
|
|
5831
|
+
throw error;
|
|
5832
|
+
}
|
|
5833
|
+
try {
|
|
5834
|
+
const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
|
|
5835
|
+
recordFreshSandboxReadinessReplacement(
|
|
5836
|
+
services.sandboxMetrics,
|
|
5837
|
+
readiness.backend,
|
|
5838
|
+
"replaced"
|
|
5839
|
+
);
|
|
5840
|
+
return replaced;
|
|
5841
|
+
} catch (replacementError) {
|
|
5842
|
+
recordFreshSandboxReadinessReplacement(
|
|
5843
|
+
services.sandboxMetrics,
|
|
5844
|
+
readiness.backend,
|
|
5845
|
+
services.cancellationSignal?.aborted ? "cancelled" : replacementError instanceof SandboxExecReadinessTimeoutError ? "failed_again" : "replacement_failed"
|
|
5846
|
+
);
|
|
5847
|
+
throw replacementError;
|
|
5848
|
+
}
|
|
5849
|
+
}
|
|
5850
|
+
}
|
|
5851
|
+
async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
|
|
5589
5852
|
const { db, settings } = services;
|
|
5590
5853
|
const os = ids.os ?? "linux";
|
|
5591
5854
|
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
@@ -5658,13 +5921,18 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5658
5921
|
holderId,
|
|
5659
5922
|
subjectId: ids.sessionId,
|
|
5660
5923
|
backend: ids.backend,
|
|
5924
|
+
warmBilling: {
|
|
5925
|
+
mode: settings.sandboxWarmBillingMode,
|
|
5926
|
+
rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, ids.backend)
|
|
5927
|
+
},
|
|
5661
5928
|
os,
|
|
5662
5929
|
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
5663
|
-
// conflicts on a live box already running a different image. A
|
|
5930
|
+
// conflicts on a live box already running a different required image. A
|
|
5664
5931
|
// SandboxImageConflictError propagates while another holder is active; a
|
|
5665
5932
|
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
5666
5933
|
// after the cold successor can safely stamp the new image.
|
|
5667
5934
|
...ids.image ? { image: ids.image } : {},
|
|
5935
|
+
...ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {},
|
|
5668
5936
|
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
5669
5937
|
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
5670
5938
|
// propagates while another holder is active; a solo change uses the same
|
|
@@ -5714,7 +5982,12 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5714
5982
|
});
|
|
5715
5983
|
await providerRenewalInFlight;
|
|
5716
5984
|
};
|
|
5717
|
-
const releaseDeadHolder = (reason) => {
|
|
5985
|
+
const releaseDeadHolder = async (reason) => {
|
|
5986
|
+
if (reason === "holder_gone" && kind === "turn" && holderLeaseHeartbeat) {
|
|
5987
|
+
const expectedEpoch = holderLeaseHeartbeat.expectedEpoch;
|
|
5988
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
5989
|
+
if (retainsHolderForWarmCapture(lease, expectedEpoch)) return;
|
|
5990
|
+
}
|
|
5718
5991
|
const message = "sandbox resume holder released: holder no longer live";
|
|
5719
5992
|
const fields = {
|
|
5720
5993
|
workspaceId: ids.workspaceId,
|
|
@@ -5737,15 +6010,16 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5737
6010
|
kind,
|
|
5738
6011
|
holderId,
|
|
5739
6012
|
expectedEpoch: heartbeat2.expectedEpoch,
|
|
5740
|
-
leaseTtlMs: heartbeat2.leaseTtlMs
|
|
5741
|
-
|
|
6013
|
+
leaseTtlMs: heartbeat2.leaseTtlMs,
|
|
6014
|
+
billingMode: services.settings.sandboxWarmBillingMode
|
|
6015
|
+
}).then(async (status) => {
|
|
5742
6016
|
if (heartbeat2 !== holderLeaseHeartbeat) return;
|
|
5743
6017
|
if (!status.holderAlive) {
|
|
5744
|
-
releaseDeadHolder(status.fence);
|
|
6018
|
+
await releaseDeadHolder(status.fence);
|
|
5745
6019
|
return;
|
|
5746
6020
|
}
|
|
5747
6021
|
if (status.fence === "epoch") {
|
|
5748
|
-
releaseDeadHolder(status.fence);
|
|
6022
|
+
await releaseDeadHolder(status.fence);
|
|
5749
6023
|
return;
|
|
5750
6024
|
}
|
|
5751
6025
|
if (status.leaseExtended || status.fence === "rotation_requested") {
|
|
@@ -5760,10 +6034,10 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5760
6034
|
sandboxGroupId: ids.sandboxGroupId,
|
|
5761
6035
|
kind,
|
|
5762
6036
|
holderId
|
|
5763
|
-
}).then((touched) => {
|
|
6037
|
+
}).then(async (touched) => {
|
|
5764
6038
|
if (heartbeat2 !== holderLeaseHeartbeat) return;
|
|
5765
6039
|
if (!touched) {
|
|
5766
|
-
releaseDeadHolder("holder_gone");
|
|
6040
|
+
await releaseDeadHolder("holder_gone");
|
|
5767
6041
|
return;
|
|
5768
6042
|
}
|
|
5769
6043
|
void maybeRenewProviderExpiration().catch(() => void 0);
|
|
@@ -5802,14 +6076,18 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5802
6076
|
if (acquired.role === "spawner") {
|
|
5803
6077
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
5804
6078
|
let createdEstablished = null;
|
|
6079
|
+
let providerCreateOperationId;
|
|
6080
|
+
let providerCreateBindingKey;
|
|
5805
6081
|
let rematerialization = null;
|
|
5806
6082
|
try {
|
|
5807
|
-
const
|
|
5808
|
-
const
|
|
5809
|
-
|
|
6083
|
+
const freshWorkspaceRecoveryId = acquired.lease.freshWorkspaceRecoveryId ?? null;
|
|
6084
|
+
const envelope = freshWorkspaceRecoveryId ? null : await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
6085
|
+
const legacyFallbackRefused = !freshWorkspaceRecoveryId && acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null && await sessionHoldsFreshWorkspaceRecovery(db, ids.workspaceId, ids.sessionId);
|
|
6086
|
+
const fallbackArchiveEnvelope = !freshWorkspaceRecoveryId && !legacyFallbackRefused && acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
|
|
6087
|
+
let spawnEnvelope = freshWorkspaceRecoveryId ? null : legacyFallbackRefused ? acquired.lease.resumeState ?? null : fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
5810
6088
|
const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
|
|
5811
6089
|
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
5812
|
-
if (acquired.lease.recovery.archive.status === "available" && acquired.lease.archiveComplete || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
|
|
6090
|
+
if (!freshWorkspaceRecoveryId && (acquired.lease.recovery.archive.status === "available" && (acquired.lease.archiveComplete || acquired.lease.historicalRecoveryAuthorized === true) || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)) {
|
|
5813
6091
|
const rematerializationId = crypto.randomUUID();
|
|
5814
6092
|
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
5815
6093
|
const begun = await beginSandboxRematerialization(db, {
|
|
@@ -5851,7 +6129,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5851
6129
|
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
5852
6130
|
legacyProviderBinding: null
|
|
5853
6131
|
};
|
|
5854
|
-
} else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
6132
|
+
} else if (!freshWorkspaceRecoveryId && acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
5855
6133
|
throw new SandboxLeaseRecoveryBlockedError(
|
|
5856
6134
|
ids.sandboxGroupId,
|
|
5857
6135
|
expectedEpoch,
|
|
@@ -5891,15 +6169,49 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5891
6169
|
...ids.environment ? { environment: ids.environment } : {},
|
|
5892
6170
|
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
5893
6171
|
...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
|
|
5898
|
-
instanceId: created.instanceId
|
|
5899
|
-
};
|
|
5900
|
-
providerRenewedAtMs = Date.now();
|
|
6172
|
+
onBeforeSandboxCreate: async (createSettings, intent, providerContext) => {
|
|
6173
|
+
if (ids.backend !== "modal") return;
|
|
6174
|
+
if (!intent || !providerContext)
|
|
6175
|
+
throw new Error("Modal create requires the provider dispatch boundary");
|
|
5901
6176
|
throwIfReleasedOrCancelled();
|
|
5902
|
-
|
|
6177
|
+
const binding = await resolveModalCheckpointProviderBindingForSession(
|
|
6178
|
+
createSettings,
|
|
6179
|
+
providerContext
|
|
6180
|
+
);
|
|
6181
|
+
const operationId = intent.operationId;
|
|
6182
|
+
await beginModalProviderCreate(db, {
|
|
6183
|
+
accountId: ids.accountId,
|
|
6184
|
+
workspaceId: ids.workspaceId,
|
|
6185
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
6186
|
+
expectedEpoch,
|
|
6187
|
+
operationId,
|
|
6188
|
+
providerBindingKey: binding.key,
|
|
6189
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
6190
|
+
selectedRevision: rematerialization?.selectedRevision ?? null,
|
|
6191
|
+
imageId: intent.imageId,
|
|
6192
|
+
imageRef: createSettings.modalImageRef ?? null,
|
|
6193
|
+
appId: intent.appId,
|
|
6194
|
+
providerName: intent.name,
|
|
6195
|
+
requestSha256: intent.requestSha256
|
|
6196
|
+
});
|
|
6197
|
+
providerCreateOperationId = operationId;
|
|
6198
|
+
providerCreateBindingKey = binding.key;
|
|
6199
|
+
},
|
|
6200
|
+
onSandboxCreated: async (created) => {
|
|
6201
|
+
createdEstablished = created;
|
|
6202
|
+
providerRenewalTarget = {
|
|
6203
|
+
backend: ids.backend,
|
|
6204
|
+
instanceId: created.instanceId
|
|
6205
|
+
};
|
|
6206
|
+
providerRenewedAtMs = Date.now();
|
|
6207
|
+
if (providerCreateBindingKey && !await modalSessionMatchesCheckpointProviderBinding(
|
|
6208
|
+
settings,
|
|
6209
|
+
created.session,
|
|
6210
|
+
providerCreateBindingKey
|
|
6211
|
+
)) {
|
|
6212
|
+
throw new Error("Modal creation receipt crossed the fenced provider namespace");
|
|
6213
|
+
}
|
|
6214
|
+
if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
|
|
5903
6215
|
if (created.backendId !== "modal") {
|
|
5904
6216
|
throw new WorkspaceArchiveIntegrityError(
|
|
5905
6217
|
"native_snapshot_reference_invalid",
|
|
@@ -5941,6 +6253,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
5941
6253
|
sandboxGroupId: ids.sandboxGroupId,
|
|
5942
6254
|
expectedEpoch,
|
|
5943
6255
|
rematerializationId: rematerialization?.id ?? null,
|
|
6256
|
+
...providerCreateOperationId ? { providerCreateOperationId } : {},
|
|
5944
6257
|
...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
|
|
5945
6258
|
instanceId: created.instanceId,
|
|
5946
6259
|
resumeBackendId: created.backendId,
|
|
@@ -6016,9 +6329,20 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
6016
6329
|
});
|
|
6017
6330
|
createdEstablished = established;
|
|
6018
6331
|
throwIfReleasedOrCancelled();
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6332
|
+
if (services.verifySpawnedSandboxReadiness) {
|
|
6333
|
+
await services.verifySpawnedSandboxReadiness(established, {
|
|
6334
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
6335
|
+
});
|
|
6336
|
+
} else {
|
|
6337
|
+
await waitForSandboxExecReadiness(
|
|
6338
|
+
established,
|
|
6339
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
6340
|
+
{
|
|
6341
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
6342
|
+
},
|
|
6343
|
+
cancellationSignal
|
|
6344
|
+
);
|
|
6345
|
+
}
|
|
6022
6346
|
await maybeRenewProviderExpiration(true);
|
|
6023
6347
|
throwIfReleasedOrCancelled();
|
|
6024
6348
|
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
@@ -6043,7 +6367,7 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
6043
6367
|
dataPlaneUrl: null,
|
|
6044
6368
|
resumeBackendId: established.backendId,
|
|
6045
6369
|
resumeState: resumeEnvelope,
|
|
6046
|
-
...established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
|
|
6370
|
+
...freshWorkspaceRecoveryId ? { freshWorkspace: { operationId: freshWorkspaceRecoveryId } } : established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
|
|
6047
6371
|
rematerialization: {
|
|
6048
6372
|
id: rematerialization.id,
|
|
6049
6373
|
verifiedRevision: rematerialization.selectedRevision
|
|
@@ -6112,6 +6436,9 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
6112
6436
|
}
|
|
6113
6437
|
await release();
|
|
6114
6438
|
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
6439
|
+
if (terminated && createdEstablished && error instanceof SandboxExecReadinessTimeoutError && error.instanceId === createdEstablished.instanceId && (createdEstablished.origin === "created" || createdEstablished.origin === "restored") && !createdEstablished.providerContinuity) {
|
|
6440
|
+
disposedUnreadyFreshSandboxErrors.add(error);
|
|
6441
|
+
}
|
|
6115
6442
|
throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
|
|
6116
6443
|
}
|
|
6117
6444
|
}
|
|
@@ -6140,9 +6467,14 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
6140
6467
|
if (services.verifyAttachedSandboxReadiness) {
|
|
6141
6468
|
await services.verifyAttachedSandboxReadiness(established);
|
|
6142
6469
|
} else {
|
|
6143
|
-
await waitForSandboxExecReadiness(
|
|
6144
|
-
|
|
6145
|
-
|
|
6470
|
+
await waitForSandboxExecReadiness(
|
|
6471
|
+
established,
|
|
6472
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
6473
|
+
{
|
|
6474
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
6475
|
+
},
|
|
6476
|
+
cancellationSignal
|
|
6477
|
+
);
|
|
6146
6478
|
}
|
|
6147
6479
|
providerRenewalTarget = {
|
|
6148
6480
|
backend: ids.backend,
|
|
@@ -6159,9 +6491,15 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
6159
6491
|
workspaceId: ids.workspaceId,
|
|
6160
6492
|
sandboxGroupId: ids.sandboxGroupId,
|
|
6161
6493
|
expectedEpoch: leaseEpoch,
|
|
6162
|
-
expectedInstanceId: live.instanceId
|
|
6494
|
+
expectedInstanceId: live.instanceId,
|
|
6495
|
+
expectedBackend: ids.backend
|
|
6163
6496
|
});
|
|
6164
6497
|
if (marked.status === "marked") {
|
|
6498
|
+
await publishDurableSessionEvents2(
|
|
6499
|
+
services.bus,
|
|
6500
|
+
ids.workspaceId,
|
|
6501
|
+
marked.backgroundCommandEvents
|
|
6502
|
+
);
|
|
6165
6503
|
await services.onSandboxLost?.({
|
|
6166
6504
|
sandboxGroupId: ids.sandboxGroupId,
|
|
6167
6505
|
instanceId: live.instanceId,
|
|
@@ -6289,7 +6627,7 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
6289
6627
|
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
6290
6628
|
}
|
|
6291
6629
|
if (lease.instanceId === established.instanceId) {
|
|
6292
|
-
services.onHomeSandboxRebound?.({
|
|
6630
|
+
await services.onHomeSandboxRebound?.({
|
|
6293
6631
|
established,
|
|
6294
6632
|
leaseEpoch: lease.leaseEpoch
|
|
6295
6633
|
});
|
|
@@ -6329,9 +6667,15 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
6329
6667
|
sandboxGroupId: ids.sandboxGroupId,
|
|
6330
6668
|
expectedEpoch: lease.leaseEpoch,
|
|
6331
6669
|
expectedInstanceId: lease.instanceId,
|
|
6670
|
+
expectedBackend: lease.backend,
|
|
6332
6671
|
diagnostic: "provider_not_found_during_home_route_rebind"
|
|
6333
6672
|
});
|
|
6334
6673
|
if (marked.status === "marked") {
|
|
6674
|
+
await publishDurableSessionEvents2(
|
|
6675
|
+
services.bus,
|
|
6676
|
+
ids.workspaceId,
|
|
6677
|
+
marked.backgroundCommandEvents
|
|
6678
|
+
);
|
|
6335
6679
|
await services.onHomeSandboxLost?.({
|
|
6336
6680
|
sandboxGroupId: ids.sandboxGroupId,
|
|
6337
6681
|
instanceId: lease.instanceId,
|
|
@@ -6348,7 +6692,7 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
6348
6692
|
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
6349
6693
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
6350
6694
|
}
|
|
6351
|
-
services.onHomeSandboxRebound?.({
|
|
6695
|
+
await services.onHomeSandboxRebound?.({
|
|
6352
6696
|
established: rebound,
|
|
6353
6697
|
leaseEpoch: lease.leaseEpoch
|
|
6354
6698
|
});
|
|
@@ -6400,7 +6744,7 @@ function afterPersistableHomeMutation(services, ids, home) {
|
|
|
6400
6744
|
if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
|
|
6401
6745
|
throw new Error("Persistable home mutation settlement lacked its bound admission");
|
|
6402
6746
|
}
|
|
6403
|
-
if (outcome === "resolved" && retainedProcess) {
|
|
6747
|
+
if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
|
|
6404
6748
|
try {
|
|
6405
6749
|
await retainWorkspaceProviderCommand(services.db, {
|
|
6406
6750
|
accountId: fence.accountId,
|
|
@@ -6445,6 +6789,8 @@ function afterPersistableHomeMutation(services, ids, home) {
|
|
|
6445
6789
|
}
|
|
6446
6790
|
return;
|
|
6447
6791
|
}
|
|
6792
|
+
if (outcome === "outcome_unknown")
|
|
6793
|
+
throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
|
|
6448
6794
|
await verifyWorkspaceMutationSettlement(services.db, {
|
|
6449
6795
|
accountId: fence.accountId,
|
|
6450
6796
|
workspaceId: ids.workspaceId,
|
|
@@ -6553,7 +6899,7 @@ function settleRetainedProcessForTurn(services, ids) {
|
|
|
6553
6899
|
sessionId: ids.sessionId,
|
|
6554
6900
|
processId: process2.id
|
|
6555
6901
|
});
|
|
6556
|
-
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
6902
|
+
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
6557
6903
|
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
6558
6904
|
}
|
|
6559
6905
|
const settlement = await settleRetainedProcess(services.db, {
|
|
@@ -6575,16 +6921,43 @@ function settleRetainedProcessForTurn(services, ids) {
|
|
|
6575
6921
|
}
|
|
6576
6922
|
};
|
|
6577
6923
|
}
|
|
6924
|
+
function isRetainedProcessSettledForTurn(services, ids) {
|
|
6925
|
+
const fence = ids.workspaceMutationFence;
|
|
6926
|
+
if (!fence) return void 0;
|
|
6927
|
+
return async ({
|
|
6928
|
+
backend,
|
|
6929
|
+
process: process2
|
|
6930
|
+
}) => {
|
|
6931
|
+
const durable = await getRetainedProcess(services.db, {
|
|
6932
|
+
workspaceId: ids.workspaceId,
|
|
6933
|
+
sessionId: ids.sessionId,
|
|
6934
|
+
processId: process2.id
|
|
6935
|
+
});
|
|
6936
|
+
if (!durable || durable.state === "active" || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch)
|
|
6937
|
+
return false;
|
|
6938
|
+
if (process2.providerCommand) {
|
|
6939
|
+
const command = await retainedProviderCommandPersistence(services.db, {
|
|
6940
|
+
accountId: fence.accountId,
|
|
6941
|
+
workspaceId: ids.workspaceId,
|
|
6942
|
+
sessionId: ids.sessionId,
|
|
6943
|
+
processId: process2.id
|
|
6944
|
+
}).load();
|
|
6945
|
+
if (!command || command.kind !== process2.providerCommand.kind || command.execId !== process2.providerCommand.execId || command.taskId !== process2.providerCommand.taskId || command.sandboxId !== process2.providerCommand.sandboxId || Boolean(command.pty) !== Boolean(process2.providerCommand.pty) || command.kind === "modal-router-v1" && process2.providerCommand.kind === "modal-router-v1" && !isDeepStrictEqual3(command.supervision, process2.providerCommand.supervision))
|
|
6946
|
+
return false;
|
|
6947
|
+
}
|
|
6948
|
+
return true;
|
|
6949
|
+
};
|
|
6950
|
+
}
|
|
6578
6951
|
function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
|
|
6579
6952
|
const fence = ids.workspaceMutationFence;
|
|
6580
6953
|
if (!fence) return void 0;
|
|
6581
|
-
return async ({ backend, process: process2 }) => {
|
|
6954
|
+
return async ({ backend, process: process2, command }) => {
|
|
6582
6955
|
const durable = await getRetainedProcess(services.db, {
|
|
6583
6956
|
workspaceId: ids.workspaceId,
|
|
6584
6957
|
sessionId: ids.sessionId,
|
|
6585
6958
|
processId: process2.id
|
|
6586
6959
|
});
|
|
6587
|
-
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
6960
|
+
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
6588
6961
|
throw new Error("Background command adoption lost its exact retained-process identity");
|
|
6589
6962
|
}
|
|
6590
6963
|
await adoptManagedSessionBackgroundCommand(services.db, {
|
|
@@ -6596,7 +6969,7 @@ function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
|
|
|
6596
6969
|
attemptId: fence.attemptId,
|
|
6597
6970
|
processId: process2.id,
|
|
6598
6971
|
expected: retainedProcessSettlementIdentity(durable),
|
|
6599
|
-
command: "
|
|
6972
|
+
command: command ?? "Background command"
|
|
6600
6973
|
});
|
|
6601
6974
|
};
|
|
6602
6975
|
}
|
|
@@ -6617,6 +6990,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
6617
6990
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
6618
6991
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
6619
6992
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
6993
|
+
const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
|
|
6620
6994
|
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
6621
6995
|
services,
|
|
6622
6996
|
ids
|
|
@@ -6769,20 +7143,27 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
6769
7143
|
},
|
|
6770
7144
|
resolveActiveBackend: resolver,
|
|
6771
7145
|
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
7146
|
+
...services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {},
|
|
6772
7147
|
providerCommandHandle: admittedCommandHandle,
|
|
7148
|
+
providerSupervisionReady: async () => settings.modalCommandSupervisionEnabled && await supervisedCommandProtocolReady(db),
|
|
6773
7149
|
...ids.workspaceMutationFence ? {
|
|
6774
|
-
providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
7150
|
+
providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
|
|
7151
|
+
db,
|
|
7152
|
+
{
|
|
7153
|
+
accountId: ids.workspaceMutationFence.accountId,
|
|
7154
|
+
workspaceId: ids.workspaceId,
|
|
7155
|
+
sessionId: ids.sessionId,
|
|
7156
|
+
processId: process2.id
|
|
7157
|
+
},
|
|
7158
|
+
services.bus ? (events) => services.bus.publish(ids.workspaceId, ids.sessionId, events) : void 0
|
|
7159
|
+
)
|
|
6780
7160
|
} : {},
|
|
6781
7161
|
...beforeMutation ? { beforeMutation } : {},
|
|
6782
7162
|
...afterMutation ? { afterMutation } : {},
|
|
6783
7163
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
6784
7164
|
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
6785
7165
|
...settleProcess ? { settleProcess } : {},
|
|
7166
|
+
...isProcessSettled ? { isProcessSettled } : {},
|
|
6786
7167
|
...ids.workspaceMutationFence ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids) } : {},
|
|
6787
7168
|
...ids.workspaceMutationFence ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids) } : {},
|
|
6788
7169
|
...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
|
|
@@ -6801,9 +7182,15 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
6801
7182
|
sandboxGroupId: home.sandboxGroupId,
|
|
6802
7183
|
expectedEpoch,
|
|
6803
7184
|
expectedInstanceId,
|
|
7185
|
+
expectedBackend: home.backend,
|
|
6804
7186
|
diagnostic: "provider_not_found_during_routed_operation"
|
|
6805
7187
|
});
|
|
6806
7188
|
if (marked.status === "marked") {
|
|
7189
|
+
await publishDurableSessionEvents2(
|
|
7190
|
+
services.bus,
|
|
7191
|
+
ids.workspaceId,
|
|
7192
|
+
marked.backgroundCommandEvents
|
|
7193
|
+
);
|
|
6807
7194
|
await services.onHomeSandboxLost?.({
|
|
6808
7195
|
sandboxGroupId: home.sandboxGroupId,
|
|
6809
7196
|
instanceId: expectedInstanceId,
|
|
@@ -6830,6 +7217,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
6830
7217
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
6831
7218
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
6832
7219
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
7220
|
+
const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
|
|
6833
7221
|
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
6834
7222
|
services,
|
|
6835
7223
|
ids
|
|
@@ -6965,21 +7353,28 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
6965
7353
|
return routedResolver(pointer);
|
|
6966
7354
|
},
|
|
6967
7355
|
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
7356
|
+
...services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {},
|
|
6968
7357
|
...args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {},
|
|
6969
7358
|
providerCommandHandle: admittedCommandHandle,
|
|
7359
|
+
providerSupervisionReady: async () => settings.modalCommandSupervisionEnabled && await supervisedCommandProtocolReady(db),
|
|
6970
7360
|
...ids.workspaceMutationFence ? {
|
|
6971
|
-
providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
|
|
6972
|
-
|
|
6973
|
-
|
|
6974
|
-
|
|
6975
|
-
|
|
6976
|
-
|
|
7361
|
+
providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
|
|
7362
|
+
db,
|
|
7363
|
+
{
|
|
7364
|
+
accountId: ids.workspaceMutationFence.accountId,
|
|
7365
|
+
workspaceId: ids.workspaceId,
|
|
7366
|
+
sessionId: ids.sessionId,
|
|
7367
|
+
processId: process2.id
|
|
7368
|
+
},
|
|
7369
|
+
services.bus ? (events) => services.bus.publish(ids.workspaceId, ids.sessionId, events) : void 0
|
|
7370
|
+
)
|
|
6977
7371
|
} : {},
|
|
6978
7372
|
...beforeMutation ? { beforeMutation } : {},
|
|
6979
7373
|
...afterMutation ? { afterMutation } : {},
|
|
6980
7374
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
6981
7375
|
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
6982
7376
|
...settleProcess ? { settleProcess } : {},
|
|
7377
|
+
...isProcessSettled ? { isProcessSettled } : {},
|
|
6983
7378
|
...ids.workspaceMutationFence ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids) } : {},
|
|
6984
7379
|
...ids.workspaceMutationFence ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids) } : {},
|
|
6985
7380
|
...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
|
|
@@ -6999,9 +7394,15 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
6999
7394
|
sandboxGroupId: home.sandboxGroupId,
|
|
7000
7395
|
expectedEpoch: backend.leaseEpoch,
|
|
7001
7396
|
expectedInstanceId: backend.providerInstanceId,
|
|
7397
|
+
expectedBackend: home.backend,
|
|
7002
7398
|
diagnostic: "provider_not_found_during_routed_operation"
|
|
7003
7399
|
});
|
|
7004
7400
|
if (marked.status === "marked") {
|
|
7401
|
+
await publishDurableSessionEvents2(
|
|
7402
|
+
services.bus,
|
|
7403
|
+
ids.workspaceId,
|
|
7404
|
+
marked.backgroundCommandEvents
|
|
7405
|
+
);
|
|
7005
7406
|
await services.onHomeSandboxLost?.({
|
|
7006
7407
|
sandboxGroupId: home.sandboxGroupId,
|
|
7007
7408
|
instanceId: backend.providerInstanceId,
|
|
@@ -7171,6 +7572,20 @@ async function establishSelfhostedTurnSession(services, args) {
|
|
|
7171
7572
|
reason: command.reason,
|
|
7172
7573
|
...command.failure ? { failure: command.failure } : {}
|
|
7173
7574
|
});
|
|
7575
|
+
if (command.outputReceipt && command.outcome === "exited" && command.exitCode !== null) {
|
|
7576
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
7577
|
+
accountId: args.accountId,
|
|
7578
|
+
workspaceId: args.workspaceId,
|
|
7579
|
+
sessionId: args.sessionId,
|
|
7580
|
+
commandId: command.commandId,
|
|
7581
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
7582
|
+
enrollmentId: command.enrollmentId,
|
|
7583
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
7584
|
+
opId: command.opId,
|
|
7585
|
+
receipt: command.outputReceipt,
|
|
7586
|
+
exitCode: command.exitCode
|
|
7587
|
+
});
|
|
7588
|
+
}
|
|
7174
7589
|
if (settlement && bus) {
|
|
7175
7590
|
await bus.publish(args.workspaceId, args.sessionId, settlement.events).catch(() => void 0);
|
|
7176
7591
|
}
|
|
@@ -7198,14 +7613,17 @@ function lazyProvisionEnabled(settings) {
|
|
|
7198
7613
|
}
|
|
7199
7614
|
|
|
7200
7615
|
// src/activities/parent-wake.ts
|
|
7616
|
+
import { getSessionParentClaudeProviderAccountAuthority } from "@opengeni/db";
|
|
7201
7617
|
import {
|
|
7202
7618
|
getSession,
|
|
7203
7619
|
getSessionGoal as getSessionGoal2,
|
|
7204
7620
|
getSessionParentPersonalConnectionDelegations,
|
|
7621
|
+
getSessionParentMcpAccountBindings,
|
|
7205
7622
|
getSessionParentXaiProviderAccountAuthority,
|
|
7206
7623
|
addSessionSystemUpdateWithSourceMutation,
|
|
7207
7624
|
claimPendingSessionSystemUpdateOutbox,
|
|
7208
7625
|
claimPendingSessionWorkflowWakes,
|
|
7626
|
+
repairPendingChildTerminalResultWakes,
|
|
7209
7627
|
childRequiresActionDedupeKey,
|
|
7210
7628
|
getSessionSystemUpdateOutboxByDedupeKey,
|
|
7211
7629
|
getOrCreateSessionSystemUpdateOutbox,
|
|
@@ -7218,6 +7636,7 @@ import {
|
|
|
7218
7636
|
sessionSystemUpdateOutboxKindPayload
|
|
7219
7637
|
} from "@opengeni/db";
|
|
7220
7638
|
import { requireSessionEventDurableFanoutCapability } from "@opengeni/events";
|
|
7639
|
+
var childTerminalRepairCursors = /* @__PURE__ */ new WeakMap();
|
|
7221
7640
|
async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episodeKey) {
|
|
7222
7641
|
try {
|
|
7223
7642
|
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
@@ -7232,11 +7651,21 @@ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episode
|
|
|
7232
7651
|
workspaceId,
|
|
7233
7652
|
childSessionId
|
|
7234
7653
|
);
|
|
7654
|
+
const mcpAccountBindings = await getSessionParentMcpAccountBindings(
|
|
7655
|
+
svc.db,
|
|
7656
|
+
workspaceId,
|
|
7657
|
+
childSessionId
|
|
7658
|
+
);
|
|
7235
7659
|
const xaiAuthority = await getSessionParentXaiProviderAccountAuthority(
|
|
7236
7660
|
svc.db,
|
|
7237
7661
|
workspaceId,
|
|
7238
7662
|
childSessionId
|
|
7239
7663
|
);
|
|
7664
|
+
const claudeAuthority = await getSessionParentClaudeProviderAccountAuthority(
|
|
7665
|
+
svc.db,
|
|
7666
|
+
workspaceId,
|
|
7667
|
+
childSessionId
|
|
7668
|
+
);
|
|
7240
7669
|
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
7241
7670
|
accountId: child.accountId,
|
|
7242
7671
|
workspaceId,
|
|
@@ -7249,12 +7678,15 @@ async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episode
|
|
|
7249
7678
|
summary: childCompletionSummary(child, goal, "idle"),
|
|
7250
7679
|
payload,
|
|
7251
7680
|
lineage: {
|
|
7681
|
+
...claudeAuthority.subjectId ? { claudeAuthoritySubjectId: claudeAuthority.subjectId } : {},
|
|
7252
7682
|
childSessionId: child.id,
|
|
7253
7683
|
parentSessionId: child.parentSessionId,
|
|
7254
7684
|
...xaiAuthority.subjectId ? { xaiAuthoritySubjectId: xaiAuthority.subjectId } : {}
|
|
7255
7685
|
},
|
|
7256
7686
|
personalConnectionDelegations,
|
|
7257
|
-
|
|
7687
|
+
mcpAccountBindings,
|
|
7688
|
+
xaiProviderAccountAuthoritySnapshot: xaiAuthority.snapshot,
|
|
7689
|
+
claudeProviderAccountAuthoritySnapshot: claudeAuthority.snapshot
|
|
7258
7690
|
});
|
|
7259
7691
|
if (outbox.status === "delivered") {
|
|
7260
7692
|
return;
|
|
@@ -7329,7 +7761,9 @@ async function deliverParentSystemUpdateOutbox(svc, outbox) {
|
|
|
7329
7761
|
summary: outbox.summary,
|
|
7330
7762
|
lineage: outbox.lineage,
|
|
7331
7763
|
personalConnectionDelegations: outbox.personalConnectionDelegations,
|
|
7332
|
-
|
|
7764
|
+
mcpAccountBindings: outbox.mcpAccountBindings,
|
|
7765
|
+
xaiProviderAccountAuthoritySnapshot: outbox.xaiProviderAccountAuthoritySnapshot,
|
|
7766
|
+
claudeProviderAccountAuthoritySnapshot: outbox.claudeProviderAccountAuthoritySnapshot
|
|
7333
7767
|
},
|
|
7334
7768
|
async (tx) => {
|
|
7335
7769
|
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
@@ -7432,6 +7866,21 @@ async function reconcilePendingSessionWorkflowWakes(svc, limit = 1e3, overrides
|
|
|
7432
7866
|
pendingAdmissionBlockers: {}
|
|
7433
7867
|
};
|
|
7434
7868
|
}
|
|
7869
|
+
const repairChildren = overrides.repairPendingChildTerminalResultWakes ?? repairPendingChildTerminalResultWakes;
|
|
7870
|
+
const inventory = await repairChildren(
|
|
7871
|
+
svc.db,
|
|
7872
|
+
Math.min(100, limit),
|
|
7873
|
+
childTerminalRepairCursors.get(svc.db) ?? null
|
|
7874
|
+
);
|
|
7875
|
+
if (inventory.cursor) childTerminalRepairCursors.set(svc.db, inventory.cursor);
|
|
7876
|
+
else childTerminalRepairCursors.delete(svc.db);
|
|
7877
|
+
if (inventory.failed > 0) {
|
|
7878
|
+
svc.observability.error("Pending child-result wake repair failed", {
|
|
7879
|
+
examined: inventory.examined,
|
|
7880
|
+
registered: inventory.registered,
|
|
7881
|
+
failed: inventory.failed
|
|
7882
|
+
});
|
|
7883
|
+
}
|
|
7435
7884
|
const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
|
|
7436
7885
|
let signaled = 0;
|
|
7437
7886
|
let pendingAdmission = 0;
|
|
@@ -7562,7 +8011,7 @@ import {
|
|
|
7562
8011
|
stableJson
|
|
7563
8012
|
} from "@opengeni/contracts";
|
|
7564
8013
|
import { createHash as createHash3 } from "crypto";
|
|
7565
|
-
import { isDeepStrictEqual as
|
|
8014
|
+
import { isDeepStrictEqual as isDeepStrictEqual4 } from "util";
|
|
7566
8015
|
var INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY = "incidentTelemetryAuthorityFence";
|
|
7567
8016
|
function incidentTelemetryPreflightDeclaration(agentConfig, incidentTriggered) {
|
|
7568
8017
|
const rawConfig = agentConfig;
|
|
@@ -7595,7 +8044,7 @@ function evaluateIncidentTelemetryPreflight(input) {
|
|
|
7595
8044
|
return blocked("incident_preflight_metadata_missing");
|
|
7596
8045
|
}
|
|
7597
8046
|
if (preflight.requiredResources.some(
|
|
7598
|
-
(required) => !input.responder.resources.some((selected) =>
|
|
8047
|
+
(required) => !input.responder.resources.some((selected) => isDeepStrictEqual4(selected, required))
|
|
7599
8048
|
)) {
|
|
7600
8049
|
return blocked("incident_responder_under_capable");
|
|
7601
8050
|
}
|
|
@@ -7971,6 +8420,150 @@ async function validateIncidentTelemetrySystemUpdateAuthority(input) {
|
|
|
7971
8420
|
return current && current.toolPolicyVersion === fence.toolPolicyVersion && current.responderDigest === fence.responderDigest && current.taskDigest === fence.taskDigest && current.alertSelectorDigest === fence.alertSelectorDigest ? { action: "accept" } : await reject();
|
|
7972
8421
|
}
|
|
7973
8422
|
|
|
8423
|
+
// src/activities/agent-turn/provider-dispatch-barrier.ts
|
|
8424
|
+
var SandboxCapabilitiesChangedError = class extends Error {
|
|
8425
|
+
code = "native_capabilities_changed_this_attempt";
|
|
8426
|
+
constructor() {
|
|
8427
|
+
super("The attached sandbox requires rebuilding this attempt's native capabilities.");
|
|
8428
|
+
this.name = "SandboxCapabilitiesChangedError";
|
|
8429
|
+
}
|
|
8430
|
+
};
|
|
8431
|
+
async function checkpointHistoryBeforeProviderDispatch(historySink, route) {
|
|
8432
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
8433
|
+
if (route?.effectiveSandboxBackend === "none" && route.routingEnabled) {
|
|
8434
|
+
const active = await route.readActiveSandbox();
|
|
8435
|
+
if (active?.activeSandboxId) throw new SandboxCapabilitiesChangedError();
|
|
8436
|
+
}
|
|
8437
|
+
}
|
|
8438
|
+
|
|
8439
|
+
// src/activities/agent-turn/claude-usage-observer.ts
|
|
8440
|
+
import { createHash as createHash4 } from "crypto";
|
|
8441
|
+
import {
|
|
8442
|
+
emptyClaudeUsage,
|
|
8443
|
+
mergeClaudeUsage,
|
|
8444
|
+
parseClaudeUsageHeaders
|
|
8445
|
+
} from "@opengeni/config";
|
|
8446
|
+
var ClaudeSubscriptionConnectionUnavailable = class extends Error {
|
|
8447
|
+
status = 409;
|
|
8448
|
+
code = "claude_subscription_connection_changed";
|
|
8449
|
+
constructor() {
|
|
8450
|
+
super(
|
|
8451
|
+
"Claude connection changed or was disconnected. Start a new turn with the current connection."
|
|
8452
|
+
);
|
|
8453
|
+
}
|
|
8454
|
+
};
|
|
8455
|
+
async function createClaudeUsageObserver(providers, latest, readCredential) {
|
|
8456
|
+
const managedProviderIds = new Set(
|
|
8457
|
+
providers.filter(
|
|
8458
|
+
(provider) => provider.kind === "claude-subscription-workspace" || provider.kind === "claude-subscription-organization"
|
|
8459
|
+
).map((provider) => provider.id)
|
|
8460
|
+
);
|
|
8461
|
+
const bindings = await Promise.all(
|
|
8462
|
+
providers.filter(
|
|
8463
|
+
(provider) => provider.kind === "claude-subscription-workspace" || provider.kind === "claude-subscription-organization"
|
|
8464
|
+
).map(async (provider) => {
|
|
8465
|
+
const scope = provider.kind === "claude-subscription-workspace" ? "workspace" : "organization";
|
|
8466
|
+
const binding = provider.anthropic?.credentialBinding;
|
|
8467
|
+
if (binding && provider.apiKey)
|
|
8468
|
+
return [
|
|
8469
|
+
provider.id,
|
|
8470
|
+
{
|
|
8471
|
+
scope,
|
|
8472
|
+
token: provider.apiKey,
|
|
8473
|
+
expectedConnectionId: binding.connectionId,
|
|
8474
|
+
expectedCredentialVersion: binding.credentialVersion
|
|
8475
|
+
}
|
|
8476
|
+
];
|
|
8477
|
+
const credential = await readCredential(scope).catch(() => null);
|
|
8478
|
+
if (!credential || credential.token !== provider.apiKey) return null;
|
|
8479
|
+
return [
|
|
8480
|
+
provider.id,
|
|
8481
|
+
{
|
|
8482
|
+
scope,
|
|
8483
|
+
token: credential.token,
|
|
8484
|
+
expectedConnectionId: credential.connectionId,
|
|
8485
|
+
expectedCredentialVersion: credential.credentialVersion
|
|
8486
|
+
}
|
|
8487
|
+
];
|
|
8488
|
+
})
|
|
8489
|
+
);
|
|
8490
|
+
const captured = new Map(bindings.filter((binding) => binding !== null));
|
|
8491
|
+
let anonymousReceipt = 0;
|
|
8492
|
+
const observeBinding = (binding, response, upstreamModelId, requestToken) => {
|
|
8493
|
+
if (requestToken === null) return;
|
|
8494
|
+
if (!binding) return;
|
|
8495
|
+
const { scope, ...capturedIdentity } = binding;
|
|
8496
|
+
const identity = { ...capturedIdentity, token: requestToken ?? capturedIdentity.token };
|
|
8497
|
+
const tokenKey = createHash4("sha256").update(identity.token).digest("hex");
|
|
8498
|
+
const rawRequestId = response.headers.get("request-id");
|
|
8499
|
+
const requestId = rawRequestId && rawRequestId.length <= 256 ? rawRequestId : void 0;
|
|
8500
|
+
const receiptKey = requestId ?? (response.status === 401 || response.status === 429 ? `anonymous-${++anonymousReceipt}` : "aggregate");
|
|
8501
|
+
const captureKey = `${receiptKey}:${scope}:${identity.expectedConnectionId}:${identity.expectedCredentialVersion}:${tokenKey}:${upstreamModelId ?? "unknown"}`;
|
|
8502
|
+
const previous = latest.get(captureKey);
|
|
8503
|
+
let observation = parseClaudeUsageHeaders(response.headers, /* @__PURE__ */ new Date(), upstreamModelId);
|
|
8504
|
+
if (observation && previous?.observation) {
|
|
8505
|
+
const merged = mergeClaudeUsage(
|
|
8506
|
+
mergeClaudeUsage(emptyClaudeUsage(binding.expectedCredentialVersion), previous.observation),
|
|
8507
|
+
observation
|
|
8508
|
+
);
|
|
8509
|
+
observation = {
|
|
8510
|
+
windows: merged.windows,
|
|
8511
|
+
observedAt: merged.observedAt,
|
|
8512
|
+
source: merged.source,
|
|
8513
|
+
requestStatus: merged.requestStatus ?? null,
|
|
8514
|
+
requestRestrictions: merged.requestRestrictions ?? []
|
|
8515
|
+
};
|
|
8516
|
+
}
|
|
8517
|
+
if (requestId || observation || response.status === 401 || response.status === 429)
|
|
8518
|
+
latest.set(captureKey, {
|
|
8519
|
+
scope,
|
|
8520
|
+
responseStatus: response.status,
|
|
8521
|
+
...response.headers.get("request-id") && response.headers.get("request-id").length <= 256 ? { requestId: response.headers.get("request-id") } : {},
|
|
8522
|
+
...upstreamModelId ? { upstreamModelId } : {},
|
|
8523
|
+
...identity,
|
|
8524
|
+
...previous?.observation && !observation ? { observation: previous.observation } : {},
|
|
8525
|
+
...observation ? { observation } : {},
|
|
8526
|
+
...response.status === 401 ? { refresh: { status: "reconnect", checkedAt: (/* @__PURE__ */ new Date()).toISOString() } } : {}
|
|
8527
|
+
});
|
|
8528
|
+
};
|
|
8529
|
+
const observe = (providerId, response, upstreamModelId, requestToken) => observeBinding(captured.get(providerId), response, upstreamModelId, requestToken);
|
|
8530
|
+
const prepareRequestWithObserver = async (providerId, headers, resolve) => {
|
|
8531
|
+
if (!managedProviderIds.has(providerId)) return { headers, observe };
|
|
8532
|
+
const binding = captured.get(providerId);
|
|
8533
|
+
if (!binding) throw new ClaudeSubscriptionConnectionUnavailable();
|
|
8534
|
+
const credential = await resolve({
|
|
8535
|
+
scope: binding.scope,
|
|
8536
|
+
expectedConnectionId: binding.expectedConnectionId,
|
|
8537
|
+
expectedCredentialVersion: binding.expectedCredentialVersion
|
|
8538
|
+
});
|
|
8539
|
+
if (!credential || credential.connectionId !== binding.expectedConnectionId || credential.credentialVersion !== binding.expectedCredentialVersion)
|
|
8540
|
+
throw new ClaudeSubscriptionConnectionUnavailable();
|
|
8541
|
+
const dispatched = Object.freeze({ ...binding, token: credential.token });
|
|
8542
|
+
captured.set(providerId, dispatched);
|
|
8543
|
+
headers.set("authorization", `Bearer ${credential.token}`);
|
|
8544
|
+
headers.delete("x-api-key");
|
|
8545
|
+
return {
|
|
8546
|
+
headers,
|
|
8547
|
+
observe: (_providerId, response, upstreamModelId, requestToken) => observeBinding(dispatched, response, upstreamModelId, requestToken)
|
|
8548
|
+
};
|
|
8549
|
+
};
|
|
8550
|
+
return Object.assign(observe, {
|
|
8551
|
+
prepareRequestWithObserver,
|
|
8552
|
+
async prepareRequest(...args) {
|
|
8553
|
+
return (await prepareRequestWithObserver(...args)).headers;
|
|
8554
|
+
},
|
|
8555
|
+
binding(providerId) {
|
|
8556
|
+
const binding = captured.get(providerId);
|
|
8557
|
+
return binding ? { ...binding } : void 0;
|
|
8558
|
+
},
|
|
8559
|
+
renew(providerId, credential) {
|
|
8560
|
+
const binding = captured.get(providerId);
|
|
8561
|
+
if (binding && binding.expectedConnectionId === credential.connectionId && binding.expectedCredentialVersion === credential.credentialVersion)
|
|
8562
|
+
captured.set(providerId, { ...binding, token: credential.token });
|
|
8563
|
+
}
|
|
8564
|
+
});
|
|
8565
|
+
}
|
|
8566
|
+
|
|
7974
8567
|
// src/activities/agent-turn/sandbox-provision.ts
|
|
7975
8568
|
import {
|
|
7976
8569
|
SandboxLeaseRecoveryBlockedError as SandboxLeaseRecoveryBlockedError2,
|
|
@@ -8446,12 +9039,14 @@ async function persistOrSignalSessionAttemptQuiescence(input) {
|
|
|
8446
9039
|
return "receipt";
|
|
8447
9040
|
}
|
|
8448
9041
|
async function drainAttemptOwnedSandboxWriters(input) {
|
|
9042
|
+
input.onStage?.("tool_writers");
|
|
8449
9043
|
if (input.toolCancellationFence) {
|
|
8450
9044
|
input.toolCancellationFence.cancel(
|
|
8451
9045
|
input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED")
|
|
8452
9046
|
);
|
|
8453
9047
|
await input.toolCancellationFence.waitForQuiescence();
|
|
8454
9048
|
}
|
|
9049
|
+
input.onStage?.("credential_renewals");
|
|
8455
9050
|
await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
|
|
8456
9051
|
await input.codemodeTokenRenewal?.stop();
|
|
8457
9052
|
await input.runCredentialRenewal?.stop();
|
|
@@ -8564,7 +9159,7 @@ async function runMandatoryHistoryPersistenceStep(stage, operation) {
|
|
|
8564
9159
|
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
8565
9160
|
}
|
|
8566
9161
|
}
|
|
8567
|
-
async function finalizeDurableTurnOpStreams(sessions, fallback) {
|
|
9162
|
+
async function finalizeDurableTurnOpStreams(sessions, fallback, toolCallIds) {
|
|
8568
9163
|
const candidates = /* @__PURE__ */ new Set();
|
|
8569
9164
|
for (const session of sessions) {
|
|
8570
9165
|
const candidate = session;
|
|
@@ -8577,17 +9172,53 @@ async function finalizeDurableTurnOpStreams(sessions, fallback) {
|
|
|
8577
9172
|
}
|
|
8578
9173
|
for (const candidate of candidates) {
|
|
8579
9174
|
try {
|
|
8580
|
-
await candidate.finalizeOpStreamOps();
|
|
9175
|
+
await candidate.finalizeOpStreamOps(toolCallIds);
|
|
8581
9176
|
} catch {
|
|
8582
9177
|
}
|
|
8583
9178
|
}
|
|
8584
9179
|
}
|
|
8585
9180
|
|
|
9181
|
+
// src/activities/agent-turn/errors.ts
|
|
9182
|
+
import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
|
|
9183
|
+
|
|
9184
|
+
// ../../node_modules/.bun/drizzle-orm@0.45.2+1d2c014f134ec9f4/node_modules/drizzle-orm/entity.js
|
|
9185
|
+
var entityKind = Symbol.for("drizzle:entityKind");
|
|
9186
|
+
var hasOwnEntityKind = Symbol.for("drizzle:hasOwnEntityKind");
|
|
9187
|
+
|
|
9188
|
+
// ../../node_modules/.bun/drizzle-orm@0.45.2+1d2c014f134ec9f4/node_modules/drizzle-orm/errors.js
|
|
9189
|
+
var DrizzleError = class extends Error {
|
|
9190
|
+
static [entityKind] = "DrizzleError";
|
|
9191
|
+
constructor({ message, cause }) {
|
|
9192
|
+
super(message);
|
|
9193
|
+
this.name = "DrizzleError";
|
|
9194
|
+
this.cause = cause;
|
|
9195
|
+
}
|
|
9196
|
+
};
|
|
9197
|
+
var DrizzleQueryError = class _DrizzleQueryError extends Error {
|
|
9198
|
+
constructor(query, params, cause) {
|
|
9199
|
+
super(`Failed query: ${query}
|
|
9200
|
+
params: ${params}`);
|
|
9201
|
+
this.query = query;
|
|
9202
|
+
this.params = params;
|
|
9203
|
+
this.cause = cause;
|
|
9204
|
+
Error.captureStackTrace(this, _DrizzleQueryError);
|
|
9205
|
+
if (cause) this.cause = cause;
|
|
9206
|
+
}
|
|
9207
|
+
};
|
|
9208
|
+
var TransactionRollbackError = class extends DrizzleError {
|
|
9209
|
+
static [entityKind] = "TransactionRollbackError";
|
|
9210
|
+
constructor() {
|
|
9211
|
+
super({ message: "Rollback" });
|
|
9212
|
+
}
|
|
9213
|
+
};
|
|
9214
|
+
|
|
8586
9215
|
// src/activities/agent-turn/errors.ts
|
|
8587
9216
|
import {
|
|
8588
9217
|
ActiveSessionHistoryLimitExceededError,
|
|
8589
9218
|
ApprovalRunStateLimitExceededError,
|
|
8590
9219
|
nestedPostgresSqlState,
|
|
9220
|
+
databaseFailureCode,
|
|
9221
|
+
isRetryablePersistenceSqlState,
|
|
8591
9222
|
safeDatabaseErrorFacts,
|
|
8592
9223
|
isRetryableDatabaseTransportFailure,
|
|
8593
9224
|
isSessionEventPersistenceError,
|
|
@@ -8595,14 +9226,27 @@ import {
|
|
|
8595
9226
|
} from "@opengeni/db";
|
|
8596
9227
|
import {
|
|
8597
9228
|
ActiveBackendUnresolvableError as ActiveBackendUnresolvableError2,
|
|
9229
|
+
AnthropicProviderRejection,
|
|
8598
9230
|
CompactionProviderResponseError,
|
|
8599
9231
|
EmptyCompactionSummaryError,
|
|
9232
|
+
compactionProviderRejection,
|
|
9233
|
+
describeCompactionProviderRejection,
|
|
8600
9234
|
isMcpRequestTimeoutError,
|
|
8601
9235
|
isMcpTransportConnectivityError,
|
|
8602
|
-
|
|
9236
|
+
isModalTaskExecStartPreDispatchUnavailableError,
|
|
9237
|
+
isModalCommandStartOutcomeUnknownError,
|
|
9238
|
+
isProviderCommandObservationUnavailableError,
|
|
9239
|
+
isRoutingMutationOutcomeUnknownError,
|
|
8603
9240
|
RoutingWorkspaceRootChangedError,
|
|
9241
|
+
ResponsesStreamingTerminalError,
|
|
9242
|
+
SandboxMaterializationVerificationError,
|
|
9243
|
+
materializationVerificationDiagnostic,
|
|
9244
|
+
PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
9245
|
+
classifyProviderQuotaError,
|
|
9246
|
+
providerQuotaExhaustedMessage,
|
|
8604
9247
|
SelfhostedWorkspaceRootChangedError,
|
|
8605
|
-
UNKNOWN_MODEL_FINISH_REASON_CODE
|
|
9248
|
+
UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
9249
|
+
AnthropicRequestError
|
|
8606
9250
|
} from "@opengeni/runtime";
|
|
8607
9251
|
import {
|
|
8608
9252
|
mcpTransportRequestFailureDiagnostic
|
|
@@ -8616,7 +9260,7 @@ import {
|
|
|
8616
9260
|
FileResourceRef,
|
|
8617
9261
|
resourceMountPath
|
|
8618
9262
|
} from "@opengeni/contracts";
|
|
8619
|
-
import { createHash as
|
|
9263
|
+
import { createHash as createHash5 } from "crypto";
|
|
8620
9264
|
import {
|
|
8621
9265
|
getActiveSessionHistoryItemsPaged,
|
|
8622
9266
|
getFilesForSubject,
|
|
@@ -8734,7 +9378,7 @@ async function modelAttachmentContentForFiles(files, readFileBytes, options = {}
|
|
|
8734
9378
|
});
|
|
8735
9379
|
continue;
|
|
8736
9380
|
}
|
|
8737
|
-
if (
|
|
9381
|
+
if (createHash5("sha256").update(bytes).digest("hex") !== checksum) {
|
|
8738
9382
|
console.error("model attachment checksum did not match finalized metadata", {
|
|
8739
9383
|
fileId: file.id
|
|
8740
9384
|
});
|
|
@@ -8772,12 +9416,8 @@ function attachmentRefsFromItem(item) {
|
|
|
8772
9416
|
}
|
|
8773
9417
|
return refs;
|
|
8774
9418
|
}
|
|
8775
|
-
function attachmentReceiptText(ref
|
|
8776
|
-
|
|
8777
|
-
return `[Earlier attachment: fileId=${ref.fileId}; mountDirectory=${resourceMountPath(ref)}. Use the existing file there, or call files__files_get_download_url with this fileId and download it with the shell.]`;
|
|
8778
|
-
}
|
|
8779
|
-
const path = sandboxFilePath(ref, file);
|
|
8780
|
-
return `[Attachment: ${file.safeFilename}; fileId=${file.id}; type=${file.contentType}; bytes=${file.sizeBytes}; path=${path}. If the local path is absent, call files__files_get_download_url with this fileId and download it with the shell.]`;
|
|
9419
|
+
function attachmentReceiptText(ref) {
|
|
9420
|
+
return `[Attachment: fileId=${ref.fileId}; mountDirectory=${resourceMountPath(ref)}. Use the existing file there, or call files__files_get_download_url with this fileId and download it with the shell.]`;
|
|
8781
9421
|
}
|
|
8782
9422
|
function createModelHistoryAttachmentProjector(policy, readFileBytes, loadAuthorizedFiles) {
|
|
8783
9423
|
const contentById = /* @__PURE__ */ new Map();
|
|
@@ -8856,7 +9496,7 @@ function createModelHistoryAttachmentProjector(policy, readFileBytes, loadAuthor
|
|
|
8856
9496
|
const attachment = currentFile ? contentById.get(ref.fileId) : void 0;
|
|
8857
9497
|
const receipt = {
|
|
8858
9498
|
type: "input_text",
|
|
8859
|
-
text: attachmentReceiptText(ref
|
|
9499
|
+
text: attachmentReceiptText(ref)
|
|
8860
9500
|
};
|
|
8861
9501
|
if (!attachment || attachment.kind !== "image") {
|
|
8862
9502
|
return [receipt];
|
|
@@ -8875,12 +9515,18 @@ function createModelHistoryAttachmentProjector(policy, readFileBytes, loadAuthor
|
|
|
8875
9515
|
}
|
|
8876
9516
|
function withCurrentUserAttachmentRefs(historyItems, refs) {
|
|
8877
9517
|
if (refs.length === 0) return historyItems;
|
|
9518
|
+
const retainedIds = new Set(
|
|
9519
|
+
historyItems.flatMap((item) => attachmentRefsFromItem(item).map((ref) => ref.fileId))
|
|
9520
|
+
);
|
|
9521
|
+
const missing = refs.filter((ref) => !retainedIds.has(ref.fileId));
|
|
9522
|
+
if (missing.length === 0) return historyItems;
|
|
8878
9523
|
for (let index = historyItems.length - 1; index >= 0; index -= 1) {
|
|
8879
9524
|
const item = historyItems[index];
|
|
9525
|
+
if (item[MODEL_ATTACHMENT_CATALOG_MARKER] === true) continue;
|
|
8880
9526
|
if (item.type !== "message" || item.role !== "user") continue;
|
|
8881
9527
|
const existing = attachmentRefsFromItem(item);
|
|
8882
9528
|
const existingIds = new Set(existing.map((ref) => ref.fileId));
|
|
8883
|
-
const additions =
|
|
9529
|
+
const additions = missing.filter((ref) => !existingIds.has(ref.fileId));
|
|
8884
9530
|
if (additions.length === 0) return historyItems;
|
|
8885
9531
|
const projected = [...historyItems];
|
|
8886
9532
|
projected[index] = { ...item, [MODEL_ATTACHMENT_REFS_FIELD]: [...existing, ...additions] };
|
|
@@ -8950,8 +9596,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
8950
9596
|
options
|
|
8951
9597
|
);
|
|
8952
9598
|
}
|
|
8953
|
-
if (trigger.type === "system.update.delivered") {
|
|
8954
|
-
if (updates.length === 0) {
|
|
9599
|
+
if (trigger.type === "system.update.delivered" || trigger.type === "session.context.compaction.requested") {
|
|
9600
|
+
if (trigger.type === "system.update.delivered" && updates.length === 0) {
|
|
8955
9601
|
throw new Error("Internal update inference has no delivered updates");
|
|
8956
9602
|
}
|
|
8957
9603
|
return await messageInput(
|
|
@@ -9116,11 +9762,125 @@ function sandboxFilePath(resource, file) {
|
|
|
9116
9762
|
|
|
9117
9763
|
// src/activities/agent-turn/errors.ts
|
|
9118
9764
|
import {
|
|
9765
|
+
CODEX_USAGE_LIMIT_ERROR_TYPE,
|
|
9119
9766
|
CodexReloginRequired,
|
|
9767
|
+
classifyCodexEncryptedArtifactRejection,
|
|
9768
|
+
classifyCodexEntitlementRejection,
|
|
9120
9769
|
classifyCodexResponseTimeoutError,
|
|
9121
9770
|
classifyCodexUsageLimitError,
|
|
9122
9771
|
isCodexTransportError
|
|
9123
9772
|
} from "@opengeni/codex";
|
|
9773
|
+
|
|
9774
|
+
// src/activities/agent-turn/codex-plan-entitlement.ts
|
|
9775
|
+
import { productLabelForModelId } from "@opengeni/config";
|
|
9776
|
+
import {
|
|
9777
|
+
codexPlanDisplayName,
|
|
9778
|
+
codexPlanEntitlementLost,
|
|
9779
|
+
codexPlanIsUpgrade,
|
|
9780
|
+
codexPlanKey
|
|
9781
|
+
} from "@opengeni/codex";
|
|
9782
|
+
import {
|
|
9783
|
+
codexPlanExcludesModel as codexPlanExcludesModel2,
|
|
9784
|
+
codexPlanPreviouslyExcludedModel,
|
|
9785
|
+
connectionModelAllowed as connectionModelAllowed2
|
|
9786
|
+
} from "@opengeni/db";
|
|
9787
|
+
var CODEX_PLAN_ENTITLEMENT_CODE = "codex_plan_entitlement";
|
|
9788
|
+
var CODEX_REQUEST_REJECTED_CODE = "codex_request_rejected";
|
|
9789
|
+
function codexAccountDisplayLabel(account) {
|
|
9790
|
+
const label = account?.label?.trim() || null;
|
|
9791
|
+
return label ? label.slice(0, 120) : null;
|
|
9792
|
+
}
|
|
9793
|
+
function accountPhrase(accountLabel) {
|
|
9794
|
+
return accountLabel ? `The ChatGPT account "${accountLabel}"` : "The connected ChatGPT account";
|
|
9795
|
+
}
|
|
9796
|
+
function rejectionDetail(rejection) {
|
|
9797
|
+
if (!rejection) return void 0;
|
|
9798
|
+
return rejection.evidence === "empty_body" ? `The Codex backend answered HTTP ${rejection.status} with no error body.` : `The Codex backend refused the model for this plan (HTTP ${rejection.status}).`;
|
|
9799
|
+
}
|
|
9800
|
+
function codexPlanEntitlementFailurePayload(input) {
|
|
9801
|
+
const plan = codexPlanDisplayName(input.planType);
|
|
9802
|
+
const model = input.modelId ? productLabelForModelId(input.modelId) : "this model";
|
|
9803
|
+
const lead = accountPhrase(input.accountLabel);
|
|
9804
|
+
const next = input.waiting ? "Opengeni is waiting for another connected account to become available." : "Upgrade it, use another connected account, or choose another model.";
|
|
9805
|
+
const error = plan ? `${lead} ${input.planChanged ? "is now" : "is"} on the ${plan} plan, which doesn't include ${model}. ${next}` : `${lead} no longer has access to ${model} on its current plan. ${next}`;
|
|
9806
|
+
const detail = rejectionDetail(input.rejection ?? null);
|
|
9807
|
+
return {
|
|
9808
|
+
error,
|
|
9809
|
+
code: CODEX_PLAN_ENTITLEMENT_CODE,
|
|
9810
|
+
retryable: false,
|
|
9811
|
+
planType: plan ? codexPlanKey(input.planType) : null,
|
|
9812
|
+
model: input.modelId,
|
|
9813
|
+
...detail ? { detail } : {}
|
|
9814
|
+
};
|
|
9815
|
+
}
|
|
9816
|
+
function codexRequestRejectedFailurePayload(input) {
|
|
9817
|
+
const plan = codexPlanDisplayName(input.planType);
|
|
9818
|
+
const lead = accountPhrase(input.accountLabel);
|
|
9819
|
+
const planClause = input.planChecked === false ? "" : plan ? `${lead} still reports the ${plan} plan, so Opengeni did not switch accounts. ` : `Opengeni could not confirm the current plan of ${lead.charAt(0).toLowerCase()}${lead.slice(1)}, so it did not switch accounts. `;
|
|
9820
|
+
return {
|
|
9821
|
+
error: `The Codex backend rejected this request (HTTP ${input.rejection.status}) without an error message. ${planClause}Try again, or choose another model if it keeps failing.`,
|
|
9822
|
+
code: CODEX_REQUEST_REJECTED_CODE,
|
|
9823
|
+
retryable: false,
|
|
9824
|
+
planType: plan ? codexPlanKey(input.planType) : null,
|
|
9825
|
+
...rejectionDetail(input.rejection) ? { detail: rejectionDetail(input.rejection) } : {}
|
|
9826
|
+
};
|
|
9827
|
+
}
|
|
9828
|
+
var CodexPlanEntitlementError = class extends Error {
|
|
9829
|
+
constructor(payload) {
|
|
9830
|
+
super(payload.error);
|
|
9831
|
+
this.payload = payload;
|
|
9832
|
+
this.name = "CodexPlanEntitlementError";
|
|
9833
|
+
}
|
|
9834
|
+
code = CODEX_PLAN_ENTITLEMENT_CODE;
|
|
9835
|
+
};
|
|
9836
|
+
function assessCodexPlanEntitlement(rejection, recheck, modelId) {
|
|
9837
|
+
const planObserved = recheck.planType !== null;
|
|
9838
|
+
const planChangedFrom = planObserved && recheck.previousPlanType !== null && codexPlanKey(recheck.previousPlanType) !== codexPlanKey(recheck.planType) ? recheck.previousPlanType : recheck.planChangedFrom;
|
|
9839
|
+
const planChanged = planObserved && planChangedFrom !== null && codexPlanKey(planChangedFrom) !== codexPlanKey(recheck.planType) && !codexPlanIsUpgrade(planChangedFrom, recheck.planType);
|
|
9840
|
+
if (codexPlanEntitlementLost({
|
|
9841
|
+
evidence: rejection.evidence,
|
|
9842
|
+
currentPlanType: recheck.planType,
|
|
9843
|
+
planChangedFrom,
|
|
9844
|
+
previouslyExcluded: planObserved && codexPlanPreviouslyExcludedModel(
|
|
9845
|
+
{ planType: recheck.planType, planEntitlementExclusion: recheck.exclusion },
|
|
9846
|
+
modelId
|
|
9847
|
+
)
|
|
9848
|
+
})) {
|
|
9849
|
+
return {
|
|
9850
|
+
kind: "entitlement_lost",
|
|
9851
|
+
planType: planObserved ? recheck.planType : recheck.previousPlanType,
|
|
9852
|
+
planObserved,
|
|
9853
|
+
planChanged,
|
|
9854
|
+
credentialVersion: recheck.credentialVersion
|
|
9855
|
+
};
|
|
9856
|
+
}
|
|
9857
|
+
return { kind: "unexplained", planType: recheck.planType };
|
|
9858
|
+
}
|
|
9859
|
+
function codexPlanEntitlementAdmissionBlock(input) {
|
|
9860
|
+
if (input.credentialId !== null) return null;
|
|
9861
|
+
const permitted = input.accounts.filter(
|
|
9862
|
+
(account) => connectionModelAllowed2(account.allowedModelIds, input.modelId)
|
|
9863
|
+
);
|
|
9864
|
+
const excluded = permitted.filter(
|
|
9865
|
+
(account) => codexPlanExcludesModel2(account, input.modelId, input.now)
|
|
9866
|
+
);
|
|
9867
|
+
if (excluded.length === 0) return null;
|
|
9868
|
+
if (input.pinnedCredentialId && input.pinSource !== "policy") {
|
|
9869
|
+
const pinned = excluded.find((account) => account.id === input.pinnedCredentialId);
|
|
9870
|
+
return pinned ? [pinned] : null;
|
|
9871
|
+
}
|
|
9872
|
+
if (!input.rotationEnabled) {
|
|
9873
|
+
const active = excluded.find((account) => account.id === input.activeCredentialId);
|
|
9874
|
+
return active ? [active] : null;
|
|
9875
|
+
}
|
|
9876
|
+
const allocatable = permitted.filter((account) => account.allocatorEnabled);
|
|
9877
|
+
const allocatableExcluded = allocatable.filter(
|
|
9878
|
+
(account) => codexPlanExcludesModel2(account, input.modelId, input.now)
|
|
9879
|
+
);
|
|
9880
|
+
return allocatable.length > 0 && allocatableExcluded.length === allocatable.length ? allocatableExcluded : null;
|
|
9881
|
+
}
|
|
9882
|
+
|
|
9883
|
+
// src/activities/agent-turn/errors.ts
|
|
9124
9884
|
import {
|
|
9125
9885
|
classifyXaiSubscriptionStreamingTerminalError,
|
|
9126
9886
|
classifyXaiSubscriptionStreamIdleTimeoutError,
|
|
@@ -9142,6 +9902,7 @@ var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery
|
|
|
9142
9902
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
9143
9903
|
var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
|
|
9144
9904
|
var MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
9905
|
+
var PROVIDER_RATE_LIMIT_BACKOFF_MS = [1e4, 2e4, 4e4, 6e4, 12e4];
|
|
9145
9906
|
var POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
|
|
9146
9907
|
var PostCompactionContinuationEmptyError = class extends Error {
|
|
9147
9908
|
code = POST_COMPACTION_CONTINUATION_EMPTY_CODE;
|
|
@@ -9159,7 +9920,13 @@ function providerRecoveryResult(input) {
|
|
|
9159
9920
|
};
|
|
9160
9921
|
}
|
|
9161
9922
|
const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
|
|
9162
|
-
const continueDelayMs = input.failureCode === "provider_rate_limited" ?
|
|
9923
|
+
const continueDelayMs = input.failureCode === "provider_rate_limited" ? Math.max(
|
|
9924
|
+
providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
9925
|
+
PROVIDER_RATE_LIMIT_BACKOFF_MS[Math.min(
|
|
9926
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
9927
|
+
PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1
|
|
9928
|
+
)]
|
|
9929
|
+
) : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "sandbox_command_start_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" || input.failureCode === "turn_execution_policy_definition_mismatch" || input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE ? Math.max(
|
|
9163
9930
|
providerDelay ?? 0,
|
|
9164
9931
|
PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
|
|
9165
9932
|
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
@@ -9207,6 +9974,10 @@ function providerRetryAfterMs(error, nowMs = Date.now()) {
|
|
|
9207
9974
|
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
9208
9975
|
const value = current;
|
|
9209
9976
|
const body = value.error && typeof value.error === "object" ? value.error : null;
|
|
9977
|
+
const milliseconds = Number(
|
|
9978
|
+
headerValue(value.headers, "retry-after-ms") ?? headerValue(value.responseHeaders, "retry-after-ms") ?? headerValue(body?.headers, "retry-after-ms") ?? void 0
|
|
9979
|
+
);
|
|
9980
|
+
if (Number.isFinite(milliseconds) && milliseconds > 0) return Math.ceil(milliseconds);
|
|
9210
9981
|
const directSeconds = Number(
|
|
9211
9982
|
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
|
|
9212
9983
|
);
|
|
@@ -9242,9 +10013,15 @@ function escapedMcpTimeoutRecoveryFailure(input) {
|
|
|
9242
10013
|
function preClaimAdmissionFailure(error) {
|
|
9243
10014
|
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
9244
10015
|
const retryableCode = retryableDatabaseFailureCode(error);
|
|
10016
|
+
const rejectedClaim = persistenceFailure?.details.stage === "session_attempts.claim" && persistenceFailure.details.retryOutcome === "not_retryable" && !retryableCode;
|
|
9245
10017
|
const detail = {
|
|
9246
|
-
disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
|
|
9247
|
-
code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant"
|
|
10018
|
+
disposition: rejectedClaim ? "blocked" : persistenceFailure || retryableCode ? "retryable" : "permanent",
|
|
10019
|
+
code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
|
|
10020
|
+
...persistenceFailure && rejectedClaim ? {
|
|
10021
|
+
sqlState: persistenceFailure.details.sqlState,
|
|
10022
|
+
reason: persistenceFailure.details.stage === "session_attempts.claim" && persistenceFailure.details.sqlState === "OG001" ? "initiator_membership_required" : persistenceFailure.details.stage === "session_attempts.claim" && persistenceFailure.details.sqlState === "OG002" ? "personal_resource_grant_required" : "database_claim_rejected",
|
|
10023
|
+
retryPolicy: "explicit_recheck"
|
|
10024
|
+
} : {}
|
|
9248
10025
|
};
|
|
9249
10026
|
return ApplicationFailure.create({
|
|
9250
10027
|
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
@@ -9253,20 +10030,122 @@ function preClaimAdmissionFailure(error) {
|
|
|
9253
10030
|
details: [detail]
|
|
9254
10031
|
});
|
|
9255
10032
|
}
|
|
9256
|
-
function
|
|
9257
|
-
const
|
|
9258
|
-
const
|
|
9259
|
-
|
|
9260
|
-
|
|
10033
|
+
function databaseRecoveryCauseGraph(error) {
|
|
10034
|
+
const graph = /* @__PURE__ */ new Map();
|
|
10035
|
+
const queue = [];
|
|
10036
|
+
let links = 0;
|
|
10037
|
+
const add = (value) => {
|
|
10038
|
+
if (!value || typeof value !== "object" || graph.has(value)) return true;
|
|
10039
|
+
if (graph.size >= 64) return false;
|
|
10040
|
+
graph.set(value, /* @__PURE__ */ new Set());
|
|
10041
|
+
queue.push(value);
|
|
10042
|
+
return true;
|
|
10043
|
+
};
|
|
10044
|
+
if (!add(error)) return null;
|
|
10045
|
+
try {
|
|
10046
|
+
for (const current of queue) {
|
|
10047
|
+
const record = current;
|
|
10048
|
+
const children = Array.isArray(current) ? current : ["cause", "original", "driverError", "error", "errors"].map((key) => record[key]);
|
|
10049
|
+
for (const child of children) {
|
|
10050
|
+
if (++links > 4096 || !add(child)) return null;
|
|
10051
|
+
if (child && typeof child === "object") graph.get(current).add(child);
|
|
10052
|
+
}
|
|
10053
|
+
}
|
|
10054
|
+
} catch {
|
|
10055
|
+
return null;
|
|
9261
10056
|
}
|
|
9262
|
-
|
|
10057
|
+
return graph;
|
|
10058
|
+
}
|
|
10059
|
+
function retryableDatabaseFailureCode(error, requireDatabaseProvenance = false) {
|
|
10060
|
+
try {
|
|
10061
|
+
const graph = databaseRecoveryCauseGraph(error);
|
|
10062
|
+
if (!graph) return null;
|
|
10063
|
+
const transports = /* @__PURE__ */ new Set();
|
|
10064
|
+
const boundaries = /* @__PURE__ */ new Set();
|
|
10065
|
+
const ownDatabaseNodes = /* @__PURE__ */ new Set();
|
|
10066
|
+
const codes = /* @__PURE__ */ new Set();
|
|
10067
|
+
for (const node of graph.keys()) {
|
|
10068
|
+
if (isRoutingMutationOutcomeUnknownError(node)) return null;
|
|
10069
|
+
const record = node;
|
|
10070
|
+
if (requireDatabaseProvenance ? isRunningTurnDatabaseTransportFailure(record) : isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno }))
|
|
10071
|
+
transports.add(node);
|
|
10072
|
+
const sqlState = isSessionEventPersistenceError(node) ? node.details.sqlState : record.name === "PostgresError" && typeof record.code === "string" ? record.code : null;
|
|
10073
|
+
if (requireDatabaseProvenance ? isRunningTurnDatabaseConnectionSqlState(sqlState) : isDatabaseConnectionSqlState(sqlState))
|
|
10074
|
+
transports.add(node);
|
|
10075
|
+
if (node instanceof DrizzleQueryError || isSessionEventPersistenceError(node)) {
|
|
10076
|
+
const queue = [node];
|
|
10077
|
+
for (const source of queue) {
|
|
10078
|
+
if (ownDatabaseNodes.has(source)) continue;
|
|
10079
|
+
ownDatabaseNodes.add(source);
|
|
10080
|
+
queue.push(...graph.get(source));
|
|
10081
|
+
}
|
|
10082
|
+
}
|
|
10083
|
+
}
|
|
10084
|
+
const hasOwnTransport = (boundary) => {
|
|
10085
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10086
|
+
const queue = [boundary];
|
|
10087
|
+
for (const node of queue) {
|
|
10088
|
+
if (seen.has(node)) continue;
|
|
10089
|
+
seen.add(node);
|
|
10090
|
+
if (transports.has(node)) return true;
|
|
10091
|
+
queue.push(...graph.get(node));
|
|
10092
|
+
}
|
|
10093
|
+
return false;
|
|
10094
|
+
};
|
|
10095
|
+
for (const node of graph.keys()) {
|
|
10096
|
+
const record = node;
|
|
10097
|
+
if (node instanceof DrizzleQueryError) {
|
|
10098
|
+
boundaries.add(node);
|
|
10099
|
+
if (hasOwnTransport(node)) codes.add("db_failure");
|
|
10100
|
+
}
|
|
10101
|
+
const typed = isSessionEventPersistenceError(node);
|
|
10102
|
+
if (!typed && record.name !== "PostgresError") continue;
|
|
10103
|
+
boundaries.add(node);
|
|
10104
|
+
const sqlState = typed ? node.details.sqlState : typeof record.code === "string" ? record.code : null;
|
|
10105
|
+
const connectionOutage = requireDatabaseProvenance ? isRunningTurnDatabaseConnectionSqlState(sqlState) : isDatabaseConnectionSqlState(sqlState);
|
|
10106
|
+
const code = typed ? retryablePersistenceFailureCode(sqlState) : connectionOutage || isRetryablePersistenceSqlState(sqlState) ? databaseFailureCode(sqlState) : null;
|
|
10107
|
+
if (!code || requireDatabaseProvenance && !connectionOutage && !(sqlState === null && hasOwnTransport(node)))
|
|
10108
|
+
return null;
|
|
10109
|
+
if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
|
|
10110
|
+
}
|
|
10111
|
+
if (!requireDatabaseProvenance && boundaries.size === 0 && transports.size > 0)
|
|
10112
|
+
codes.add("db_failure");
|
|
10113
|
+
for (const code of ["db_failure", "db_deadlock", "db_serialization_failure"])
|
|
10114
|
+
if (codes.has(code)) return code;
|
|
10115
|
+
return null;
|
|
10116
|
+
} catch {
|
|
9263
10117
|
return null;
|
|
9264
10118
|
}
|
|
9265
|
-
|
|
10119
|
+
}
|
|
10120
|
+
var RUNNING_TURN_DATABASE_TRANSPORT_CODES = /* @__PURE__ */ new Set([
|
|
10121
|
+
"ECONNREFUSED",
|
|
10122
|
+
"ECONNRESET",
|
|
10123
|
+
"CONNECT_TIMEOUT"
|
|
10124
|
+
]);
|
|
10125
|
+
function isRunningTurnDatabaseTransportFailure(record) {
|
|
10126
|
+
return [record.code, record.errno].some(
|
|
10127
|
+
(code) => typeof code === "string" && RUNNING_TURN_DATABASE_TRANSPORT_CODES.has(code)
|
|
10128
|
+
);
|
|
10129
|
+
}
|
|
10130
|
+
function isRunningTurnDatabaseConnectionSqlState(sqlState) {
|
|
10131
|
+
return sqlState !== null && (/^08[0-9A-Z]{3}$/.test(sqlState) || ["57P01", "57P02", "57P03"].includes(sqlState));
|
|
10132
|
+
}
|
|
10133
|
+
function isDatabaseConnectionSqlState(sqlState) {
|
|
10134
|
+
return sqlState !== null && (sqlState.startsWith("08") || ["57P01", "57P02", "57P03"].includes(sqlState));
|
|
10135
|
+
}
|
|
10136
|
+
function retryablePersistenceFailureCode(sqlState) {
|
|
10137
|
+
if (!(sqlState === null || sqlState.startsWith("08") || isRetryablePersistenceSqlState(sqlState) || sqlState.startsWith("53") || sqlState === "55P03" || sqlState === "57014" || sqlState === "57P01" || sqlState === "57P02" || sqlState === "57P03" || sqlState.startsWith("58"))) {
|
|
10138
|
+
return null;
|
|
10139
|
+
}
|
|
10140
|
+
return databaseFailureCode(sqlState);
|
|
9266
10141
|
}
|
|
9267
10142
|
function postClaimDatabaseRecoveryFailure(input) {
|
|
9268
|
-
const code = retryableDatabaseFailureCode(input.error);
|
|
9269
|
-
if (!code || input.executionGeneration < 1
|
|
10143
|
+
const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
|
|
10144
|
+
if (!code || input.executionGeneration < 1 || input.requireDatabaseProvenance && code !== "db_failure")
|
|
10145
|
+
return null;
|
|
10146
|
+
if (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted || (input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) && input.providerRecovery) {
|
|
10147
|
+
return null;
|
|
10148
|
+
}
|
|
9270
10149
|
if (input.providerRecovery && (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) || input.providerRecovery.providerRecoveryCount <= 0 || input.providerRecovery.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode))) {
|
|
9271
10150
|
return null;
|
|
9272
10151
|
}
|
|
@@ -9275,6 +10154,8 @@ function postClaimDatabaseRecoveryFailure(input) {
|
|
|
9275
10154
|
triggerEventId: input.triggerEventId,
|
|
9276
10155
|
executionGeneration: input.executionGeneration,
|
|
9277
10156
|
code,
|
|
10157
|
+
...input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {},
|
|
10158
|
+
...input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
|
|
9278
10159
|
...input.providerRecovery ? {
|
|
9279
10160
|
providerFailureCode: input.providerRecovery.failureCode,
|
|
9280
10161
|
providerRecoveryCount: input.providerRecovery.providerRecoveryCount
|
|
@@ -9319,6 +10200,20 @@ function sandboxLifecycleTransitionDiagnostic(error) {
|
|
|
9319
10200
|
}
|
|
9320
10201
|
return null;
|
|
9321
10202
|
}
|
|
10203
|
+
function modelPreparationFailureEventPayload(error, durationMs) {
|
|
10204
|
+
const transition = sandboxLifecycleTransitionDiagnostic(error);
|
|
10205
|
+
return {
|
|
10206
|
+
phase: "model_preparation",
|
|
10207
|
+
durationMs: Math.max(0, Math.round(durationMs)),
|
|
10208
|
+
expectedTransition: transition !== null,
|
|
10209
|
+
...transition ? {
|
|
10210
|
+
failureCategory: "drain_capture_wait",
|
|
10211
|
+
failureStage: "lifecycle_wait",
|
|
10212
|
+
failureCode: transition.reason,
|
|
10213
|
+
retryable: true
|
|
10214
|
+
} : {}
|
|
10215
|
+
};
|
|
10216
|
+
}
|
|
9322
10217
|
function sandboxRouteTransitionCode(error) {
|
|
9323
10218
|
const pending = [error];
|
|
9324
10219
|
const seen = /* @__PURE__ */ new WeakSet();
|
|
@@ -9330,6 +10225,9 @@ function sandboxRouteTransitionCode(error) {
|
|
|
9330
10225
|
seen.add(current);
|
|
9331
10226
|
try {
|
|
9332
10227
|
const record = current;
|
|
10228
|
+
if ((current instanceof SandboxCapabilitiesChangedError || record.name === "SandboxCapabilitiesChangedError") && record.code === "native_capabilities_changed_this_attempt") {
|
|
10229
|
+
return "native_capabilities_changed_this_attempt";
|
|
10230
|
+
}
|
|
9333
10231
|
if ((current instanceof ActiveBackendUnresolvableError2 || record.name === "ActiveBackendUnresolvableError") && record.code === "home_unavailable_this_turn") {
|
|
9334
10232
|
return "home_unavailable_this_turn";
|
|
9335
10233
|
}
|
|
@@ -9384,22 +10282,51 @@ function safeErrorForTelemetry(error) {
|
|
|
9384
10282
|
safe.origin = diagnostic.origin;
|
|
9385
10283
|
return safe;
|
|
9386
10284
|
}
|
|
10285
|
+
var COMPACTION_PROVIDER_REJECTION_GUIDANCE = "The provider refused this exact request, so repeating it fails the same way until the conversation changes. If a new message fails again, start a new session.";
|
|
9387
10286
|
function compactionFailureReasonFromError(error) {
|
|
10287
|
+
const rejection = compactionProviderRejection(error);
|
|
10288
|
+
if (rejection) {
|
|
10289
|
+
return compactionFailureReason(
|
|
10290
|
+
`the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`
|
|
10291
|
+
);
|
|
10292
|
+
}
|
|
10293
|
+
const quota = classifyProviderQuotaExhaustionError(error);
|
|
10294
|
+
if (quota) {
|
|
10295
|
+
return compactionFailureReason(
|
|
10296
|
+
`${providerQuotaExhaustedMessage(quota.scope)} Active history was preserved.`
|
|
10297
|
+
);
|
|
10298
|
+
}
|
|
9388
10299
|
if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError) {
|
|
9389
10300
|
return compactionFailureReason(error.message);
|
|
9390
10301
|
}
|
|
9391
10302
|
const errorName = error instanceof Error && error.name ? error.name : "unknown error";
|
|
9392
10303
|
return compactionFailureReason(`unexpected ${errorName}`);
|
|
9393
10304
|
}
|
|
10305
|
+
function compactionFailureTurnEventPayload(error, overrides = {}) {
|
|
10306
|
+
const rejection = compactionProviderRejection(error);
|
|
10307
|
+
const quota = rejection ? null : classifyProviderQuotaExhaustionError(error);
|
|
10308
|
+
return {
|
|
10309
|
+
error: overrides.error ?? compactionFailureReasonFromError(error),
|
|
10310
|
+
code: "context_compaction_failed",
|
|
10311
|
+
retryable: false,
|
|
10312
|
+
recovery: "user_message",
|
|
10313
|
+
compacted: false,
|
|
10314
|
+
...rejection ? { providerRejection: rejection } : {},
|
|
10315
|
+
...quota ? { quotaScope: quota.scope } : {}
|
|
10316
|
+
};
|
|
10317
|
+
}
|
|
9394
10318
|
function isCompactionSummaryFailure(error) {
|
|
9395
10319
|
return error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError;
|
|
9396
10320
|
}
|
|
9397
10321
|
function shouldRecoverCompactionProviderFailure(error) {
|
|
9398
10322
|
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
9399
10323
|
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
10324
|
+
if (classifyCodexEncryptedArtifactRejection(error)) return true;
|
|
10325
|
+
if (classifyCodexEntitlementRejection(error)) return true;
|
|
9400
10326
|
return agentRunFailurePayload(error).retryable === true;
|
|
9401
10327
|
}
|
|
9402
10328
|
function classifyContextWindowOverflowError(error) {
|
|
10329
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
9403
10330
|
const fields = collectErrorStrings(error);
|
|
9404
10331
|
const matched = fields.find(
|
|
9405
10332
|
(value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value) || /exceeds?\s+(?:the\s+)?context\s+window/i.test(value) || /maximum\s+context\s+length/i.test(value) || /context\s+window[^.]*exceed/i.test(value)
|
|
@@ -9445,6 +10372,7 @@ function collectErrorStrings(value, seen = /* @__PURE__ */ new WeakSet()) {
|
|
|
9445
10372
|
}
|
|
9446
10373
|
seen.add(value);
|
|
9447
10374
|
const out = [];
|
|
10375
|
+
if (value instanceof ResponsesStreamingTerminalError) out.push(value.detail);
|
|
9448
10376
|
const record = value;
|
|
9449
10377
|
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
9450
10378
|
const field = record[key];
|
|
@@ -9462,8 +10390,11 @@ function isExactStatuslessUpstreamConnectivityMessage(message) {
|
|
|
9462
10390
|
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
9463
10391
|
}
|
|
9464
10392
|
function providerSafetyRefusalDiagnostic(error) {
|
|
10393
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
10394
|
+
return error.category === "safety" ? error.detail : void 0;
|
|
10395
|
+
}
|
|
9465
10396
|
return collectErrorStrings(error).find(
|
|
9466
|
-
(value) => /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
|
|
10397
|
+
(value) => /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy|misalignment_policy_violation)$/.test(
|
|
9467
10398
|
value
|
|
9468
10399
|
) || /\bthis request was blocked by our safety systems\b/i.test(value)
|
|
9469
10400
|
);
|
|
@@ -9472,6 +10403,9 @@ function isProviderSafetyRefusal(error) {
|
|
|
9472
10403
|
return providerSafetyRefusalDiagnostic(error) !== void 0;
|
|
9473
10404
|
}
|
|
9474
10405
|
function isTransientProviderError(error) {
|
|
10406
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
10407
|
+
return error.category === "unavailable";
|
|
10408
|
+
}
|
|
9475
10409
|
if (isProviderSafetyRefusal(error)) return false;
|
|
9476
10410
|
const status = typeof error === "object" && error !== null ? Number(
|
|
9477
10411
|
error.status ?? error.statusCode
|
|
@@ -9491,6 +10425,13 @@ function isTransientProviderError(error) {
|
|
|
9491
10425
|
message
|
|
9492
10426
|
);
|
|
9493
10427
|
}
|
|
10428
|
+
function hasCodexUsageLimitType(error) {
|
|
10429
|
+
return collectErrorStrings(error).some((value) => value.includes(CODEX_USAGE_LIMIT_ERROR_TYPE));
|
|
10430
|
+
}
|
|
10431
|
+
function classifyProviderQuotaExhaustionError(error) {
|
|
10432
|
+
if (isCodexTransportError(error) || isXaiSubscriptionTransportError(error)) return null;
|
|
10433
|
+
return classifyProviderQuotaError(error);
|
|
10434
|
+
}
|
|
9494
10435
|
function classifyXaiCredentialFailure(error) {
|
|
9495
10436
|
if (isProviderSafetyRefusal(error)) return null;
|
|
9496
10437
|
let relogin = error;
|
|
@@ -9545,12 +10486,50 @@ function findPostgresDriverError(error) {
|
|
|
9545
10486
|
}
|
|
9546
10487
|
return null;
|
|
9547
10488
|
}
|
|
10489
|
+
function isRawDatabaseQueryError(error) {
|
|
10490
|
+
return error instanceof Error && "query" in error && typeof error.query === "string" && "params" in error && Array.isArray(error.params);
|
|
10491
|
+
}
|
|
10492
|
+
function anthropicRequestDiagnostic(error) {
|
|
10493
|
+
if (error instanceof AnthropicRequestError) return error;
|
|
10494
|
+
return error instanceof Error && error.cause instanceof AnthropicRequestError ? error.cause : void 0;
|
|
10495
|
+
}
|
|
9548
10496
|
function agentRunFailurePayload(error, options = {}) {
|
|
10497
|
+
const failure = baseAgentRunFailurePayload(error, options);
|
|
10498
|
+
const diagnostic = materializationVerificationDiagnostic(error);
|
|
10499
|
+
const anthropic = anthropicRequestDiagnostic(error);
|
|
10500
|
+
if (anthropic) {
|
|
10501
|
+
const authenticationRejected = anthropic.status === 401 && (error === anthropic || error instanceof Error && error.status === 401);
|
|
10502
|
+
return {
|
|
10503
|
+
...failure,
|
|
10504
|
+
code: failure.code ?? (authenticationRejected ? "anthropic_authentication_error" : anthropic.code),
|
|
10505
|
+
retryable: failure.retryable ?? false,
|
|
10506
|
+
...anthropic.detail ? { detail: anthropic.detail } : {},
|
|
10507
|
+
...anthropic.request_id ? { requestId: anthropic.request_id } : {}
|
|
10508
|
+
};
|
|
10509
|
+
}
|
|
10510
|
+
return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
|
|
10511
|
+
}
|
|
10512
|
+
function agentRunRecoveryFailurePayload(error, failure) {
|
|
10513
|
+
if (!anthropicRequestDiagnostic(error)) return failure;
|
|
10514
|
+
const recovery = { ...failure };
|
|
10515
|
+
delete recovery.detail;
|
|
10516
|
+
return recovery;
|
|
10517
|
+
}
|
|
10518
|
+
function baseAgentRunFailurePayload(error, options = {}) {
|
|
10519
|
+
if (error instanceof SandboxMaterializationVerificationError) {
|
|
10520
|
+
return {
|
|
10521
|
+
error: error.message,
|
|
10522
|
+
code: error.code,
|
|
10523
|
+
retryable: false,
|
|
10524
|
+
materializationDiagnostic: error.diagnostic
|
|
10525
|
+
};
|
|
10526
|
+
}
|
|
9549
10527
|
if (error instanceof RetainedAttachmentTransportLimitError) {
|
|
9550
10528
|
return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
|
|
9551
10529
|
}
|
|
9552
10530
|
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
9553
|
-
const
|
|
10531
|
+
const databaseFailure = isSessionEventPersistenceError(error.cause) || findPostgresDriverError(error.cause) !== null || isRawDatabaseQueryError(error.cause);
|
|
10532
|
+
const underlying = databaseFailure ? agentRunFailurePayload(error.cause, options) : {
|
|
9554
10533
|
error: error.cause instanceof Error ? error.cause.message : String(error.cause)
|
|
9555
10534
|
};
|
|
9556
10535
|
return {
|
|
@@ -9558,6 +10537,29 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
9558
10537
|
historyPersistenceStage: error.stage
|
|
9559
10538
|
};
|
|
9560
10539
|
}
|
|
10540
|
+
const postgresDriverError = findPostgresDriverError(error);
|
|
10541
|
+
const rawOrmFailure = isRawDatabaseQueryError(error);
|
|
10542
|
+
if ((postgresDriverError || rawOrmFailure) && !isSessionEventPersistenceError(error)) {
|
|
10543
|
+
const database = safeDatabaseErrorFacts(postgresDriverError ?? error);
|
|
10544
|
+
const sqlState = postgresDriverError ? nestedPostgresSqlState(postgresDriverError) : null;
|
|
10545
|
+
return {
|
|
10546
|
+
error: "Opengeni encountered a database error.",
|
|
10547
|
+
code: databaseFailureCode(sqlState),
|
|
10548
|
+
sqlState,
|
|
10549
|
+
...Object.keys(database).length > 0 ? { database } : {}
|
|
10550
|
+
};
|
|
10551
|
+
}
|
|
10552
|
+
if (error instanceof ClaudeSubscriptionConnectionUnavailable) {
|
|
10553
|
+
return { error: error.message, code: error.code, retryable: false };
|
|
10554
|
+
}
|
|
10555
|
+
if (error instanceof AnthropicProviderRejection) {
|
|
10556
|
+
return {
|
|
10557
|
+
error: error.message,
|
|
10558
|
+
code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
|
|
10559
|
+
retryable: false,
|
|
10560
|
+
...error.request_id ? { requestId: error.request_id } : {}
|
|
10561
|
+
};
|
|
10562
|
+
}
|
|
9561
10563
|
const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
|
|
9562
10564
|
if (safetyRefusalDiagnostic !== void 0) {
|
|
9563
10565
|
return {
|
|
@@ -9567,6 +10569,28 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
9567
10569
|
detail: safetyRefusalDiagnostic
|
|
9568
10570
|
};
|
|
9569
10571
|
}
|
|
10572
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
10573
|
+
const quota2 = error.category === "unknown" || error.category === "rate_limit" ? classifyProviderQuotaExhaustionError({
|
|
10574
|
+
code: error.code,
|
|
10575
|
+
error: { code: error.code, type: error.type, message: error.detail },
|
|
10576
|
+
retryAfterSeconds: error.retryAfterSeconds
|
|
10577
|
+
}) : null;
|
|
10578
|
+
if (quota2) {
|
|
10579
|
+
return {
|
|
10580
|
+
error: providerQuotaExhaustedMessage(quota2.scope),
|
|
10581
|
+
code: PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
10582
|
+
retryable: false,
|
|
10583
|
+
quotaScope: quota2.scope,
|
|
10584
|
+
detail: error.detail
|
|
10585
|
+
};
|
|
10586
|
+
}
|
|
10587
|
+
return {
|
|
10588
|
+
error: error.category === "rate_limit" ? "Model provider rate limit hit. Try again in a minute or lower the reasoning effort." : error.category === "unavailable" ? "The model provider is temporarily unavailable. The same turn will retry after a short delay." : "The model provider rejected the response. Automatic retries stopped.",
|
|
10589
|
+
code: error.category === "rate_limit" ? "provider_rate_limited" : error.category === "unavailable" ? "provider_unavailable" : "provider_request_rejected",
|
|
10590
|
+
retryable: error.category === "rate_limit" || error.category === "unavailable",
|
|
10591
|
+
detail: error.detail
|
|
10592
|
+
};
|
|
10593
|
+
}
|
|
9570
10594
|
const message = error instanceof Error ? error.message : String(error);
|
|
9571
10595
|
const status = typeof error === "object" && error !== null ? Number(
|
|
9572
10596
|
error.status ?? error.statusCode
|
|
@@ -9595,9 +10619,23 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
9595
10619
|
retryable: true
|
|
9596
10620
|
};
|
|
9597
10621
|
}
|
|
9598
|
-
if (
|
|
10622
|
+
if (isProviderCommandObservationUnavailableError(error)) {
|
|
9599
10623
|
return {
|
|
9600
|
-
error: "
|
|
10624
|
+
error: "A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
|
|
10625
|
+
code: "sandbox_command_observation_unavailable",
|
|
10626
|
+
retryable: false
|
|
10627
|
+
};
|
|
10628
|
+
}
|
|
10629
|
+
if (isModalCommandStartOutcomeUnknownError(error)) {
|
|
10630
|
+
return {
|
|
10631
|
+
error: "A managed sandbox command has an unknown outcome. Its original invocation remains fenced; setup is blocked without replay until the incomplete operation can be reconciled.",
|
|
10632
|
+
code: "sandbox_command_start_outcome_unknown",
|
|
10633
|
+
retryable: false
|
|
10634
|
+
};
|
|
10635
|
+
}
|
|
10636
|
+
if (!isRoutingMutationOutcomeUnknownError(error) && isModalTaskExecStartPreDispatchUnavailableError(error)) {
|
|
10637
|
+
return {
|
|
10638
|
+
error: "The managed sandbox command router was not ready before the command was sent. The same turn will retry after a short delay.",
|
|
9601
10639
|
code: "sandbox_command_start_unavailable",
|
|
9602
10640
|
retryable: true
|
|
9603
10641
|
};
|
|
@@ -9653,8 +10691,26 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
9653
10691
|
...Object.keys(details.database).length > 0 ? { database: details.database } : {}
|
|
9654
10692
|
};
|
|
9655
10693
|
}
|
|
10694
|
+
if (error instanceof CodexPlanEntitlementError) {
|
|
10695
|
+
return error.payload;
|
|
10696
|
+
}
|
|
10697
|
+
const entitlementRejection = classifyCodexEntitlementRejection(error);
|
|
10698
|
+
if (entitlementRejection) {
|
|
10699
|
+
return entitlementRejection.evidence === "plan_entitlement" ? codexPlanEntitlementFailurePayload({
|
|
10700
|
+
accountLabel: null,
|
|
10701
|
+
planType: null,
|
|
10702
|
+
planChanged: false,
|
|
10703
|
+
modelId: null,
|
|
10704
|
+
rejection: entitlementRejection
|
|
10705
|
+
}) : codexRequestRejectedFailurePayload({
|
|
10706
|
+
accountLabel: null,
|
|
10707
|
+
planType: null,
|
|
10708
|
+
rejection: entitlementRejection,
|
|
10709
|
+
planChecked: false
|
|
10710
|
+
});
|
|
10711
|
+
}
|
|
9656
10712
|
const usageLimit = classifyCodexUsageLimitError(error);
|
|
9657
|
-
if (usageLimit) {
|
|
10713
|
+
if (usageLimit && (isCodexTransportError(error) || hasCodexUsageLimitType(error))) {
|
|
9658
10714
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
9659
10715
|
}
|
|
9660
10716
|
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
@@ -9699,7 +10755,25 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
9699
10755
|
retryable: true
|
|
9700
10756
|
};
|
|
9701
10757
|
}
|
|
9702
|
-
if (status ===
|
|
10758
|
+
if (status === 402 && code === "anthropic_billing_error") {
|
|
10759
|
+
return {
|
|
10760
|
+
error: "Claude could not bill this request. Check the account's billing and payment details.",
|
|
10761
|
+
code: "provider_billing_error",
|
|
10762
|
+
retryable: false,
|
|
10763
|
+
...message ? { detail: message } : {}
|
|
10764
|
+
};
|
|
10765
|
+
}
|
|
10766
|
+
const quota = classifyProviderQuotaExhaustionError(error);
|
|
10767
|
+
if (quota) {
|
|
10768
|
+
return {
|
|
10769
|
+
error: providerQuotaExhaustedMessage(quota.scope),
|
|
10770
|
+
code: PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
10771
|
+
retryable: false,
|
|
10772
|
+
quotaScope: quota.scope,
|
|
10773
|
+
...message ? { detail: message } : {}
|
|
10774
|
+
};
|
|
10775
|
+
}
|
|
10776
|
+
if (status === 429 || (status === void 0 || !Number.isFinite(status)) && (code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message))) {
|
|
9703
10777
|
return {
|
|
9704
10778
|
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
9705
10779
|
code: "provider_rate_limited",
|
|
@@ -9710,30 +10784,18 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
9710
10784
|
if (isTransientProviderError(error)) {
|
|
9711
10785
|
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
9712
10786
|
return {
|
|
9713
|
-
error: "
|
|
10787
|
+
error: "Opengeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
9714
10788
|
code: "upstream_connectivity_unavailable",
|
|
9715
10789
|
retryable: true
|
|
9716
10790
|
};
|
|
9717
10791
|
}
|
|
9718
10792
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
9719
10793
|
}
|
|
9720
|
-
const postgresDriverError = findPostgresDriverError(error);
|
|
9721
|
-
if (postgresDriverError) {
|
|
9722
|
-
const database = safeDatabaseErrorFacts(postgresDriverError);
|
|
9723
|
-
const sqlState = nestedPostgresSqlState(postgresDriverError);
|
|
9724
|
-
if (sqlState !== null || Object.keys(database).length > 0) {
|
|
9725
|
-
return {
|
|
9726
|
-
error: message,
|
|
9727
|
-
sqlState,
|
|
9728
|
-
...Object.keys(database).length > 0 ? { database } : {}
|
|
9729
|
-
};
|
|
9730
|
-
}
|
|
9731
|
-
}
|
|
9732
10794
|
return { error: message };
|
|
9733
10795
|
}
|
|
9734
10796
|
var CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 6e4;
|
|
9735
10797
|
function codexCredentialCooldownUntil(failure, account, now) {
|
|
9736
|
-
if (failure.kind === "auth" || failure.kind === "forbidden") {
|
|
10798
|
+
if (failure.kind === "auth" || failure.kind === "forbidden" || failure.kind === "plan_entitlement") {
|
|
9737
10799
|
return null;
|
|
9738
10800
|
}
|
|
9739
10801
|
const providerReset = failure.cooldownSeconds !== null && Number.isFinite(failure.cooldownSeconds) && failure.cooldownSeconds > 0 ? new Date(now.getTime() + Math.ceil(failure.cooldownSeconds) * 1e3) : null;
|
|
@@ -9766,6 +10828,9 @@ function classifyCodexCredentialFailure(error) {
|
|
|
9766
10828
|
if (!isCodexTransportError(error)) {
|
|
9767
10829
|
return null;
|
|
9768
10830
|
}
|
|
10831
|
+
if (classifyCodexEntitlementRejection(error)) {
|
|
10832
|
+
return null;
|
|
10833
|
+
}
|
|
9769
10834
|
const usageLimit = classifyCodexUsageLimitError(error);
|
|
9770
10835
|
if (usageLimit) {
|
|
9771
10836
|
return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
|
|
@@ -9825,6 +10890,28 @@ function codexUsageLimitFailurePayload(info, detail, opts) {
|
|
|
9825
10890
|
};
|
|
9826
10891
|
}
|
|
9827
10892
|
var CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 6e4;
|
|
10893
|
+
function classifyClaudeCredentialFailure(error) {
|
|
10894
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
10895
|
+
let current = error;
|
|
10896
|
+
for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
|
|
10897
|
+
if (current instanceof ClaudeSubscriptionReconnectRequired)
|
|
10898
|
+
return { kind: "auth", cooldownMs: null };
|
|
10899
|
+
if (current instanceof AnthropicRequestError && current.status === 401)
|
|
10900
|
+
return {
|
|
10901
|
+
kind: "auth",
|
|
10902
|
+
cooldownMs: null,
|
|
10903
|
+
...current.request_id ? { requestId: current.request_id } : {}
|
|
10904
|
+
};
|
|
10905
|
+
if (current instanceof AnthropicRequestError && current.status === 429)
|
|
10906
|
+
return {
|
|
10907
|
+
kind: "rate_limit",
|
|
10908
|
+
cooldownMs: providerRetryAfterMs(current) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
10909
|
+
...current.request_id ? { requestId: current.request_id } : {}
|
|
10910
|
+
};
|
|
10911
|
+
current = current.cause;
|
|
10912
|
+
}
|
|
10913
|
+
return null;
|
|
10914
|
+
}
|
|
9828
10915
|
|
|
9829
10916
|
export {
|
|
9830
10917
|
createSharedActivityServices,
|
|
@@ -9841,20 +10928,23 @@ export {
|
|
|
9841
10928
|
signalPendingCodexCapacityWakeTargets,
|
|
9842
10929
|
armAndReconcileCodexCapacityWait,
|
|
9843
10930
|
createCodexCapacityActivities,
|
|
10931
|
+
agentRunAdmissionDenial,
|
|
9844
10932
|
createDocumentActivities,
|
|
10933
|
+
turnCredentialRestriction,
|
|
9845
10934
|
createGoalActivities,
|
|
9846
10935
|
withFirstPartyTools,
|
|
9847
10936
|
createKnowledgeSourceSyncActivities,
|
|
9848
10937
|
rigProviderImageSourceImage,
|
|
9849
|
-
settingsWithPackSandboxImage,
|
|
9850
|
-
resolveWorkspacePackRuntime,
|
|
9851
10938
|
createRuntimeBatcher,
|
|
9852
10939
|
currentActivityContext,
|
|
9853
10940
|
startActivityHeartbeat,
|
|
9854
10941
|
nextStreamEvent,
|
|
10942
|
+
warnDrainSnapshotFailure,
|
|
10943
|
+
publishDurableSessionEvents2 as publishDurableSessionEvents,
|
|
9855
10944
|
persistWorkspaceArchiveCandidate,
|
|
9856
10945
|
putVersion1TarArchiveOrInline,
|
|
9857
10946
|
sandboxLeaseHolderIdForAttempt,
|
|
10947
|
+
createFreshSandboxReadinessReplacementBudget,
|
|
9858
10948
|
waitForWarmSnapshot,
|
|
9859
10949
|
maybePersistWarmWorkspaceSnapshot,
|
|
9860
10950
|
resumeBoxForTurn,
|
|
@@ -9880,9 +10970,17 @@ export {
|
|
|
9880
10970
|
incidentTelemetryAuthorityFence,
|
|
9881
10971
|
resolveIncidentTelemetryResponderMetadata,
|
|
9882
10972
|
validateIncidentTelemetrySystemUpdateAuthority,
|
|
10973
|
+
checkpointHistoryBeforeProviderDispatch,
|
|
10974
|
+
createClaudeUsageObserver,
|
|
9883
10975
|
projectRejectedProviderArtifacts,
|
|
9884
10976
|
createModelHistoryAttachmentProjector,
|
|
9885
10977
|
turnInput,
|
|
10978
|
+
codexAccountDisplayLabel,
|
|
10979
|
+
codexPlanEntitlementFailurePayload,
|
|
10980
|
+
codexRequestRejectedFailurePayload,
|
|
10981
|
+
CodexPlanEntitlementError,
|
|
10982
|
+
assessCodexPlanEntitlement,
|
|
10983
|
+
codexPlanEntitlementAdmissionBlock,
|
|
9886
10984
|
SandboxDeadlineRotationError,
|
|
9887
10985
|
turnOperationCancellationFailure,
|
|
9888
10986
|
throwIfTurnOperationCancelled,
|
|
@@ -9920,19 +11018,23 @@ export {
|
|
|
9920
11018
|
postClaimDatabaseRecoveryFailure,
|
|
9921
11019
|
isWorkerShutdownCancellation,
|
|
9922
11020
|
sandboxLifecycleTransitionDiagnostic,
|
|
11021
|
+
modelPreparationFailureEventPayload,
|
|
9923
11022
|
sandboxRouteTransitionCode,
|
|
9924
11023
|
compactionFailureReason,
|
|
9925
11024
|
safeErrorDiagnostic,
|
|
9926
11025
|
safeErrorForTelemetry,
|
|
9927
11026
|
compactionFailureReasonFromError,
|
|
11027
|
+
compactionFailureTurnEventPayload,
|
|
9928
11028
|
isCompactionSummaryFailure,
|
|
9929
11029
|
shouldRecoverCompactionProviderFailure,
|
|
9930
11030
|
classifyContextWindowOverflowError,
|
|
9931
11031
|
classifyXaiCredentialFailure,
|
|
9932
11032
|
agentRunFailurePayload,
|
|
11033
|
+
agentRunRecoveryFailurePayload,
|
|
9933
11034
|
codexCredentialCooldownUntil,
|
|
9934
11035
|
classifyCodexCredentialFailure,
|
|
9935
11036
|
codexUsageLimitFailurePayload,
|
|
9936
|
-
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
11037
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
11038
|
+
classifyClaudeCredentialFailure
|
|
9937
11039
|
};
|
|
9938
|
-
//# sourceMappingURL=chunk-
|
|
11040
|
+
//# sourceMappingURL=chunk-LJX7Z2KP.js.map
|