@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,4698 @@
|
|
|
1
|
+
import {
|
|
2
|
+
observabilityEventLogger
|
|
3
|
+
} from "./chunk-CE7TPKBS.js";
|
|
4
|
+
|
|
5
|
+
// src/activity-services.ts
|
|
6
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
7
|
+
import { configureChildLifecycleNotices, createDb } from "@opengeni/db";
|
|
8
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
9
|
+
import { createObservability } from "@opengeni/observability";
|
|
10
|
+
import { createObjectStorage } from "@opengeni/storage";
|
|
11
|
+
|
|
12
|
+
// src/personal-github-git-credentials.ts
|
|
13
|
+
import {
|
|
14
|
+
isPersonalGitHubConnection,
|
|
15
|
+
PersonalGitHubConnectionMetadata
|
|
16
|
+
} from "@opengeni/contracts/personal-github";
|
|
17
|
+
import {
|
|
18
|
+
getConnectionMetadata,
|
|
19
|
+
getPersonalGitHubRepositorySelectionState,
|
|
20
|
+
getSessionAuthorityProjection,
|
|
21
|
+
getSessionTurnForAttempt,
|
|
22
|
+
resolveAcceptedConnectionUse
|
|
23
|
+
} from "@opengeni/db";
|
|
24
|
+
import {
|
|
25
|
+
PERSONAL_GITHUB_GIT_BROKER_TOKEN_TTL_SECONDS,
|
|
26
|
+
personalGitHubGitBrokerRouteId,
|
|
27
|
+
sealPersonalGitHubGitBrokerClaims
|
|
28
|
+
} from "@opengeni/github";
|
|
29
|
+
import { randomBytes, randomUUID } from "crypto";
|
|
30
|
+
import { isDeepStrictEqual } from "util";
|
|
31
|
+
var PERSONAL_GITHUB_SERVER_ID = "github:personal";
|
|
32
|
+
function buildPersonalGitHubGitCredentials(db, settings) {
|
|
33
|
+
if (!settings.githubPersonalOauthEnabled) return null;
|
|
34
|
+
const secret = settings.integrationsStateSecret?.trim();
|
|
35
|
+
const publicOrigin = personalGitBrokerOrigin(settings.publicBaseUrl);
|
|
36
|
+
if (!secret || !publicOrigin) {
|
|
37
|
+
throw new Error("personal GitHub Git broker configuration is unavailable");
|
|
38
|
+
}
|
|
39
|
+
return async (request) => {
|
|
40
|
+
if (request.provider && request.provider !== "github") {
|
|
41
|
+
throw new Error("personal GitHub Git broker accepts only GitHub repositories");
|
|
42
|
+
}
|
|
43
|
+
if (!request.credentialBindingId || !request.repositoryRefs?.length) {
|
|
44
|
+
throw new Error("personal GitHub Git broker requires one exact credential binding");
|
|
45
|
+
}
|
|
46
|
+
const turn = await getSessionTurnForAttempt(
|
|
47
|
+
db,
|
|
48
|
+
request.workspaceId,
|
|
49
|
+
request.sessionId,
|
|
50
|
+
request.attemptId
|
|
51
|
+
);
|
|
52
|
+
const session = await getSessionAuthorityProjection(db, request.workspaceId, request.sessionId);
|
|
53
|
+
if (!turn || !session || turn.id !== request.turnId || turn.executionGeneration !== request.executionGeneration || !isDeepStrictEqual(turn.initiator, request.initiator) || !isDeepStrictEqual(turn.initiatorContext, request.initiatorContext) || session.rootSessionId !== request.rootSessionId) {
|
|
54
|
+
throw new Error("personal GitHub Git credential authority is no longer current");
|
|
55
|
+
}
|
|
56
|
+
const delegation = exactPersonalGitHubDelegation(
|
|
57
|
+
turn.personalConnectionDelegations,
|
|
58
|
+
request.credentialBindingId
|
|
59
|
+
);
|
|
60
|
+
const snapshot = delegation.personalGitHubRepositorySelection;
|
|
61
|
+
const repositories = exactRequestedRepositories(
|
|
62
|
+
request.repositoryRefs,
|
|
63
|
+
snapshot.credentialBindingId,
|
|
64
|
+
snapshot.repositories
|
|
65
|
+
);
|
|
66
|
+
const resolution = await resolveAcceptedConnectionUse(db, {
|
|
67
|
+
accountId: request.accountId,
|
|
68
|
+
workspaceId: request.workspaceId,
|
|
69
|
+
sessionId: request.sessionId,
|
|
70
|
+
turnId: request.turnId,
|
|
71
|
+
attemptId: request.attemptId,
|
|
72
|
+
executionGeneration: request.executionGeneration,
|
|
73
|
+
physicalRequestId: randomUUID(),
|
|
74
|
+
usePhase: "credential_resolution",
|
|
75
|
+
serverId: PERSONAL_GITHUB_SERVER_ID,
|
|
76
|
+
connectionId: delegation.connectionId,
|
|
77
|
+
providerDomain: "github.com",
|
|
78
|
+
connectionKind: "oauth2",
|
|
79
|
+
subjectScope: "subject",
|
|
80
|
+
ownerSubjectId: delegation.ownerSubjectId
|
|
81
|
+
});
|
|
82
|
+
if (resolution.status !== "authorized" || resolution.originWorkspaceId !== delegation.originWorkspaceId || resolution.attribution.connectionId !== delegation.connectionId || resolution.attribution.connectionGeneration !== snapshot.connectionAuthorityGeneration || resolution.attribution.ownerSubjectId !== delegation.ownerSubjectId) {
|
|
83
|
+
throw new Error("personal GitHub Git connection authority is no longer current");
|
|
84
|
+
}
|
|
85
|
+
const connection = await getConnectionMetadata(
|
|
86
|
+
db,
|
|
87
|
+
delegation.originWorkspaceId,
|
|
88
|
+
delegation.connectionId,
|
|
89
|
+
delegation.ownerSubjectId
|
|
90
|
+
);
|
|
91
|
+
if (!connection || connection.accountId !== request.accountId || connection.workspaceId !== delegation.originWorkspaceId || connection.subjectId !== delegation.ownerSubjectId || connection.status !== "active" || // connections.version is credential-refresh CAS, not use authority.
|
|
92
|
+
// resolveAcceptedConnectionUse above fences the independent common
|
|
93
|
+
// connection-authority generation carried by the accepted snapshot.
|
|
94
|
+
connection.grantedScopes.length !== 1 || connection.grantedScopes[0] !== "repo" || !isPersonalGitHubConnection(connection)) {
|
|
95
|
+
throw new Error("personal GitHub Git connection is unavailable");
|
|
96
|
+
}
|
|
97
|
+
const metadata = PersonalGitHubConnectionMetadata.parse(connection.metadata);
|
|
98
|
+
const currentSelection = await getPersonalGitHubRepositorySelectionState(db, {
|
|
99
|
+
accountId: request.accountId,
|
|
100
|
+
originWorkspaceId: delegation.originWorkspaceId,
|
|
101
|
+
subjectId: delegation.ownerSubjectId,
|
|
102
|
+
connectionId: delegation.connectionId
|
|
103
|
+
});
|
|
104
|
+
if (!currentSelection || currentSelection.connectionAuthorityGeneration !== snapshot.connectionAuthorityGeneration || currentSelection.selectionGeneration !== snapshot.selectionGeneration || currentSelection.credentialBindingId !== snapshot.credentialBindingId || currentSelection.credentialBindingId !== metadata.credentialBindingId || currentSelection.providerPrincipalId !== metadata.providerPrincipalId || metadata.githubUserId !== metadata.providerPrincipalId) {
|
|
105
|
+
throw new Error("personal GitHub repository authority changed");
|
|
106
|
+
}
|
|
107
|
+
const currentById = new Map(
|
|
108
|
+
currentSelection.repositories.map((repository) => [repository.repositoryId, repository])
|
|
109
|
+
);
|
|
110
|
+
for (const repository of repositories) {
|
|
111
|
+
const current = currentById.get(repository.repositoryId);
|
|
112
|
+
if (!current || current.fullName !== repository.fullName || current.canonicalUrl !== repository.canonicalUrl || current.selectionGeneration !== repository.selectionGeneration || current.disabled || repository.access === "write" && (current.selectedAccess !== "write" || current.archived || !personalGitHubCanWrite(current.permissions)) || !personalGitHubCanRead(current.permissions)) {
|
|
113
|
+
throw new Error("personal GitHub repository authority changed");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const identity = {
|
|
117
|
+
name: metadata.githubLogin,
|
|
118
|
+
email: `${metadata.githubUserId}+${metadata.githubLogin}@users.noreply.github.com`
|
|
119
|
+
};
|
|
120
|
+
if (request.purpose === "identity") {
|
|
121
|
+
return {
|
|
122
|
+
workspaceId: request.workspaceId,
|
|
123
|
+
credentialBindingId: request.credentialBindingId,
|
|
124
|
+
provider: "github",
|
|
125
|
+
providerHost: "github.com",
|
|
126
|
+
identity
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
130
|
+
const baseClaims = {
|
|
131
|
+
version: 1,
|
|
132
|
+
accountId: request.accountId,
|
|
133
|
+
workspaceId: request.workspaceId,
|
|
134
|
+
sessionId: request.sessionId,
|
|
135
|
+
rootSessionId: request.rootSessionId,
|
|
136
|
+
turnId: request.turnId,
|
|
137
|
+
attemptId: request.attemptId,
|
|
138
|
+
executionGeneration: request.executionGeneration,
|
|
139
|
+
originWorkspaceId: delegation.originWorkspaceId,
|
|
140
|
+
connectionId: delegation.connectionId,
|
|
141
|
+
connectionAuthorityGeneration: snapshot.connectionAuthorityGeneration,
|
|
142
|
+
ownerSubjectId: delegation.ownerSubjectId,
|
|
143
|
+
credentialBindingId: snapshot.credentialBindingId,
|
|
144
|
+
selectionGeneration: snapshot.selectionGeneration
|
|
145
|
+
};
|
|
146
|
+
const repositoryClaims = repositories.map(
|
|
147
|
+
(repository) => ({
|
|
148
|
+
...repository,
|
|
149
|
+
routeId: personalGitHubGitBrokerRouteId(secret, {
|
|
150
|
+
...baseClaims,
|
|
151
|
+
repository
|
|
152
|
+
})
|
|
153
|
+
})
|
|
154
|
+
);
|
|
155
|
+
const claims = {
|
|
156
|
+
...baseClaims,
|
|
157
|
+
nonce: randomBytes(24).toString("base64url"),
|
|
158
|
+
issuedAt: now,
|
|
159
|
+
expiresAt: now + PERSONAL_GITHUB_GIT_BROKER_TOKEN_TTL_SECONDS
|
|
160
|
+
};
|
|
161
|
+
return {
|
|
162
|
+
token: sealPersonalGitHubGitBrokerClaims(secret, claims),
|
|
163
|
+
expiresAt: new Date(claims.expiresAt * 1e3).toISOString(),
|
|
164
|
+
workspaceId: request.workspaceId,
|
|
165
|
+
credentialBindingId: request.credentialBindingId,
|
|
166
|
+
provider: "github",
|
|
167
|
+
providerHost: "github.com",
|
|
168
|
+
identity,
|
|
169
|
+
transport: {
|
|
170
|
+
kind: "http_broker",
|
|
171
|
+
repositories: repositoryClaims.map((repository) => ({
|
|
172
|
+
repositoryUri: repository.canonicalUrl,
|
|
173
|
+
brokerUri: `${publicOrigin}/v1/git/personal/${repository.routeId}`
|
|
174
|
+
}))
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function personalGitHubCanRead(permissions) {
|
|
180
|
+
return permissions.pull || permissions.triage || permissions.push || permissions.maintain || permissions.admin;
|
|
181
|
+
}
|
|
182
|
+
function personalGitHubCanWrite(permissions) {
|
|
183
|
+
return permissions.push || permissions.maintain || permissions.admin;
|
|
184
|
+
}
|
|
185
|
+
function personalGitBrokerOrigin(value) {
|
|
186
|
+
if (!value) return null;
|
|
187
|
+
const url = new URL(value);
|
|
188
|
+
if (url.protocol !== "https:" || url.username || url.password || url.pathname !== "/" || url.search || url.hash) {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
return url.origin;
|
|
192
|
+
}
|
|
193
|
+
function exactPersonalGitHubDelegation(delegations, credentialBindingId) {
|
|
194
|
+
const matches = delegations.filter(
|
|
195
|
+
(delegation2) => delegation2.serverId === PERSONAL_GITHUB_SERVER_ID && delegation2.connectionType === "github_personal" && delegation2.personalGitHubRepositorySelection?.credentialBindingId === credentialBindingId
|
|
196
|
+
);
|
|
197
|
+
const delegation = matches[0];
|
|
198
|
+
if (matches.length !== 1 || !delegation || delegation.providerDomain !== "github.com" || delegation.kind !== "oauth2" || !delegation.originWorkspaceId || !delegation.userDelegation || !delegation.personalGitHubRepositorySelection) {
|
|
199
|
+
throw new Error("personal GitHub Git delegation is unavailable");
|
|
200
|
+
}
|
|
201
|
+
return delegation;
|
|
202
|
+
}
|
|
203
|
+
function exactRequestedRepositories(refs, credentialBindingId, snapshots) {
|
|
204
|
+
if (refs.length !== snapshots.length) {
|
|
205
|
+
throw new Error("personal GitHub Git request does not match the accepted repository set");
|
|
206
|
+
}
|
|
207
|
+
const byCanonicalUrl = new Map(
|
|
208
|
+
snapshots.map((repository) => [repository.canonicalUrl, repository])
|
|
209
|
+
);
|
|
210
|
+
const seen = /* @__PURE__ */ new Set();
|
|
211
|
+
return refs.map((ref) => {
|
|
212
|
+
const snapshot = byCanonicalUrl.get(ref.uri);
|
|
213
|
+
if (!snapshot || seen.has(ref.uri) || ref.provider !== "github" || ref.credentialBindingId !== credentialBindingId || ref.connectionId !== void 0 || ref.installationId !== void 0 || String(ref.repositoryId) !== snapshot.repositoryId || ref.ref !== snapshot.ref || ref.access !== snapshot.access) {
|
|
214
|
+
throw new Error("personal GitHub Git request exceeds the accepted repository authority");
|
|
215
|
+
}
|
|
216
|
+
seen.add(ref.uri);
|
|
217
|
+
return snapshot;
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// src/pr-review-credentials.ts
|
|
222
|
+
import { environmentsEncryptionKeyBytes } from "@opengeni/config";
|
|
223
|
+
import { prReviewRegistrationIdFromCredentialBinding } from "@opengeni/core";
|
|
224
|
+
import { decryptVariableSetValue, resolvePrReviewGitCredential } from "@opengeni/db";
|
|
225
|
+
import {
|
|
226
|
+
createGitHubAppInstallationTokenWithExpiry,
|
|
227
|
+
createGitHubAppInstallationTokenWithSigningSettings
|
|
228
|
+
} from "@opengeni/github";
|
|
229
|
+
function createStandaloneConnectionCredentialsPort(settings, db) {
|
|
230
|
+
return {
|
|
231
|
+
gitCredentials: async (request) => await resolveStandaloneGitCredentials(settings, db, request)
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
async function resolveStandaloneGitCredentials(settings, db, request) {
|
|
235
|
+
const provider = request.provider ?? "github";
|
|
236
|
+
const registrationId = request.credentialBindingId ? prReviewRegistrationIdFromCredentialBinding(request.credentialBindingId) : null;
|
|
237
|
+
if (!registrationId) {
|
|
238
|
+
if (provider !== "github") {
|
|
239
|
+
throw new Error(`${provider} Git credentials require an explicit PR Review or host binding`);
|
|
240
|
+
}
|
|
241
|
+
if (request.purpose === "identity") {
|
|
242
|
+
return { workspaceId: request.workspaceId };
|
|
243
|
+
}
|
|
244
|
+
const minted = await createGitHubAppInstallationTokenWithExpiry(settings, {
|
|
245
|
+
installationId: request.installationId,
|
|
246
|
+
repositoryIds: request.repositoryIds
|
|
247
|
+
});
|
|
248
|
+
return {
|
|
249
|
+
token: minted.token,
|
|
250
|
+
workspaceId: request.workspaceId,
|
|
251
|
+
...minted.expiresAt ? { expiresAt: minted.expiresAt } : {}
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
const authority = await resolvePrReviewGitCredential(db, {
|
|
255
|
+
accountId: request.accountId,
|
|
256
|
+
workspaceId: request.workspaceId,
|
|
257
|
+
registrationId,
|
|
258
|
+
provider,
|
|
259
|
+
sessionId: request.sessionId,
|
|
260
|
+
rootSessionId: request.rootSessionId,
|
|
261
|
+
turnId: request.turnId,
|
|
262
|
+
attemptId: request.attemptId,
|
|
263
|
+
executionGeneration: request.executionGeneration,
|
|
264
|
+
repositoryRefs: (request.repositoryRefs ?? []).map((reference) => ({
|
|
265
|
+
uri: reference.uri,
|
|
266
|
+
...reference.expectedCommitSha !== void 0 ? { expectedCommitSha: reference.expectedCommitSha } : {},
|
|
267
|
+
...reference.repositoryId !== void 0 ? { repositoryId: reference.repositoryId } : {},
|
|
268
|
+
...reference.installationId !== void 0 ? { installationId: reference.installationId } : {},
|
|
269
|
+
...reference.projectId !== void 0 ? { projectId: reference.projectId } : {}
|
|
270
|
+
}))
|
|
271
|
+
});
|
|
272
|
+
const echoes = {
|
|
273
|
+
workspaceId: request.workspaceId,
|
|
274
|
+
...request.credentialBindingId ? { credentialBindingId: request.credentialBindingId } : {},
|
|
275
|
+
provider,
|
|
276
|
+
...request.providerHost ? { providerHost: request.providerHost } : {}
|
|
277
|
+
};
|
|
278
|
+
if (request.purpose === "identity") {
|
|
279
|
+
return authority.credentialKind === "github_app" ? {
|
|
280
|
+
...echoes,
|
|
281
|
+
identity: {
|
|
282
|
+
name: "pr-review[bot]",
|
|
283
|
+
email: `${authority.appId ?? "pr-review"}+pr-review[bot]@users.noreply.github.com`
|
|
284
|
+
}
|
|
285
|
+
} : echoes;
|
|
286
|
+
}
|
|
287
|
+
if (authority.credentialKind === "github_app") {
|
|
288
|
+
assertGitHubMintMatchesRepositoryRefs(request);
|
|
289
|
+
const encryptionKey2 = environmentsEncryptionKeyBytes(settings);
|
|
290
|
+
if (!encryptionKey2 || !authority.appId || !authority.credentialEncrypted) {
|
|
291
|
+
throw new Error("PR Review GitHub App credential is unavailable");
|
|
292
|
+
}
|
|
293
|
+
const minted = await createGitHubAppInstallationTokenWithSigningSettings(
|
|
294
|
+
{
|
|
295
|
+
githubAppId: authority.appId,
|
|
296
|
+
githubAppPrivateKey: decryptVariableSetValue(encryptionKey2, authority.credentialEncrypted)
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
installationId: request.installationId,
|
|
300
|
+
repositoryIds: request.repositoryIds
|
|
301
|
+
}
|
|
302
|
+
);
|
|
303
|
+
return {
|
|
304
|
+
...echoes,
|
|
305
|
+
token: minted.token,
|
|
306
|
+
...minted.expiresAt ? { expiresAt: minted.expiresAt } : {}
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
if (authority.expiresAt && Date.parse(authority.expiresAt) <= Date.now()) {
|
|
310
|
+
throw new Error(`PR Review ${provider} provider credential has expired`);
|
|
311
|
+
}
|
|
312
|
+
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
313
|
+
if (!encryptionKey || !authority.credentialEncrypted) {
|
|
314
|
+
throw new Error(`PR Review ${provider} provider credential is unavailable`);
|
|
315
|
+
}
|
|
316
|
+
return {
|
|
317
|
+
...echoes,
|
|
318
|
+
token: decryptVariableSetValue(encryptionKey, authority.credentialEncrypted),
|
|
319
|
+
...authority.expiresAt ? { expiresAt: authority.expiresAt } : {}
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
function assertGitHubMintMatchesRepositoryRefs(request) {
|
|
323
|
+
const refs = request.repositoryRefs ?? [];
|
|
324
|
+
const repositoryIds = refs.map((reference) => positiveInteger(reference.repositoryId));
|
|
325
|
+
const installationIds = refs.map((reference) => positiveInteger(reference.installationId));
|
|
326
|
+
if (repositoryIds.some((value) => value === null) || installationIds.some((value) => value === null)) {
|
|
327
|
+
throw new Error("PR Review GitHub credential request lacks exact repository authority");
|
|
328
|
+
}
|
|
329
|
+
const exactRepositoryIds = [...new Set(repositoryIds)].sort(
|
|
330
|
+
(left, right) => left - right
|
|
331
|
+
);
|
|
332
|
+
const requestedRepositoryIds = [...new Set(request.repositoryIds)].sort(
|
|
333
|
+
(left, right) => left - right
|
|
334
|
+
);
|
|
335
|
+
const exactInstallationIds = [...new Set(installationIds)];
|
|
336
|
+
if (exactInstallationIds.length !== 1 || exactInstallationIds[0] !== request.installationId || exactRepositoryIds.length !== request.repositoryIds.length || exactRepositoryIds.some((value, index) => value !== requestedRepositoryIds[index])) {
|
|
337
|
+
throw new Error(
|
|
338
|
+
"PR Review GitHub token mint authority does not match its repository resources"
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
function positiveInteger(value) {
|
|
343
|
+
if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) return value;
|
|
344
|
+
if (typeof value === "string" && /^\d+$/.test(value)) {
|
|
345
|
+
const parsed = Number(value);
|
|
346
|
+
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
|
|
347
|
+
}
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// src/activity-services.ts
|
|
352
|
+
function createSharedActivityServices(dependencies) {
|
|
353
|
+
let servicesPromise = null;
|
|
354
|
+
return async function services() {
|
|
355
|
+
servicesPromise ??= (async () => {
|
|
356
|
+
const settings = dependencies.settings ?? getSettings();
|
|
357
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
358
|
+
const searchPath = dbSearchPath(settings);
|
|
359
|
+
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
360
|
+
...searchPath ? { searchPath } : {},
|
|
361
|
+
rlsStrategy: settings.rlsStrategy
|
|
362
|
+
});
|
|
363
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
364
|
+
const db = dependencies.db ?? dbClient.db;
|
|
365
|
+
configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
|
|
366
|
+
return {
|
|
367
|
+
settings,
|
|
368
|
+
db,
|
|
369
|
+
bus: dependencies.bus ?? await createNatsEventBus(
|
|
370
|
+
settings.natsUrl,
|
|
371
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
372
|
+
{ logger: observabilityEventLogger(observability) }
|
|
373
|
+
),
|
|
374
|
+
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
375
|
+
observability,
|
|
376
|
+
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
377
|
+
signalSessionAttemptQuiesced: dependencies.signalSessionAttemptQuiesced ?? null,
|
|
378
|
+
inspectSessionAttemptActivity: dependencies.inspectSessionAttemptActivity ?? null,
|
|
379
|
+
signalCodexCapacityWorkflow: dependencies.signalCodexCapacityWorkflow ?? null,
|
|
380
|
+
startSandboxReaperWorkflow: dependencies.startSandboxReaperWorkflow ?? null,
|
|
381
|
+
startVideoGenerationWorkflow: dependencies.startVideoGenerationWorkflow ?? null,
|
|
382
|
+
entitlements: dependencies.entitlements ?? null,
|
|
383
|
+
connectionCredentials: dependencies.connectionCredentials ?? createStandaloneConnectionCredentialsPort(settings, db),
|
|
384
|
+
personalGitHubCredentials: dependencies.personalGitHubCredentials ?? buildPersonalGitHubGitCredentials(db, settings)
|
|
385
|
+
};
|
|
386
|
+
})();
|
|
387
|
+
return await servicesPromise;
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// src/activities/codex-capacity.ts
|
|
392
|
+
import {
|
|
393
|
+
fetchCodexUsageForAccount,
|
|
394
|
+
getCodexCapacityWaitForSession,
|
|
395
|
+
getXaiCapacityWaitForSession,
|
|
396
|
+
listCodexAccountStatuses,
|
|
397
|
+
listPendingCodexCapacityWakeTargets,
|
|
398
|
+
reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
|
|
399
|
+
reconcileXaiCapacityWait as reconcileXaiCapacityWaitDb
|
|
400
|
+
} from "@opengeni/db";
|
|
401
|
+
|
|
402
|
+
// src/activities/codex-rotation.ts
|
|
403
|
+
function effectiveRotationStrategy(_stored) {
|
|
404
|
+
return "sharded";
|
|
405
|
+
}
|
|
406
|
+
function classifyCodexPin(args) {
|
|
407
|
+
const { pinnedCredentialId, pinSource, strategy, rotationEnabled } = args;
|
|
408
|
+
const pinned = pinnedCredentialId != null;
|
|
409
|
+
if (pinned && pinSource !== "policy") {
|
|
410
|
+
return "manual";
|
|
411
|
+
}
|
|
412
|
+
const shardedActive = rotationEnabled && effectiveRotationStrategy(strategy) === "sharded";
|
|
413
|
+
if (shardedActive) {
|
|
414
|
+
return "sharded";
|
|
415
|
+
}
|
|
416
|
+
if (pinned && pinSource === "policy") {
|
|
417
|
+
return "clearStale";
|
|
418
|
+
}
|
|
419
|
+
return "unpinned";
|
|
420
|
+
}
|
|
421
|
+
var MIN_IDLE_MS = 6e4;
|
|
422
|
+
var DEFAULT_RESET_COOLDOWN_MS = 6e4;
|
|
423
|
+
var CODEX_USAGE_EXHAUSTED_PCT = 100;
|
|
424
|
+
function effectiveWindowUsed(usedPercent, resetAt, now) {
|
|
425
|
+
return resetAt && resetAt.getTime() <= now.getTime() ? 0 : usedPercent ?? 0;
|
|
426
|
+
}
|
|
427
|
+
function bindingUsedPct(acct, now) {
|
|
428
|
+
return Math.max(
|
|
429
|
+
effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now),
|
|
430
|
+
effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now)
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
function bindingRemaining(acct, now) {
|
|
434
|
+
const primaryRemaining = 100 - effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now);
|
|
435
|
+
const secondaryRemaining = 100 - effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now);
|
|
436
|
+
return Math.min(primaryRemaining, secondaryRemaining);
|
|
437
|
+
}
|
|
438
|
+
function cooling(acct, now) {
|
|
439
|
+
return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
|
|
440
|
+
}
|
|
441
|
+
function isCodexCredentialHealthy(acct, now) {
|
|
442
|
+
return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct, now) < CODEX_USAGE_EXHAUSTED_PCT;
|
|
443
|
+
}
|
|
444
|
+
function isCodexCredentialEligible(acct, now) {
|
|
445
|
+
return acct.allocatorEnabled && isCodexCredentialHealthy(acct, now);
|
|
446
|
+
}
|
|
447
|
+
function fnv1a32(input) {
|
|
448
|
+
let hash = 2166136261;
|
|
449
|
+
for (let i = 0; i < input.length; i++) {
|
|
450
|
+
hash ^= input.charCodeAt(i);
|
|
451
|
+
hash = Math.imul(hash, 16777619);
|
|
452
|
+
}
|
|
453
|
+
return hash >>> 0;
|
|
454
|
+
}
|
|
455
|
+
function shardCredentialForSession(args) {
|
|
456
|
+
const { sessionId, accounts, now } = args;
|
|
457
|
+
const eligibles = accounts.filter((acct) => isCodexCredentialEligible(acct, now));
|
|
458
|
+
if (eligibles.length === 0) {
|
|
459
|
+
return null;
|
|
460
|
+
}
|
|
461
|
+
const index = fnv1a32(sessionId) % eligibles.length;
|
|
462
|
+
return eligibles[index].id;
|
|
463
|
+
}
|
|
464
|
+
function earliestCodexReset(accounts, now) {
|
|
465
|
+
return earliestReset(accounts, now);
|
|
466
|
+
}
|
|
467
|
+
function chooseShardedHome(args) {
|
|
468
|
+
const { sessionId, currentPolicyPin, accounts, now } = args;
|
|
469
|
+
const pinRow = currentPolicyPin ? accounts.find((acct) => acct.id === currentPolicyPin) ?? null : null;
|
|
470
|
+
if (pinRow && isCodexCredentialEligible(pinRow, now)) {
|
|
471
|
+
return { kind: "home", credentialId: currentPolicyPin, rewritePin: false };
|
|
472
|
+
}
|
|
473
|
+
const home = shardCredentialForSession({ sessionId, accounts, now });
|
|
474
|
+
if (home == null) {
|
|
475
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, now) };
|
|
476
|
+
}
|
|
477
|
+
return { kind: "home", credentialId: home, rewritePin: true };
|
|
478
|
+
}
|
|
479
|
+
function availableAt(acct, now) {
|
|
480
|
+
const nowMs = now.getTime();
|
|
481
|
+
const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
|
|
482
|
+
const candidates = [];
|
|
483
|
+
if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
|
|
484
|
+
candidates.push(acct.exhaustedUntil);
|
|
485
|
+
}
|
|
486
|
+
const windowClear = (over, resetAt) => {
|
|
487
|
+
if (!over) return;
|
|
488
|
+
candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
|
|
489
|
+
};
|
|
490
|
+
windowClear(
|
|
491
|
+
effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
|
|
492
|
+
acct.primaryResetAt
|
|
493
|
+
);
|
|
494
|
+
windowClear(
|
|
495
|
+
effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
|
|
496
|
+
acct.secondaryResetAt
|
|
497
|
+
);
|
|
498
|
+
if (candidates.length === 0) {
|
|
499
|
+
return defaultClear;
|
|
500
|
+
}
|
|
501
|
+
return candidates.reduce((a, b) => b.getTime() > a.getTime() ? b : a);
|
|
502
|
+
}
|
|
503
|
+
function earliestReset(accounts, now) {
|
|
504
|
+
return accounts.map((acct) => availableAt(acct, now)).reduce((a, b) => b.getTime() < a.getTime() ? b : a);
|
|
505
|
+
}
|
|
506
|
+
function authoritativeCodexCapacityResetAt(accounts, now) {
|
|
507
|
+
const future = [];
|
|
508
|
+
for (const account of accounts) {
|
|
509
|
+
if (!account.allocatorEnabled || account.status !== "active") continue;
|
|
510
|
+
if (account.exhaustedUntil && account.exhaustedUntil.getTime() > now.getTime()) {
|
|
511
|
+
future.push(account.exhaustedUntil);
|
|
512
|
+
}
|
|
513
|
+
if ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.primaryResetAt && account.primaryResetAt.getTime() > now.getTime()) {
|
|
514
|
+
future.push(account.primaryResetAt);
|
|
515
|
+
}
|
|
516
|
+
if ((account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.secondaryResetAt && account.secondaryResetAt.getTime() > now.getTime()) {
|
|
517
|
+
future.push(account.secondaryResetAt);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
if (future.length === 0) return null;
|
|
521
|
+
return future.reduce(
|
|
522
|
+
(earliest, candidate) => candidate.getTime() < earliest.getTime() ? candidate : earliest
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
function computeIdleDelayMs(earliestResetAt, now, maxMs) {
|
|
526
|
+
const delta = earliestResetAt.getTime() - now.getTime();
|
|
527
|
+
return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
|
|
528
|
+
}
|
|
529
|
+
var REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5e3;
|
|
530
|
+
var REACTIVE_ROTATION_MARGIN = 2;
|
|
531
|
+
var REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS;
|
|
532
|
+
function computeReactiveRotationResume(args) {
|
|
533
|
+
const streak = args.priorConsecutiveRotations + 1;
|
|
534
|
+
const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
|
|
535
|
+
if (streak > bound) {
|
|
536
|
+
return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
|
|
537
|
+
}
|
|
538
|
+
if (!args.cooldownPersisted) {
|
|
539
|
+
return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
|
|
540
|
+
}
|
|
541
|
+
return { continueDelayMs: 0, idleUntilReset: false };
|
|
542
|
+
}
|
|
543
|
+
function chooseRotationActive(args) {
|
|
544
|
+
const { activeCredentialId, priorCredentialId, accounts, now } = args;
|
|
545
|
+
const rotationStrategy = effectiveRotationStrategy(args.rotationStrategy);
|
|
546
|
+
const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
|
|
547
|
+
if (allocatableAccounts.length === 0) {
|
|
548
|
+
return { kind: "none" };
|
|
549
|
+
}
|
|
550
|
+
const eligibles = allocatableAccounts.filter((acct) => isCodexCredentialEligible(acct, now));
|
|
551
|
+
const decide = (chosen2) => {
|
|
552
|
+
if (!chosen2) {
|
|
553
|
+
return {
|
|
554
|
+
kind: "allCapped",
|
|
555
|
+
earliestResetAt: earliestReset(allocatableAccounts, now)
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
return {
|
|
559
|
+
kind: "active",
|
|
560
|
+
credentialId: chosen2.id,
|
|
561
|
+
moved: chosen2.id !== activeCredentialId
|
|
562
|
+
};
|
|
563
|
+
};
|
|
564
|
+
if (rotationStrategy === "round_robin") {
|
|
565
|
+
if (eligibles.length === 0) {
|
|
566
|
+
return {
|
|
567
|
+
kind: "allCapped",
|
|
568
|
+
earliestResetAt: earliestReset(allocatableAccounts, now)
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
const priorIdx = priorCredentialId ? allocatableAccounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
|
|
572
|
+
const ordered = priorIdx >= 0 ? [
|
|
573
|
+
...allocatableAccounts.slice(priorIdx + 1),
|
|
574
|
+
...allocatableAccounts.slice(0, priorIdx + 1)
|
|
575
|
+
] : allocatableAccounts;
|
|
576
|
+
const chosen2 = ordered.find((acct) => isCodexCredentialEligible(acct, now));
|
|
577
|
+
return decide(chosen2);
|
|
578
|
+
}
|
|
579
|
+
if (rotationStrategy === "drain_then_next") {
|
|
580
|
+
const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : void 0;
|
|
581
|
+
if (priorRow && isCodexCredentialEligible(priorRow, now)) {
|
|
582
|
+
return decide(priorRow);
|
|
583
|
+
}
|
|
584
|
+
return decide(eligibles[0]);
|
|
585
|
+
}
|
|
586
|
+
const activeLeases = (acct) => "activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
|
|
587
|
+
const selections = (acct) => "selectionCount" in acct ? acct.selectionCount : 0;
|
|
588
|
+
const lastSelected = (acct) => "lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
|
|
589
|
+
const chosen = eligibles.reduce((best, acct) => {
|
|
590
|
+
if (!best) {
|
|
591
|
+
return acct;
|
|
592
|
+
}
|
|
593
|
+
if (activeLeases(acct) !== activeLeases(best)) {
|
|
594
|
+
return activeLeases(acct) < activeLeases(best) ? acct : best;
|
|
595
|
+
}
|
|
596
|
+
if (bindingRemaining(acct, now) !== bindingRemaining(best, now)) {
|
|
597
|
+
return bindingRemaining(acct, now) > bindingRemaining(best, now) ? acct : best;
|
|
598
|
+
}
|
|
599
|
+
if (selections(acct) !== selections(best)) {
|
|
600
|
+
return selections(acct) < selections(best) ? acct : best;
|
|
601
|
+
}
|
|
602
|
+
return lastSelected(acct) < lastSelected(best) ? acct : best;
|
|
603
|
+
}, void 0);
|
|
604
|
+
return decide(chosen);
|
|
605
|
+
}
|
|
606
|
+
function chooseLegacyRotationActive(args) {
|
|
607
|
+
if (args.rotationStrategy === "most_remaining") {
|
|
608
|
+
const active = args.activeCredentialId ? args.accounts.find((account) => account.id === args.activeCredentialId) : void 0;
|
|
609
|
+
if (active && isCodexCredentialEligible(active, args.now)) {
|
|
610
|
+
return { kind: "active", credentialId: active.id, moved: false };
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
return chooseRotationActive({
|
|
614
|
+
...args,
|
|
615
|
+
accounts: args.accounts.map((account) => ({
|
|
616
|
+
...account,
|
|
617
|
+
activeLeaseCount: 0,
|
|
618
|
+
selectionCount: 0,
|
|
619
|
+
lastSelectedAt: null
|
|
620
|
+
}))
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
function selectCodexCredentialLeaseForTurn(args) {
|
|
624
|
+
const {
|
|
625
|
+
accounts,
|
|
626
|
+
activeCredentialId,
|
|
627
|
+
rotationEnabled,
|
|
628
|
+
leaseRotationEnabled,
|
|
629
|
+
rotationStrategy,
|
|
630
|
+
existingCredentialId
|
|
631
|
+
} = args.context;
|
|
632
|
+
const leasingEnabled = args.leasingEnabled && leaseRotationEnabled;
|
|
633
|
+
const strategy = effectiveRotationStrategy(rotationStrategy);
|
|
634
|
+
const existing = existingCredentialId ? accounts.find((account) => account.id === existingCredentialId) : void 0;
|
|
635
|
+
if (existing && isCodexCredentialHealthy(existing, args.now)) {
|
|
636
|
+
return {
|
|
637
|
+
credentialId: existing.id,
|
|
638
|
+
advanceActivePointer: false,
|
|
639
|
+
decision: {
|
|
640
|
+
kind: "active",
|
|
641
|
+
credentialId: existing.id,
|
|
642
|
+
moved: existing.id !== activeCredentialId
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
const connectedIds = new Set(
|
|
647
|
+
accounts.filter((account) => account.allocatorEnabled).map((account) => account.id)
|
|
648
|
+
);
|
|
649
|
+
const pinDisposition = classifyCodexPin({
|
|
650
|
+
pinnedCredentialId: args.sessionPinnedCredentialId,
|
|
651
|
+
pinSource: args.sessionPinSource,
|
|
652
|
+
strategy,
|
|
653
|
+
rotationEnabled
|
|
654
|
+
});
|
|
655
|
+
if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
|
|
656
|
+
const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
|
|
657
|
+
if (!pinned?.allocatorEnabled) {
|
|
658
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
659
|
+
}
|
|
660
|
+
if (isCodexCredentialHealthy(pinned, args.now)) {
|
|
661
|
+
return {
|
|
662
|
+
credentialId: pinned.id,
|
|
663
|
+
decision: { kind: "active", credentialId: pinned.id, moved: false },
|
|
664
|
+
advanceActivePointer: false
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
return {
|
|
668
|
+
credentialId: null,
|
|
669
|
+
decision: {
|
|
670
|
+
kind: "allCapped",
|
|
671
|
+
earliestResetAt: availableAt(pinned, args.now)
|
|
672
|
+
},
|
|
673
|
+
advanceActivePointer: false
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
if (pinDisposition === "sharded") {
|
|
677
|
+
if (accounts.length === 0) {
|
|
678
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
679
|
+
}
|
|
680
|
+
const shard = chooseShardedHome({
|
|
681
|
+
sessionId: args.sessionId,
|
|
682
|
+
currentPolicyPin: args.sessionPinSource === "policy" ? args.sessionPinnedCredentialId : null,
|
|
683
|
+
accounts,
|
|
684
|
+
now: args.now
|
|
685
|
+
});
|
|
686
|
+
if (shard.kind === "allCapped") {
|
|
687
|
+
return {
|
|
688
|
+
credentialId: null,
|
|
689
|
+
decision: shard,
|
|
690
|
+
advanceActivePointer: false
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
return {
|
|
694
|
+
credentialId: shard.credentialId,
|
|
695
|
+
decision: {
|
|
696
|
+
kind: "active",
|
|
697
|
+
credentialId: shard.credentialId,
|
|
698
|
+
moved: shard.credentialId !== activeCredentialId
|
|
699
|
+
},
|
|
700
|
+
advanceActivePointer: false
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
if (!rotationEnabled) {
|
|
704
|
+
const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
|
|
705
|
+
if (!active?.allocatorEnabled) {
|
|
706
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
707
|
+
}
|
|
708
|
+
if (!isCodexCredentialHealthy(active, args.now)) {
|
|
709
|
+
return {
|
|
710
|
+
credentialId: null,
|
|
711
|
+
decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
|
|
712
|
+
advanceActivePointer: false
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
return {
|
|
716
|
+
credentialId: active.id,
|
|
717
|
+
decision: { kind: "active", credentialId: active.id, moved: false },
|
|
718
|
+
advanceActivePointer: false
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
|
|
722
|
+
const credentialId = connectedIds.has(args.sessionPinnedCredentialId) ? args.sessionPinnedCredentialId : activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
|
|
723
|
+
return {
|
|
724
|
+
credentialId,
|
|
725
|
+
decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" }
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
const priorId = args.sessionLastCredentialId ?? activeCredentialId;
|
|
729
|
+
const choose = leasingEnabled ? chooseRotationActive : chooseLegacyRotationActive;
|
|
730
|
+
const decision = choose({
|
|
731
|
+
rotationStrategy: strategy,
|
|
732
|
+
activeCredentialId,
|
|
733
|
+
// Per-session continuity is the round-robin/drain cursor. Falling back to
|
|
734
|
+
// the workspace pointer is only correct when this session has never run.
|
|
735
|
+
priorCredentialId: priorId,
|
|
736
|
+
accounts,
|
|
737
|
+
now: args.now
|
|
738
|
+
});
|
|
739
|
+
return {
|
|
740
|
+
credentialId: decision.kind === "active" ? decision.credentialId : null,
|
|
741
|
+
decision
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
// src/activities/codex-capacity.ts
|
|
746
|
+
async function refreshCodexUsageAndRepairCapacityWaiters(refreshes, repairPendingWakes) {
|
|
747
|
+
await Promise.all(refreshes.map((refresh) => refresh().catch(() => void 0)));
|
|
748
|
+
await repairPendingWakes();
|
|
749
|
+
}
|
|
750
|
+
async function signalCodexCapacityWakeTargets(services, targets) {
|
|
751
|
+
await Promise.allSettled(
|
|
752
|
+
targets.map(
|
|
753
|
+
(target) => services.signalCodexCapacityWorkflow ? services.signalCodexCapacityWorkflow({
|
|
754
|
+
accountId: target.accountId,
|
|
755
|
+
workspaceId: target.workspaceId,
|
|
756
|
+
sessionId: target.sessionId,
|
|
757
|
+
workflowId: target.workflowId,
|
|
758
|
+
wakeRevision: target.wakeRevision
|
|
759
|
+
}) : services.wakeSessionWorkflow ? services.wakeSessionWorkflow({
|
|
760
|
+
accountId: target.accountId,
|
|
761
|
+
workspaceId: target.workspaceId,
|
|
762
|
+
sessionId: target.sessionId,
|
|
763
|
+
workflowId: target.workflowId,
|
|
764
|
+
wakeRevision: target.workflowWakeRevision
|
|
765
|
+
}) : Promise.resolve()
|
|
766
|
+
)
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
async function signalPendingCodexCapacityWakeTargets(services, workspaceId) {
|
|
770
|
+
const targets = await listPendingCodexCapacityWakeTargets(services.db, workspaceId).catch(
|
|
771
|
+
() => []
|
|
772
|
+
);
|
|
773
|
+
await signalCodexCapacityWakeTargets(services, targets);
|
|
774
|
+
}
|
|
775
|
+
function codexCapacityDecision(context, settings) {
|
|
776
|
+
const now = /* @__PURE__ */ new Date();
|
|
777
|
+
const selected = selectCodexCredentialLeaseForTurn({
|
|
778
|
+
context,
|
|
779
|
+
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
780
|
+
sessionId: context.sessionId,
|
|
781
|
+
sessionPinnedCredentialId: context.sessionPinnedCredentialId,
|
|
782
|
+
sessionPinSource: context.sessionPinSource,
|
|
783
|
+
sessionLastCredentialId: context.sessionLastCredentialId,
|
|
784
|
+
now
|
|
785
|
+
});
|
|
786
|
+
const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
|
|
787
|
+
const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
|
|
788
|
+
if (selected.credentialId && selectedIsAvailable) {
|
|
789
|
+
return {
|
|
790
|
+
kind: "available",
|
|
791
|
+
credentialId: selected.credentialId,
|
|
792
|
+
diagnostic: {
|
|
793
|
+
connectedCount: context.accounts.length,
|
|
794
|
+
eligibleCount: context.accounts.filter((account) => isCodexCredentialEligible(account, now)).length
|
|
795
|
+
}
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
const authoritativeReset = authoritativeCodexCapacityResetAt(context.accounts, now);
|
|
799
|
+
return {
|
|
800
|
+
kind: "unavailable",
|
|
801
|
+
earliestResetAt: authoritativeReset,
|
|
802
|
+
resetKind: authoritativeReset ? "authoritative" : "bounded_refresh",
|
|
803
|
+
diagnostic: {
|
|
804
|
+
connectedCount: context.accounts.length,
|
|
805
|
+
allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
|
|
806
|
+
policyHash: context.policyHash
|
|
807
|
+
}
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
async function refreshCapacityMetadata(services, workspaceId) {
|
|
811
|
+
const accounts = await listCodexAccountStatuses(services.db, workspaceId).catch(() => []);
|
|
812
|
+
const stale = accounts.filter(
|
|
813
|
+
(account) => account.allocatorEnabled && account.status === "active" && ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || (account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || account.usageCheckedAt === null)
|
|
814
|
+
);
|
|
815
|
+
await refreshCodexUsageAndRepairCapacityWaiters(
|
|
816
|
+
stale.map(
|
|
817
|
+
(account) => () => fetchCodexUsageForAccount(services.db, services.settings, workspaceId, account.id)
|
|
818
|
+
),
|
|
819
|
+
() => signalPendingCodexCapacityWakeTargets(services, workspaceId)
|
|
820
|
+
);
|
|
821
|
+
}
|
|
822
|
+
function createCodexCapacityActivities(services) {
|
|
823
|
+
async function getCodexCapacityWait(input) {
|
|
824
|
+
const { db } = await services();
|
|
825
|
+
const codexWaiter = await getCodexCapacityWaitForSession(
|
|
826
|
+
db,
|
|
827
|
+
input.workspaceId,
|
|
828
|
+
input.sessionId
|
|
829
|
+
);
|
|
830
|
+
const xaiWaiter = codexWaiter ? null : await getXaiCapacityWaitForSession(db, input.workspaceId, input.sessionId);
|
|
831
|
+
const waiter = codexWaiter ?? xaiWaiter;
|
|
832
|
+
return waiter ? {
|
|
833
|
+
...xaiWaiter ? { provider: "xai" } : {},
|
|
834
|
+
waiterId: waiter.id,
|
|
835
|
+
generation: waiter.generation,
|
|
836
|
+
// A capacity mutation may have committed while its Temporal signal
|
|
837
|
+
// was lost or while the workflow continued-as-new. Reconstruct that
|
|
838
|
+
// outbox edge as an immediate re-evaluation rather than waiting for
|
|
839
|
+
// the older timer.
|
|
840
|
+
nextCheckAt: waiter.wakeRevision > waiter.observedWakeRevision ? (/* @__PURE__ */ new Date(0)).toISOString() : waiter.nextCheckAt.toISOString(),
|
|
841
|
+
wakeRevision: waiter.wakeRevision
|
|
842
|
+
} : null;
|
|
843
|
+
}
|
|
844
|
+
async function reconcileCodexCapacityWait(input) {
|
|
845
|
+
const resolved = await services();
|
|
846
|
+
if (input.provider === "xai") {
|
|
847
|
+
const current2 = await getXaiCapacityWaitForSession(
|
|
848
|
+
resolved.db,
|
|
849
|
+
input.workspaceId,
|
|
850
|
+
input.sessionId
|
|
851
|
+
);
|
|
852
|
+
if (!current2 || current2.id !== input.waiterId || current2.generation !== input.generation) {
|
|
853
|
+
return { action: "stale" };
|
|
854
|
+
}
|
|
855
|
+
const result2 = await reconcileXaiCapacityWaitDb(resolved.db, {
|
|
856
|
+
accountId: input.accountId,
|
|
857
|
+
workspaceId: input.workspaceId,
|
|
858
|
+
sessionId: input.sessionId,
|
|
859
|
+
waiterId: input.waiterId,
|
|
860
|
+
generation: input.generation
|
|
861
|
+
});
|
|
862
|
+
if (result2.events.length > 0) {
|
|
863
|
+
try {
|
|
864
|
+
await resolved.bus.publish(input.workspaceId, input.sessionId, result2.events);
|
|
865
|
+
} catch {
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
if (result2.action === "resumed") return { action: "resumed" };
|
|
869
|
+
if (result2.action === "waiting") {
|
|
870
|
+
return {
|
|
871
|
+
action: "waiting",
|
|
872
|
+
provider: "xai",
|
|
873
|
+
waiterId: result2.waiter.id,
|
|
874
|
+
generation: result2.waiter.generation,
|
|
875
|
+
nextCheckAt: result2.waiter.nextCheckAt.toISOString(),
|
|
876
|
+
wakeRevision: result2.waiter.wakeRevision
|
|
877
|
+
};
|
|
878
|
+
}
|
|
879
|
+
return { action: result2.action };
|
|
880
|
+
}
|
|
881
|
+
const current = await getCodexCapacityWaitForSession(
|
|
882
|
+
resolved.db,
|
|
883
|
+
input.workspaceId,
|
|
884
|
+
input.sessionId
|
|
885
|
+
);
|
|
886
|
+
if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
|
|
887
|
+
return { action: "stale" };
|
|
888
|
+
}
|
|
889
|
+
if (input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now()) {
|
|
890
|
+
await refreshCapacityMetadata(resolved, input.workspaceId);
|
|
891
|
+
}
|
|
892
|
+
const result = await reconcileCodexCapacityWaitDb(
|
|
893
|
+
resolved.db,
|
|
894
|
+
{
|
|
895
|
+
accountId: input.accountId,
|
|
896
|
+
workspaceId: input.workspaceId,
|
|
897
|
+
sessionId: input.sessionId,
|
|
898
|
+
waiterId: input.waiterId,
|
|
899
|
+
generation: input.generation
|
|
900
|
+
},
|
|
901
|
+
(context) => codexCapacityDecision(context, resolved.settings)
|
|
902
|
+
);
|
|
903
|
+
if (result.events.length > 0) {
|
|
904
|
+
try {
|
|
905
|
+
await resolved.bus.publish(input.workspaceId, input.sessionId, result.events);
|
|
906
|
+
} catch {
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
if (result.action === "resumed") {
|
|
910
|
+
return { action: "resumed" };
|
|
911
|
+
}
|
|
912
|
+
if (result.action === "waiting") {
|
|
913
|
+
return {
|
|
914
|
+
action: "waiting",
|
|
915
|
+
waiterId: result.waiter.id,
|
|
916
|
+
generation: result.waiter.generation,
|
|
917
|
+
nextCheckAt: result.waiter.nextCheckAt.toISOString(),
|
|
918
|
+
wakeRevision: result.waiter.wakeRevision
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
return { action: result.action };
|
|
922
|
+
}
|
|
923
|
+
return { getCodexCapacityWait, reconcileCodexCapacityWait };
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
// src/activities/goals.ts
|
|
927
|
+
import {
|
|
928
|
+
allowedFirstPartyMcpToolsForSession,
|
|
929
|
+
configuredStaticUsageLimits,
|
|
930
|
+
policyProviderIdForModel
|
|
931
|
+
} from "@opengeni/config";
|
|
932
|
+
import {
|
|
933
|
+
evaluateWorkspaceModelPolicy,
|
|
934
|
+
mergeToolRefs
|
|
935
|
+
} from "@opengeni/contracts";
|
|
936
|
+
import { isCodexBilledModel } from "@opengeni/codex";
|
|
937
|
+
import {
|
|
938
|
+
enqueueSessionWorkflowWakeIfRunnable,
|
|
939
|
+
getBillingBalance,
|
|
940
|
+
getLatestStartedSessionTurn,
|
|
941
|
+
getWorkspaceModelPolicy,
|
|
942
|
+
getSessionGoal,
|
|
943
|
+
isCodexBilledTurn,
|
|
944
|
+
materializeGoalContinuation,
|
|
945
|
+
requireSession,
|
|
946
|
+
sumUsageQuantity
|
|
947
|
+
} from "@opengeni/db";
|
|
948
|
+
function createGoalActivities(services) {
|
|
949
|
+
async function enqueueGoalRetryWake(input) {
|
|
950
|
+
const { db } = await services();
|
|
951
|
+
await enqueueSessionWorkflowWakeIfRunnable(db, {
|
|
952
|
+
accountId: input.accountId,
|
|
953
|
+
workspaceId: input.workspaceId,
|
|
954
|
+
sessionId: input.sessionId,
|
|
955
|
+
temporalWorkflowId: input.workflowId,
|
|
956
|
+
reason: "goal_retry",
|
|
957
|
+
// A permanently invalid goal must not become a tight workflow loop. One
|
|
958
|
+
// durable retry after a short delay preserves liveness without scanning.
|
|
959
|
+
notBefore: new Date(Date.now() + 3e4)
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
async function maybeContinueGoal(input) {
|
|
963
|
+
const { settings, db, bus } = await services();
|
|
964
|
+
const existingGoal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
965
|
+
if (!existingGoal || existingGoal.status !== "active") {
|
|
966
|
+
return { action: "none" };
|
|
967
|
+
}
|
|
968
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
969
|
+
const latestStartedTurn = await getLatestStartedSessionTurn(
|
|
970
|
+
db,
|
|
971
|
+
input.workspaceId,
|
|
972
|
+
input.sessionId
|
|
973
|
+
);
|
|
974
|
+
let continuationModel = latestStartedTurn?.model ?? session.model;
|
|
975
|
+
const continuationReasoningEffort = latestStartedTurn?.reasoningEffort ?? session.reasoningEffort;
|
|
976
|
+
const continuationLatencyMode = latestStartedTurn?.latencyMode ?? session.latencyMode;
|
|
977
|
+
let modelPolicyBlocked = null;
|
|
978
|
+
const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
|
|
979
|
+
if (workspaceModelPolicy) {
|
|
980
|
+
const policyBlocks = (modelId) => !evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
|
|
981
|
+
providerId: policyProviderIdForModel(settings, modelId),
|
|
982
|
+
modelId
|
|
983
|
+
}).allowed;
|
|
984
|
+
if (policyBlocks(continuationModel)) {
|
|
985
|
+
if (continuationModel !== session.model && !policyBlocks(session.model)) {
|
|
986
|
+
continuationModel = session.model;
|
|
987
|
+
} else {
|
|
988
|
+
modelPolicyBlocked = `workspace model policy blocks model "${continuationModel}"; pick an allowed model or change the workspace model policy`;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
if (!modelPolicyBlocked && session.codexCompactionMode === "remote_v2" && !isCodexBilledModel(continuationModel)) {
|
|
993
|
+
modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
|
|
994
|
+
}
|
|
995
|
+
const isCodexRun = await isCodexBilledTurn({
|
|
996
|
+
db,
|
|
997
|
+
settings,
|
|
998
|
+
workspaceId: input.workspaceId,
|
|
999
|
+
model: continuationModel
|
|
1000
|
+
});
|
|
1001
|
+
const budgetBlocked = await goalRunBudgetBlocked(
|
|
1002
|
+
settings,
|
|
1003
|
+
db,
|
|
1004
|
+
input.accountId,
|
|
1005
|
+
input.workspaceId,
|
|
1006
|
+
isCodexRun
|
|
1007
|
+
);
|
|
1008
|
+
const decision = await materializeGoalContinuation(db, {
|
|
1009
|
+
accountId: input.accountId,
|
|
1010
|
+
workspaceId: input.workspaceId,
|
|
1011
|
+
sessionId: input.sessionId,
|
|
1012
|
+
workflowId: input.workflowId,
|
|
1013
|
+
defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
|
|
1014
|
+
// Pacing between consecutive no-input continuations (never a cap). The
|
|
1015
|
+
// materializer re-arms a delayed outbox wake and returns `deferred`.
|
|
1016
|
+
idleBackoff: {
|
|
1017
|
+
scheduleMs: settings.goalIdleBackoffMs,
|
|
1018
|
+
maxMs: settings.goalIdleBackoffMaxMs
|
|
1019
|
+
},
|
|
1020
|
+
// A model-policy block takes precedence: it is deterministic (a budget
|
|
1021
|
+
// pause can clear on its own; a policy pause needs a model/policy change)
|
|
1022
|
+
// and rides the same visible-pause channel.
|
|
1023
|
+
budgetBlocked: modelPolicyBlocked ?? budgetBlocked,
|
|
1024
|
+
policy: {
|
|
1025
|
+
model: continuationModel,
|
|
1026
|
+
reasoningEffort: continuationReasoningEffort,
|
|
1027
|
+
latencyMode: continuationLatencyMode,
|
|
1028
|
+
tools: withFirstPartyTools(settings, session.tools),
|
|
1029
|
+
sandboxBackend: session.sandboxBackend
|
|
1030
|
+
},
|
|
1031
|
+
// The hold guidance is only given when `goal_wait` is actually in this
|
|
1032
|
+
// session's effective first-party selection (the same source the worker
|
|
1033
|
+
// signs into the delegated token and the API uses to register tools), so
|
|
1034
|
+
// a pre-existing narrowed selection is never told to call a missing tool.
|
|
1035
|
+
prompt: (goal, autoContinuation, cap) => {
|
|
1036
|
+
const effectiveFirstPartyTools = allowedFirstPartyMcpToolsForSession(
|
|
1037
|
+
settings,
|
|
1038
|
+
session.firstPartyMcpTools
|
|
1039
|
+
);
|
|
1040
|
+
return goalContinuationPrompt(goal, autoContinuation, cap, {
|
|
1041
|
+
goalWaitAvailable: effectiveFirstPartyTools.includes("goal_wait"),
|
|
1042
|
+
humanInputRespondAvailable: effectiveFirstPartyTools.includes(
|
|
1043
|
+
"session_human_input_respond"
|
|
1044
|
+
)
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
});
|
|
1048
|
+
if (decision.events.length > 0) {
|
|
1049
|
+
await bus.publish(input.workspaceId, input.sessionId, decision.events);
|
|
1050
|
+
}
|
|
1051
|
+
return { action: decision.action };
|
|
1052
|
+
}
|
|
1053
|
+
return {
|
|
1054
|
+
enqueueGoalRetryWake,
|
|
1055
|
+
maybeContinueGoal
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1058
|
+
function goalContinuationPrompt(_goal, _autoContinuation, _cap, options = {}) {
|
|
1059
|
+
const waitingGuidance = options.goalWaitAvailable ? [
|
|
1060
|
+
"Waiting on child sessions or external events:",
|
|
1061
|
+
"- When the next progress depends on child sessions you spawned or on an external event, do not sleep, loop, or poll sessions_list/session_get/session_events to wait for it.",
|
|
1062
|
+
"- Re-check sessions_list or session_get once; if the work is still in flight, call opengeni__goal_wait with a concrete reason and a deadline (untilSeconds), then end your turn immediately. You will be woken by a child result, a message, a human prompt, or at the deadline, and this goal stays active.",
|
|
1063
|
+
"- A hold is for child/external progress only. If you are blocked on a human decision, use opengeni__goal_pause under the blocked audit below instead.",
|
|
1064
|
+
""
|
|
1065
|
+
] : [];
|
|
1066
|
+
const childNoticeGuidance = [
|
|
1067
|
+
"Child lifecycle notices:",
|
|
1068
|
+
"- A `child_requires_action` update means a worker you spawned is blocked on a question or a tool approval and will not progress until it is answered. " + (options.humanInputRespondAvailable ? "If you know the answer to its question, answer it with opengeni__session_human_input_respond (pass the worker's sessionId, the requestId from the notice, and a response). " : "") + "Tool approvals can only be decided by a human. If you cannot resolve the blocker yourself, report the exact blocker (worker session id, the question) to the user and wait or pause instead of retrying the worker.",
|
|
1069
|
+
"- `child_requires_action_resolved`, `child_paused`, `child_waiting_capacity`, and `child_progress` updates are informational: a resolved notice means the worker is moving again; a paused worker needs a human or you to resume it; a capacity wait resumes by itself; a progress note needs no action.",
|
|
1070
|
+
""
|
|
1071
|
+
];
|
|
1072
|
+
return [
|
|
1073
|
+
"Continue working toward the active session goal.",
|
|
1074
|
+
"",
|
|
1075
|
+
"Continuation behavior:",
|
|
1076
|
+
"- This goal persists across turns. Ending this turn does not require shrinking the objective to what fits now.",
|
|
1077
|
+
"- Keep the full objective intact. If it cannot be finished now, make concrete progress toward the real requested end state, leave the goal active, and do not redefine success around a smaller or easier task.",
|
|
1078
|
+
"- Temporary rough edges are acceptable while the work is moving in the right direction. Completion still requires the requested end state to be true and verified.",
|
|
1079
|
+
"",
|
|
1080
|
+
"Work from evidence:",
|
|
1081
|
+
"Use the current workspace and external state as authoritative. Previous conversation context can help locate relevant work, but inspect the current state before relying on it. Improve, replace, or remove existing work as needed to satisfy the actual objective.",
|
|
1082
|
+
"",
|
|
1083
|
+
"Progress visibility:",
|
|
1084
|
+
"If a planning tool is available and the next work is meaningfully multi-step, use it to show a concise plan tied to the real objective. Keep the plan current as steps complete or the next best action changes. Skip planning overhead for trivial one-step progress, and do not treat a plan update as a substitute for doing the work.",
|
|
1085
|
+
"",
|
|
1086
|
+
"Fidelity:",
|
|
1087
|
+
"- Optimize each turn for movement toward the requested end state, not for the smallest stable-looking subset or easiest passing change.",
|
|
1088
|
+
"- Do not substitute a narrower, safer, smaller, merely compatible, or easier-to-test solution because it is more likely to pass current tests.",
|
|
1089
|
+
"- Treat alignment as movement toward the requested end state. An edit is aligned only if it makes the requested final state more true; useful-looking behavior that preserves a different end state is misaligned.",
|
|
1090
|
+
"",
|
|
1091
|
+
"Completion audit:",
|
|
1092
|
+
"Before deciding that the goal is achieved, treat completion as unproven and verify it against the actual current state:",
|
|
1093
|
+
"- Derive concrete requirements from the objective and any referenced files, plans, specifications, issues, or user instructions.",
|
|
1094
|
+
"- Preserve the original scope; do not redefine success around the work that already exists.",
|
|
1095
|
+
"- For every explicit requirement, named artifact, command, test, gate, invariant, and deliverable, identify and inspect the authoritative evidence that would prove it.",
|
|
1096
|
+
"- Match verification scope to requirement scope. Treat uncertain, indirect, incomplete, or missing evidence as not achieved and continue working.",
|
|
1097
|
+
"- The audit must prove completion, not merely fail to find obvious remaining work.",
|
|
1098
|
+
"",
|
|
1099
|
+
"Do not rely on intent, partial progress, memory of earlier work, or a plausible final answer as proof of completion. Call opengeni__goal_complete with concrete evidence only when the full objective is actually achieved and no required work remains.",
|
|
1100
|
+
"",
|
|
1101
|
+
...waitingGuidance,
|
|
1102
|
+
...childNoticeGuidance,
|
|
1103
|
+
"Blocked audit:",
|
|
1104
|
+
"- Do not call opengeni__goal_pause the first time a blocker appears.",
|
|
1105
|
+
"- Pause only when the same blocking condition has repeated for at least three consecutive goal turns and meaningful progress is impossible without user input or an external-state change.",
|
|
1106
|
+
"- Do not pause merely because the work is hard, slow, uncertain, incomplete, or would benefit from clarification.",
|
|
1107
|
+
"- Once that threshold is satisfied, call opengeni__goal_pause with the concrete blocker instead of repeatedly reporting it while leaving the goal active.",
|
|
1108
|
+
"",
|
|
1109
|
+
"Do not call opengeni__goal_complete or opengeni__goal_pause unless the corresponding audit above is satisfied."
|
|
1110
|
+
].join("\n");
|
|
1111
|
+
}
|
|
1112
|
+
function withFirstPartyTools(settings, tools) {
|
|
1113
|
+
if (!settings.mcpServers.some((server) => server.id === "opengeni")) {
|
|
1114
|
+
return tools;
|
|
1115
|
+
}
|
|
1116
|
+
return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
|
|
1117
|
+
}
|
|
1118
|
+
async function goalRunBudgetBlocked(settings, db, accountId, workspaceId, isCodexRun) {
|
|
1119
|
+
if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
1120
|
+
const balance = await getBillingBalance(db, accountId);
|
|
1121
|
+
if (balance.balanceMicros <= 0) {
|
|
1122
|
+
return "insufficient OpenGeni credits";
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
1126
|
+
const limits = configuredStaticUsageLimits(settings);
|
|
1127
|
+
if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
|
|
1128
|
+
const used = await sumUsageQuantity(db, {
|
|
1129
|
+
accountId,
|
|
1130
|
+
eventType: "model.cost",
|
|
1131
|
+
since: startOfUtcMonth()
|
|
1132
|
+
});
|
|
1133
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
1134
|
+
return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
1138
|
+
const used = await sumUsageQuantity(db, {
|
|
1139
|
+
workspaceId,
|
|
1140
|
+
eventType: "agent_run.created",
|
|
1141
|
+
since: startOfUtcMonth()
|
|
1142
|
+
});
|
|
1143
|
+
if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
1144
|
+
return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
return null;
|
|
1149
|
+
}
|
|
1150
|
+
function startOfUtcMonth() {
|
|
1151
|
+
const now = /* @__PURE__ */ new Date();
|
|
1152
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// src/activities/sandbox-images.ts
|
|
1156
|
+
import {
|
|
1157
|
+
resolveRigProviderImageSelection,
|
|
1158
|
+
rigProviderImageSourceImage,
|
|
1159
|
+
settingsWithPackSandboxImage,
|
|
1160
|
+
settingsWithRigImage,
|
|
1161
|
+
settingsWithRigProviderImage
|
|
1162
|
+
} from "@opengeni/core";
|
|
1163
|
+
|
|
1164
|
+
// src/activities/packs.ts
|
|
1165
|
+
import { getWorkspace, listInstalledPortableSkills } from "@opengeni/db";
|
|
1166
|
+
import {
|
|
1167
|
+
legacySandboxRuntimeFromPacks,
|
|
1168
|
+
packInstallationUsesLegacyRuntime,
|
|
1169
|
+
resolveWorkspaceLegacyRuntimePacks
|
|
1170
|
+
} from "@opengeni/core";
|
|
1171
|
+
import {
|
|
1172
|
+
buildPortableSkillArtifact
|
|
1173
|
+
} from "@opengeni/runtime";
|
|
1174
|
+
var emptyPackRuntime = {
|
|
1175
|
+
sandboxImage: null,
|
|
1176
|
+
sandboxProviderImages: null,
|
|
1177
|
+
skillActivations: []
|
|
1178
|
+
};
|
|
1179
|
+
async function resolveWorkspacePackRuntime(db, workspaceId) {
|
|
1180
|
+
const packs = await resolveWorkspaceLegacyRuntimePacks(db, workspaceId);
|
|
1181
|
+
if (packs.length === 0) return emptyPackRuntime;
|
|
1182
|
+
return workspacePackRuntimeFromPacks(packs);
|
|
1183
|
+
}
|
|
1184
|
+
async function resolveWorkspaceInstalledSkillRuntime(db, workspaceId) {
|
|
1185
|
+
const installedSkills = await listInstalledPortableSkills(db, workspaceId);
|
|
1186
|
+
const activations = [];
|
|
1187
|
+
for (const installed of installedSkills) {
|
|
1188
|
+
const artifact = buildPortableSkillArtifact(installed.files);
|
|
1189
|
+
if (artifact.name !== installed.name || artifact.description !== installed.description || artifact.contentSha256 !== installed.contentSha256) {
|
|
1190
|
+
throw new Error(
|
|
1191
|
+
`Installed Skill artifact verification failed for ${installed.capabilityId}; reinstall it from the pinned source`
|
|
1192
|
+
);
|
|
1193
|
+
}
|
|
1194
|
+
activations.push({
|
|
1195
|
+
source: "installation",
|
|
1196
|
+
id: installed.capabilityId,
|
|
1197
|
+
artifact: {
|
|
1198
|
+
name: artifact.name,
|
|
1199
|
+
description: artifact.description,
|
|
1200
|
+
files: artifact.files.map((file) => ({ path: file.path, content: file.content }))
|
|
1201
|
+
},
|
|
1202
|
+
version: installed.version,
|
|
1203
|
+
contentSha256: artifact.contentSha256,
|
|
1204
|
+
reason: installed.source === "library" ? "installed from the curated Skill library" : installed.source === "pack" ? "installed as a Pack-owned Skill" : `installed from ${installed.sourceUrl}`
|
|
1205
|
+
});
|
|
1206
|
+
}
|
|
1207
|
+
return { activations };
|
|
1208
|
+
}
|
|
1209
|
+
function workspacePackRuntimeFromPacks(packs) {
|
|
1210
|
+
const sandboxRuntime = legacySandboxRuntimeFromPacks(packs);
|
|
1211
|
+
const skillActivations = [];
|
|
1212
|
+
const skillOwners = /* @__PURE__ */ new Map();
|
|
1213
|
+
for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
1214
|
+
for (const skill of pack.skills) {
|
|
1215
|
+
const key = skill.name.toLowerCase();
|
|
1216
|
+
const existingOwner = skillOwners.get(key);
|
|
1217
|
+
if (existingOwner !== void 0 && existingOwner !== pack.id) {
|
|
1218
|
+
throw new Error(
|
|
1219
|
+
`Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
skillOwners.set(key, pack.id);
|
|
1223
|
+
const artifact = {
|
|
1224
|
+
name: skill.name,
|
|
1225
|
+
description: skill.description ?? null,
|
|
1226
|
+
files: skill.files.map((file) => ({ path: file.path, content: file.content }))
|
|
1227
|
+
};
|
|
1228
|
+
skillActivations.push({
|
|
1229
|
+
source: "pack",
|
|
1230
|
+
id: `pack:${pack.id}:${skill.name}`,
|
|
1231
|
+
artifact,
|
|
1232
|
+
reason: `active legacy Pack ${pack.id}`
|
|
1233
|
+
});
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
return {
|
|
1237
|
+
sandboxImage: sandboxRuntime.sandboxImage,
|
|
1238
|
+
sandboxProviderImages: sandboxRuntime.sandboxProviderImages,
|
|
1239
|
+
skillActivations
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// src/activities/streaming.ts
|
|
1244
|
+
import { TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS } from "@opengeni/core";
|
|
1245
|
+
import { Context } from "@temporalio/activity";
|
|
1246
|
+
var TRAILING_FLUSH_MS = 33;
|
|
1247
|
+
var MAX_BATCH_EVENTS = 50;
|
|
1248
|
+
var MAX_BUFFERED_EVENTS = 1e3;
|
|
1249
|
+
function createRuntimeBatcher(flushEvents, options = {}) {
|
|
1250
|
+
const now = options.now ?? (() => performance.now());
|
|
1251
|
+
let pending = [];
|
|
1252
|
+
let lastFlush = Date.now();
|
|
1253
|
+
let trailingTimer = null;
|
|
1254
|
+
let activeBatchSize = 0;
|
|
1255
|
+
let drainPromise = null;
|
|
1256
|
+
let flushFailed = false;
|
|
1257
|
+
let flushFailure;
|
|
1258
|
+
const structural = /* @__PURE__ */ new Set([
|
|
1259
|
+
"terminal.pty.output.delta",
|
|
1260
|
+
"agent.toolCall.created",
|
|
1261
|
+
"agent.toolCall.output",
|
|
1262
|
+
"agent.message.completed",
|
|
1263
|
+
"tool.auth_needed",
|
|
1264
|
+
"credential.auth_needed",
|
|
1265
|
+
"session.requiresAction",
|
|
1266
|
+
"turn.completed",
|
|
1267
|
+
"turn.failed",
|
|
1268
|
+
"turn.cancelled"
|
|
1269
|
+
]);
|
|
1270
|
+
return {
|
|
1271
|
+
push: async (event) => {
|
|
1272
|
+
throwIfFlushFailed();
|
|
1273
|
+
pending.push({ type: event.type, payload: event.payload });
|
|
1274
|
+
const elapsed = Date.now() - lastFlush;
|
|
1275
|
+
if (structural.has(event.type)) {
|
|
1276
|
+
await flush();
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1279
|
+
if (pending.length >= MAX_BATCH_EVENTS || elapsed >= TRAILING_FLUSH_MS) {
|
|
1280
|
+
void startDrain().catch(() => void 0);
|
|
1281
|
+
}
|
|
1282
|
+
if (activeBatchSize + pending.length >= MAX_BUFFERED_EVENTS) {
|
|
1283
|
+
await flush();
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
armTrailingTimer();
|
|
1287
|
+
},
|
|
1288
|
+
flush
|
|
1289
|
+
};
|
|
1290
|
+
function armTrailingTimer() {
|
|
1291
|
+
if (trailingTimer !== null) {
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
trailingTimer = setTimeout(() => {
|
|
1295
|
+
trailingTimer = null;
|
|
1296
|
+
void flush().catch(() => void 0);
|
|
1297
|
+
}, TRAILING_FLUSH_MS);
|
|
1298
|
+
if ("unref" in trailingTimer && typeof trailingTimer.unref === "function") {
|
|
1299
|
+
trailingTimer.unref();
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
function clearTrailingTimer() {
|
|
1303
|
+
if (trailingTimer !== null) {
|
|
1304
|
+
clearTimeout(trailingTimer);
|
|
1305
|
+
trailingTimer = null;
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
async function flush() {
|
|
1309
|
+
throwIfFlushFailed();
|
|
1310
|
+
clearTrailingTimer();
|
|
1311
|
+
await startDrain();
|
|
1312
|
+
throwIfFlushFailed();
|
|
1313
|
+
}
|
|
1314
|
+
function startDrain() {
|
|
1315
|
+
if (drainPromise) {
|
|
1316
|
+
return drainPromise;
|
|
1317
|
+
}
|
|
1318
|
+
if (flushFailed) {
|
|
1319
|
+
return Promise.reject(flushFailure);
|
|
1320
|
+
}
|
|
1321
|
+
if (pending.length === 0) {
|
|
1322
|
+
return Promise.resolve();
|
|
1323
|
+
}
|
|
1324
|
+
clearTrailingTimer();
|
|
1325
|
+
const running = (async () => {
|
|
1326
|
+
while (pending.length > 0) {
|
|
1327
|
+
const events = pending.splice(0, MAX_BATCH_EVENTS);
|
|
1328
|
+
activeBatchSize = events.length;
|
|
1329
|
+
lastFlush = Date.now();
|
|
1330
|
+
const startedAt = now();
|
|
1331
|
+
try {
|
|
1332
|
+
await flushEvents(events);
|
|
1333
|
+
} catch (error) {
|
|
1334
|
+
flushFailed = true;
|
|
1335
|
+
flushFailure = error;
|
|
1336
|
+
throw error;
|
|
1337
|
+
} finally {
|
|
1338
|
+
activeBatchSize = 0;
|
|
1339
|
+
if (options.onFlush) {
|
|
1340
|
+
try {
|
|
1341
|
+
options.onFlush({
|
|
1342
|
+
events: events.length,
|
|
1343
|
+
durationSeconds: Math.max(0, (now() - startedAt) / 1e3)
|
|
1344
|
+
});
|
|
1345
|
+
} catch {
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
})();
|
|
1351
|
+
drainPromise = running.finally(() => {
|
|
1352
|
+
drainPromise = null;
|
|
1353
|
+
});
|
|
1354
|
+
return drainPromise;
|
|
1355
|
+
}
|
|
1356
|
+
function throwIfFlushFailed() {
|
|
1357
|
+
if (flushFailed) {
|
|
1358
|
+
throw flushFailure;
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
function currentActivityContext() {
|
|
1363
|
+
try {
|
|
1364
|
+
return Context.current();
|
|
1365
|
+
} catch {
|
|
1366
|
+
return null;
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
function startActivityHeartbeat(context, details, schedule = setInterval) {
|
|
1370
|
+
if (!context) {
|
|
1371
|
+
return null;
|
|
1372
|
+
}
|
|
1373
|
+
const timer = schedule(() => {
|
|
1374
|
+
context.heartbeat({ ...details, at: (/* @__PURE__ */ new Date()).toISOString() });
|
|
1375
|
+
}, TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS);
|
|
1376
|
+
if ("unref" in timer && typeof timer.unref === "function") {
|
|
1377
|
+
timer.unref();
|
|
1378
|
+
}
|
|
1379
|
+
return timer;
|
|
1380
|
+
}
|
|
1381
|
+
async function nextStreamEvent(iterator, context) {
|
|
1382
|
+
if (!context) {
|
|
1383
|
+
return await iterator.next();
|
|
1384
|
+
}
|
|
1385
|
+
const signal = context.cancellationSignal;
|
|
1386
|
+
if (signal.aborted) {
|
|
1387
|
+
throw signal.reason;
|
|
1388
|
+
}
|
|
1389
|
+
return await new Promise((resolve, reject) => {
|
|
1390
|
+
let settled = false;
|
|
1391
|
+
const settle = (complete) => {
|
|
1392
|
+
if (settled) return;
|
|
1393
|
+
settled = true;
|
|
1394
|
+
signal.removeEventListener("abort", onAbort);
|
|
1395
|
+
complete();
|
|
1396
|
+
};
|
|
1397
|
+
const onAbort = () => settle(() => reject(signal.reason));
|
|
1398
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1399
|
+
if (signal.aborted) {
|
|
1400
|
+
onAbort();
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
1403
|
+
let next;
|
|
1404
|
+
try {
|
|
1405
|
+
next = iterator.next();
|
|
1406
|
+
} catch (error) {
|
|
1407
|
+
settle(() => reject(error));
|
|
1408
|
+
return;
|
|
1409
|
+
}
|
|
1410
|
+
void Promise.resolve(next).then(
|
|
1411
|
+
(result) => settle(() => resolve(result)),
|
|
1412
|
+
(error) => settle(() => reject(error))
|
|
1413
|
+
);
|
|
1414
|
+
});
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
// src/activities/parent-wake.ts
|
|
1418
|
+
import {
|
|
1419
|
+
getSession,
|
|
1420
|
+
getSessionGoal as getSessionGoal2,
|
|
1421
|
+
getSessionParentPersonalConnectionDelegations,
|
|
1422
|
+
getSessionParentXaiProviderAccountAuthority,
|
|
1423
|
+
addSessionSystemUpdateWithSourceMutation,
|
|
1424
|
+
claimPendingSessionSystemUpdateOutbox,
|
|
1425
|
+
claimPendingSessionWorkflowWakes,
|
|
1426
|
+
childRequiresActionDedupeKey,
|
|
1427
|
+
getSessionSystemUpdateOutboxByDedupeKey,
|
|
1428
|
+
getOrCreateSessionSystemUpdateOutbox,
|
|
1429
|
+
markSessionWorkflowWakeFailed,
|
|
1430
|
+
markSessionSystemUpdateOutboxDeliveredInTransaction,
|
|
1431
|
+
markSessionSystemUpdateOutboxFailed,
|
|
1432
|
+
sessionSystemUpdateOutboxKindPayload
|
|
1433
|
+
} from "@opengeni/db";
|
|
1434
|
+
async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episodeKey) {
|
|
1435
|
+
try {
|
|
1436
|
+
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
1437
|
+
if (!child || !child.parentSessionId) {
|
|
1438
|
+
return;
|
|
1439
|
+
}
|
|
1440
|
+
const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
|
|
1441
|
+
const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
|
|
1442
|
+
const payload = childCompletionPayload(child, goal);
|
|
1443
|
+
const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
|
|
1444
|
+
svc.db,
|
|
1445
|
+
workspaceId,
|
|
1446
|
+
childSessionId
|
|
1447
|
+
);
|
|
1448
|
+
const xaiAuthority = await getSessionParentXaiProviderAccountAuthority(
|
|
1449
|
+
svc.db,
|
|
1450
|
+
workspaceId,
|
|
1451
|
+
childSessionId
|
|
1452
|
+
);
|
|
1453
|
+
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
1454
|
+
accountId: child.accountId,
|
|
1455
|
+
workspaceId,
|
|
1456
|
+
sourceSessionId: child.id,
|
|
1457
|
+
targetSessionId: child.parentSessionId,
|
|
1458
|
+
kind: "child_terminal_result",
|
|
1459
|
+
classification: goal?.status === "paused" ? "action_required" : "success",
|
|
1460
|
+
sourceId: child.id,
|
|
1461
|
+
dedupeKey: clientEventId,
|
|
1462
|
+
summary: childCompletionSummary(child, goal, "idle"),
|
|
1463
|
+
payload,
|
|
1464
|
+
lineage: {
|
|
1465
|
+
childSessionId: child.id,
|
|
1466
|
+
parentSessionId: child.parentSessionId,
|
|
1467
|
+
...xaiAuthority.subjectId ? { xaiAuthoritySubjectId: xaiAuthority.subjectId } : {}
|
|
1468
|
+
},
|
|
1469
|
+
personalConnectionDelegations,
|
|
1470
|
+
xaiProviderAccountAuthoritySnapshot: xaiAuthority.snapshot
|
|
1471
|
+
});
|
|
1472
|
+
if (outbox.status === "delivered") {
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
1476
|
+
} catch (error) {
|
|
1477
|
+
svc.observability.error("Failed to wake parent session on worker idle boundary", {
|
|
1478
|
+
childSessionId,
|
|
1479
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
async function deliverChildLifecycleOutboxToParent(svc, workspaceId, childSessionId, dedupeKey, options = {}) {
|
|
1484
|
+
try {
|
|
1485
|
+
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
1486
|
+
if (!child || !child.parentSessionId) return;
|
|
1487
|
+
const outbox = await getSessionSystemUpdateOutboxByDedupeKey(svc.db, {
|
|
1488
|
+
accountId: child.accountId,
|
|
1489
|
+
workspaceId,
|
|
1490
|
+
dedupeKey
|
|
1491
|
+
});
|
|
1492
|
+
if (!outbox) {
|
|
1493
|
+
if (options.requireRow === false) return;
|
|
1494
|
+
throw new Error(`Committed child-lifecycle outbox disappeared: ${dedupeKey}`);
|
|
1495
|
+
}
|
|
1496
|
+
if (outbox.status === "delivered") return;
|
|
1497
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
1498
|
+
} catch (error) {
|
|
1499
|
+
svc.observability.error("Failed to deliver committed child-lifecycle notice", {
|
|
1500
|
+
childSessionId,
|
|
1501
|
+
dedupeKey,
|
|
1502
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1503
|
+
});
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
async function deliverFailedChildTurnToParent(svc, workspaceId, childSessionId, turnId) {
|
|
1507
|
+
await deliverChildLifecycleOutboxToParent(
|
|
1508
|
+
svc,
|
|
1509
|
+
workspaceId,
|
|
1510
|
+
childSessionId,
|
|
1511
|
+
`child-completion:${childSessionId}:turn:${turnId}`
|
|
1512
|
+
);
|
|
1513
|
+
}
|
|
1514
|
+
async function deliverChildRequiresActionToParent(svc, workspaceId, childSessionId, input) {
|
|
1515
|
+
if (!svc.settings.childLifecycleNoticesEnabled) return;
|
|
1516
|
+
await deliverChildLifecycleOutboxToParent(
|
|
1517
|
+
svc,
|
|
1518
|
+
workspaceId,
|
|
1519
|
+
childSessionId,
|
|
1520
|
+
childRequiresActionDedupeKey({
|
|
1521
|
+
childSessionId,
|
|
1522
|
+
turnId: input.turnId,
|
|
1523
|
+
turnGeneration: input.turnGeneration
|
|
1524
|
+
}),
|
|
1525
|
+
// A requires_action freeze without any human-input/approval request (or a
|
|
1526
|
+
// child without a parent) commits no row; that is not an error.
|
|
1527
|
+
{ requireRow: false }
|
|
1528
|
+
);
|
|
1529
|
+
}
|
|
1530
|
+
async function deliverParentSystemUpdateOutbox(svc, outbox) {
|
|
1531
|
+
try {
|
|
1532
|
+
const result = await addSessionSystemUpdateWithSourceMutation(
|
|
1533
|
+
svc.db,
|
|
1534
|
+
{
|
|
1535
|
+
accountId: outbox.accountId,
|
|
1536
|
+
workspaceId: outbox.workspaceId,
|
|
1537
|
+
sessionId: outbox.targetSessionId,
|
|
1538
|
+
...sessionSystemUpdateOutboxKindPayload(outbox),
|
|
1539
|
+
classification: outbox.classification,
|
|
1540
|
+
sourceId: outbox.sourceId,
|
|
1541
|
+
dedupeKey: outbox.dedupeKey,
|
|
1542
|
+
summary: outbox.summary,
|
|
1543
|
+
lineage: outbox.lineage,
|
|
1544
|
+
personalConnectionDelegations: outbox.personalConnectionDelegations,
|
|
1545
|
+
xaiProviderAccountAuthoritySnapshot: outbox.xaiProviderAccountAuthoritySnapshot
|
|
1546
|
+
},
|
|
1547
|
+
async (tx) => {
|
|
1548
|
+
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
1549
|
+
}
|
|
1550
|
+
);
|
|
1551
|
+
if (result.reason === "session_cancelled") {
|
|
1552
|
+
return;
|
|
1553
|
+
}
|
|
1554
|
+
if (result.added && result.events.length > 0) {
|
|
1555
|
+
await svc.bus.publish(outbox.workspaceId, outbox.targetSessionId, result.events);
|
|
1556
|
+
}
|
|
1557
|
+
if (result.shouldWake && svc.wakeSessionWorkflow) {
|
|
1558
|
+
if (result.workflowWakeRevision === null) {
|
|
1559
|
+
throw new Error("Runnable system update has no workflow wake revision");
|
|
1560
|
+
}
|
|
1561
|
+
await svc.wakeSessionWorkflow({
|
|
1562
|
+
accountId: outbox.accountId,
|
|
1563
|
+
workspaceId: outbox.workspaceId,
|
|
1564
|
+
sessionId: outbox.targetSessionId,
|
|
1565
|
+
workflowId: result.temporalWorkflowId ?? `session-${outbox.targetSessionId}`,
|
|
1566
|
+
wakeRevision: result.workflowWakeRevision
|
|
1567
|
+
});
|
|
1568
|
+
}
|
|
1569
|
+
svc.observability.info("Delivered child lifecycle notice to parent session", {
|
|
1570
|
+
childSessionId: outbox.sourceSessionId,
|
|
1571
|
+
parentSessionId: outbox.targetSessionId,
|
|
1572
|
+
kind: outbox.kind,
|
|
1573
|
+
dedupeKey: outbox.dedupeKey,
|
|
1574
|
+
woke: result.shouldWake
|
|
1575
|
+
});
|
|
1576
|
+
} catch (error) {
|
|
1577
|
+
await markSessionSystemUpdateOutboxFailed(
|
|
1578
|
+
svc.db,
|
|
1579
|
+
outbox,
|
|
1580
|
+
error instanceof Error ? error.message : String(error)
|
|
1581
|
+
).catch(() => void 0);
|
|
1582
|
+
throw error;
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
async function reconcilePendingParentSystemUpdates(svc, limit = 100, overrides = {}) {
|
|
1586
|
+
const claimPendingSessionSystemUpdateOutboxFn = overrides.claimPendingSessionSystemUpdateOutbox ?? claimPendingSessionSystemUpdateOutbox;
|
|
1587
|
+
const rows = await claimPendingSessionSystemUpdateOutboxFn(svc.db, limit);
|
|
1588
|
+
let delivered = 0;
|
|
1589
|
+
let failed = 0;
|
|
1590
|
+
for (const row of rows) {
|
|
1591
|
+
try {
|
|
1592
|
+
await deliverParentSystemUpdateOutbox(svc, row);
|
|
1593
|
+
delivered += 1;
|
|
1594
|
+
} catch {
|
|
1595
|
+
failed += 1;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
return { claimed: rows.length, delivered, failed };
|
|
1599
|
+
}
|
|
1600
|
+
async function reconcilePendingSessionWorkflowWakes(svc, limit = 1e3, overrides = {}) {
|
|
1601
|
+
const claimPendingSessionWorkflowWakesFn = overrides.claimPendingSessionWorkflowWakes ?? claimPendingSessionWorkflowWakes;
|
|
1602
|
+
if (!svc.wakeSessionWorkflow) {
|
|
1603
|
+
return { claimed: 0, delivered: 0, failed: 0 };
|
|
1604
|
+
}
|
|
1605
|
+
const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
|
|
1606
|
+
let delivered = 0;
|
|
1607
|
+
let failed = 0;
|
|
1608
|
+
const queue = [...repairs];
|
|
1609
|
+
const workers = Array.from({ length: Math.min(20, queue.length) }, async () => {
|
|
1610
|
+
for (; ; ) {
|
|
1611
|
+
const repair = queue.shift();
|
|
1612
|
+
if (!repair) return;
|
|
1613
|
+
try {
|
|
1614
|
+
await svc.wakeSessionWorkflow({
|
|
1615
|
+
accountId: repair.accountId,
|
|
1616
|
+
workspaceId: repair.workspaceId,
|
|
1617
|
+
sessionId: repair.sessionId,
|
|
1618
|
+
workflowId: repair.temporalWorkflowId,
|
|
1619
|
+
wakeRevision: repair.wakeRevision,
|
|
1620
|
+
...repair.interruptionRequested ? { interruptionRequested: true } : {}
|
|
1621
|
+
});
|
|
1622
|
+
delivered += 1;
|
|
1623
|
+
} catch (error) {
|
|
1624
|
+
failed += 1;
|
|
1625
|
+
await markSessionWorkflowWakeFailed(
|
|
1626
|
+
svc.db,
|
|
1627
|
+
repair,
|
|
1628
|
+
error instanceof Error ? error.message : String(error)
|
|
1629
|
+
).catch(() => void 0);
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
});
|
|
1633
|
+
await Promise.all(workers);
|
|
1634
|
+
return { claimed: repairs.length, delivered, failed };
|
|
1635
|
+
}
|
|
1636
|
+
function childCompletionPayload(child, goal) {
|
|
1637
|
+
return {
|
|
1638
|
+
type: "child_terminal_result",
|
|
1639
|
+
childSessionId: child.id,
|
|
1640
|
+
status: "idle",
|
|
1641
|
+
...goal ? {
|
|
1642
|
+
goal: {
|
|
1643
|
+
status: goal.status,
|
|
1644
|
+
text: goal.text,
|
|
1645
|
+
...goal.evidence ? { evidence: goal.evidence } : {},
|
|
1646
|
+
...goal.rationale ? { rationale: goal.rationale } : {},
|
|
1647
|
+
...goal.pausedReason ? { pausedReason: goal.pausedReason } : {}
|
|
1648
|
+
}
|
|
1649
|
+
} : {}
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1652
|
+
function childCompletionSummary(child, goal, terminalStatus) {
|
|
1653
|
+
const lines = [];
|
|
1654
|
+
if (terminalStatus === "failed") {
|
|
1655
|
+
lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
|
|
1656
|
+
} else if (goal?.status === "completed") {
|
|
1657
|
+
lines.push(
|
|
1658
|
+
`A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`
|
|
1659
|
+
);
|
|
1660
|
+
} else if (goal?.status === "paused") {
|
|
1661
|
+
lines.push(
|
|
1662
|
+
`A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`
|
|
1663
|
+
);
|
|
1664
|
+
} else {
|
|
1665
|
+
lines.push(
|
|
1666
|
+
`A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`
|
|
1667
|
+
);
|
|
1668
|
+
}
|
|
1669
|
+
if (goal) {
|
|
1670
|
+
lines.push(`Worker goal: ${goal.text}`);
|
|
1671
|
+
if (goal.status === "completed" && goal.evidence) {
|
|
1672
|
+
lines.push(`Completion evidence: ${goal.evidence}`);
|
|
1673
|
+
}
|
|
1674
|
+
if (goal.status === "paused" && goal.rationale) {
|
|
1675
|
+
lines.push(`Pause rationale: ${goal.rationale}`);
|
|
1676
|
+
}
|
|
1677
|
+
}
|
|
1678
|
+
return lines.join("\n");
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
// src/sandbox-archive-storage.ts
|
|
1682
|
+
import {
|
|
1683
|
+
parseWorkspaceArchiveObjectRef,
|
|
1684
|
+
workspaceArchiveObjectKey
|
|
1685
|
+
} from "@opengeni/contracts";
|
|
1686
|
+
var WorkspaceArchiveObjectStorageRequiredError = class extends Error {
|
|
1687
|
+
code = "workspace_archive_object_storage_required";
|
|
1688
|
+
constructor(backend) {
|
|
1689
|
+
super(`${backend} workspace archives require object storage`);
|
|
1690
|
+
this.name = "WorkspaceArchiveObjectStorageRequiredError";
|
|
1691
|
+
}
|
|
1692
|
+
};
|
|
1693
|
+
function collectWorkspaceArchiveObjectKeys(resumeState) {
|
|
1694
|
+
const sessionState = resumeState?.sessionState && typeof resumeState.sessionState === "object" ? resumeState.sessionState : null;
|
|
1695
|
+
const keys = /* @__PURE__ */ new Set();
|
|
1696
|
+
for (const value of [sessionState?.workspaceArchiveRef, sessionState?.workspaceArchivePrevRef]) {
|
|
1697
|
+
const ref = parseWorkspaceArchiveObjectRef(value);
|
|
1698
|
+
if (ref) keys.add(ref.key);
|
|
1699
|
+
}
|
|
1700
|
+
return keys;
|
|
1701
|
+
}
|
|
1702
|
+
async function putTarWorkspaceArchiveObject(input) {
|
|
1703
|
+
if (input.archive.descriptor.version !== 1) {
|
|
1704
|
+
throw new Error("Object-storage workspace archives are portable tar only");
|
|
1705
|
+
}
|
|
1706
|
+
const key = workspaceArchiveObjectKey({
|
|
1707
|
+
accountId: input.accountId,
|
|
1708
|
+
workspaceId: input.workspaceId,
|
|
1709
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
1710
|
+
revision: input.archive.descriptor.revision
|
|
1711
|
+
});
|
|
1712
|
+
await input.objectStorage.putObject({
|
|
1713
|
+
key,
|
|
1714
|
+
contentType: "application/x-tar",
|
|
1715
|
+
body: input.archive.bytes,
|
|
1716
|
+
sha256: input.archive.descriptor.archiveSha256
|
|
1717
|
+
});
|
|
1718
|
+
return {
|
|
1719
|
+
schema: "sandbox_archive_object_v1",
|
|
1720
|
+
key,
|
|
1721
|
+
sha256: input.archive.descriptor.archiveSha256,
|
|
1722
|
+
bytes: input.archive.descriptor.archiveBytes,
|
|
1723
|
+
backend: input.objectStorage.backend
|
|
1724
|
+
};
|
|
1725
|
+
}
|
|
1726
|
+
async function putVersion1TarArchiveOrInline(input) {
|
|
1727
|
+
if (input.archive.descriptor.version !== 1) {
|
|
1728
|
+
return { workspaceArchive: input.archive.base64 };
|
|
1729
|
+
}
|
|
1730
|
+
if (input.backend === "opensandbox" && !input.objectStorage) {
|
|
1731
|
+
throw new WorkspaceArchiveObjectStorageRequiredError("opensandbox");
|
|
1732
|
+
}
|
|
1733
|
+
if (!input.objectStorage) {
|
|
1734
|
+
return { workspaceArchive: input.archive.base64 };
|
|
1735
|
+
}
|
|
1736
|
+
try {
|
|
1737
|
+
const workspaceArchiveRef = await putTarWorkspaceArchiveObject({
|
|
1738
|
+
objectStorage: input.objectStorage,
|
|
1739
|
+
accountId: input.accountId,
|
|
1740
|
+
workspaceId: input.workspaceId,
|
|
1741
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
1742
|
+
archive: { bytes: input.archive.bytes, descriptor: input.archive.descriptor }
|
|
1743
|
+
});
|
|
1744
|
+
try {
|
|
1745
|
+
input.metrics?.onWorkspaceArchiveObject?.({
|
|
1746
|
+
outcome: "put",
|
|
1747
|
+
backend: input.objectStorage.backend
|
|
1748
|
+
});
|
|
1749
|
+
} catch {
|
|
1750
|
+
}
|
|
1751
|
+
console.info("workspace archive object put", {
|
|
1752
|
+
key: workspaceArchiveRef.key,
|
|
1753
|
+
bytes: workspaceArchiveRef.bytes,
|
|
1754
|
+
sha256: workspaceArchiveRef.sha256
|
|
1755
|
+
});
|
|
1756
|
+
return { workspaceArchiveRef };
|
|
1757
|
+
} catch (error) {
|
|
1758
|
+
try {
|
|
1759
|
+
input.metrics?.onWorkspaceArchiveObject?.({
|
|
1760
|
+
outcome: "put_failed",
|
|
1761
|
+
backend: input.objectStorage.backend
|
|
1762
|
+
});
|
|
1763
|
+
} catch {
|
|
1764
|
+
}
|
|
1765
|
+
throw error;
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
async function deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref, metrics) {
|
|
1769
|
+
if (!objectStorage || !ref) return;
|
|
1770
|
+
try {
|
|
1771
|
+
await objectStorage.deleteObject(ref.key);
|
|
1772
|
+
try {
|
|
1773
|
+
metrics?.onWorkspaceArchiveObject?.({
|
|
1774
|
+
outcome: "deleted_unpublished",
|
|
1775
|
+
backend: objectStorage.backend
|
|
1776
|
+
});
|
|
1777
|
+
} catch {
|
|
1778
|
+
}
|
|
1779
|
+
} catch {
|
|
1780
|
+
console.error("unpublished workspace archive object delete failed", { key: ref.key });
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
async function deleteWorkspaceArchiveObjectKeys(objectStorage, keys) {
|
|
1784
|
+
for (const key of keys) {
|
|
1785
|
+
await objectStorage.deleteObject(key);
|
|
1786
|
+
}
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
// src/activities/common.ts
|
|
1790
|
+
import { mergeResourceRefs, mergeToolRefs as mergeToolRefs2 } from "@opengeni/contracts";
|
|
1791
|
+
function assertReusableSessionRevivable(status) {
|
|
1792
|
+
if (status === "cancelled") {
|
|
1793
|
+
throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
function scheduledUserMessagePayload(prompt, resources, tools, taskId, runId) {
|
|
1797
|
+
return {
|
|
1798
|
+
type: "scheduled_occurrence",
|
|
1799
|
+
text: prompt,
|
|
1800
|
+
scheduledTaskId: taskId,
|
|
1801
|
+
scheduledTaskRunId: runId,
|
|
1802
|
+
...resources.length ? { resources } : {},
|
|
1803
|
+
...tools.length ? { tools } : {}
|
|
1804
|
+
};
|
|
1805
|
+
}
|
|
1806
|
+
function workflowIdForSession(sessionId) {
|
|
1807
|
+
return `session-${sessionId}`;
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// src/activities/incident-telemetry-preflight.ts
|
|
1811
|
+
import {
|
|
1812
|
+
IncidentTelemetryPreflight,
|
|
1813
|
+
stableJson
|
|
1814
|
+
} from "@opengeni/contracts";
|
|
1815
|
+
import { createHash } from "crypto";
|
|
1816
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "util";
|
|
1817
|
+
var INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY = "incidentTelemetryAuthorityFence";
|
|
1818
|
+
function incidentTelemetryPreflightDeclaration(agentConfig, incidentTriggered) {
|
|
1819
|
+
const rawConfig = agentConfig;
|
|
1820
|
+
const executionClass = rawConfig.executionClass;
|
|
1821
|
+
const rawPreflight = rawConfig.incidentTelemetryPreflight;
|
|
1822
|
+
if (executionClass === void 0 && rawPreflight === void 0 && !incidentTriggered) {
|
|
1823
|
+
return { action: "not_required" };
|
|
1824
|
+
}
|
|
1825
|
+
if (executionClass !== "incident_telemetry" || rawPreflight === void 0) {
|
|
1826
|
+
return { action: "blocked", reason: "incident_preflight_metadata_missing" };
|
|
1827
|
+
}
|
|
1828
|
+
const parsed = IncidentTelemetryPreflight.safeParse(rawPreflight);
|
|
1829
|
+
if (!parsed.success) {
|
|
1830
|
+
const dataSourceInvalid = parsed.error.issues.some((issue) => issue.path[0] === "dataSource");
|
|
1831
|
+
return {
|
|
1832
|
+
action: "blocked",
|
|
1833
|
+
reason: dataSourceInvalid ? "incident_data_source_unsuitable" : "incident_preflight_metadata_missing"
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
return { action: "required", preflight: parsed.data };
|
|
1837
|
+
}
|
|
1838
|
+
function evaluateIncidentTelemetryPreflight(input) {
|
|
1839
|
+
const declaration = incidentTelemetryPreflightDeclaration(
|
|
1840
|
+
input.agentConfig,
|
|
1841
|
+
input.incidentTriggered
|
|
1842
|
+
);
|
|
1843
|
+
if (declaration.action !== "required") return declaration;
|
|
1844
|
+
const preflight = declaration.preflight;
|
|
1845
|
+
if (!input.responder.metadataComplete) {
|
|
1846
|
+
return blocked("incident_preflight_metadata_missing");
|
|
1847
|
+
}
|
|
1848
|
+
if (preflight.requiredResources.some(
|
|
1849
|
+
(required) => !input.responder.resources.some((selected) => isDeepStrictEqual2(selected, required))
|
|
1850
|
+
)) {
|
|
1851
|
+
return blocked("incident_responder_under_capable");
|
|
1852
|
+
}
|
|
1853
|
+
const selectedMcpServerIds = new Set(input.responder.mcpServerIds);
|
|
1854
|
+
if (preflight.requiredMcpServerIds.some((id) => !selectedMcpServerIds.has(id))) {
|
|
1855
|
+
return blocked("incident_responder_under_capable");
|
|
1856
|
+
}
|
|
1857
|
+
const selectedFirstPartyTools = new Set(input.responder.firstPartyMcpTools);
|
|
1858
|
+
if (preflight.requiredFirstPartyMcpTools.some((tool) => !selectedFirstPartyTools.has(tool))) {
|
|
1859
|
+
return blocked("incident_responder_under_capable");
|
|
1860
|
+
}
|
|
1861
|
+
const selectedFirstPartyPermissions = new Set(input.responder.firstPartyMcpPermissions);
|
|
1862
|
+
if (preflight.requiredFirstPartyMcpPermissions.some(
|
|
1863
|
+
(permission) => !selectedFirstPartyPermissions.has(permission)
|
|
1864
|
+
)) {
|
|
1865
|
+
return blocked("incident_responder_under_capable");
|
|
1866
|
+
}
|
|
1867
|
+
if (preflight.requiredRig && !rigSatisfies(input.responder.rig, preflight.requiredRig)) {
|
|
1868
|
+
return blocked("incident_responder_under_capable");
|
|
1869
|
+
}
|
|
1870
|
+
const variableSetsByName = new Map(input.responder.variableSets.map((set) => [set.name, set]));
|
|
1871
|
+
if (preflight.requiredVariableSetNames.some((name) => !variableSetsByName.has(name))) {
|
|
1872
|
+
return blocked("incident_responder_under_capable");
|
|
1873
|
+
}
|
|
1874
|
+
const attachedVariableNames = new Set(
|
|
1875
|
+
input.responder.variableSets.flatMap((set) => set.variables.map((variable) => variable.name))
|
|
1876
|
+
);
|
|
1877
|
+
if (preflight.requiredVariableNames.some((name) => !attachedVariableNames.has(name))) {
|
|
1878
|
+
return blocked("incident_responder_under_capable");
|
|
1879
|
+
}
|
|
1880
|
+
const route = preflight.dataSource.route;
|
|
1881
|
+
if (route.kind === "mcp" && !selectedMcpServerIds.has(route.serverId)) {
|
|
1882
|
+
return blocked("incident_responder_under_capable");
|
|
1883
|
+
}
|
|
1884
|
+
if (route.kind === "first_party" && !selectedFirstPartyTools.has(route.tool)) {
|
|
1885
|
+
return blocked("incident_responder_under_capable");
|
|
1886
|
+
}
|
|
1887
|
+
if (route.kind === "variable_set") {
|
|
1888
|
+
const selected = variableSetsByName.get(route.variableSetName);
|
|
1889
|
+
const names = new Set(selected?.variables.map((variable) => variable.name) ?? []);
|
|
1890
|
+
if (!selected || route.variableNames.some((name) => !names.has(name))) {
|
|
1891
|
+
return blocked("incident_responder_under_capable");
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
if (route.kind === "rig_credential_hook" && !rigSatisfies(input.responder.rig, {
|
|
1895
|
+
name: preflight.requiredRig?.name ?? "",
|
|
1896
|
+
credentialHookIds: [route.credentialHookId]
|
|
1897
|
+
})) {
|
|
1898
|
+
return blocked("incident_responder_under_capable");
|
|
1899
|
+
}
|
|
1900
|
+
if (!seriesMetadataSatisfies(preflight.dataSource)) {
|
|
1901
|
+
return blocked("incident_data_source_unsuitable");
|
|
1902
|
+
}
|
|
1903
|
+
if (!alertSelectorsSatisfy(preflight.dataSource, input.alertOccurrenceLabels)) {
|
|
1904
|
+
return blocked("incident_data_source_unsuitable");
|
|
1905
|
+
}
|
|
1906
|
+
return { action: "ready" };
|
|
1907
|
+
}
|
|
1908
|
+
function incidentTelemetryAuthorityFence(input) {
|
|
1909
|
+
const { task, responder } = input;
|
|
1910
|
+
if (responder.toolPolicyVersion === null) return null;
|
|
1911
|
+
const declaration = incidentTelemetryPreflightDeclaration(
|
|
1912
|
+
task.agentConfig,
|
|
1913
|
+
input.alertOccurrenceLabels !== null
|
|
1914
|
+
);
|
|
1915
|
+
if (declaration.action !== "required" || !input.alertOccurrenceLabels) return null;
|
|
1916
|
+
const selectorEntries = declaration.preflight.dataSource.alertSelectorLabels.map((label) => [label, input.alertOccurrenceLabels?.[label]]).sort(([left], [right]) => left.localeCompare(right));
|
|
1917
|
+
if (selectorEntries.some(([, value]) => typeof value !== "string" || value.length === 0)) {
|
|
1918
|
+
return null;
|
|
1919
|
+
}
|
|
1920
|
+
const normalized = {
|
|
1921
|
+
metadataComplete: responder.metadataComplete,
|
|
1922
|
+
resources: [...responder.resources].map((resource) => stableJson(resource)).sort(),
|
|
1923
|
+
mcpServerIds: [...responder.mcpServerIds].sort(),
|
|
1924
|
+
firstPartyMcpTools: [...responder.firstPartyMcpTools].sort(),
|
|
1925
|
+
firstPartyMcpPermissions: [...responder.firstPartyMcpPermissions].sort(),
|
|
1926
|
+
rig: responder.rig ? {
|
|
1927
|
+
name: responder.rig.name,
|
|
1928
|
+
credentialHooks: [...responder.rig.credentialHooks].sort(),
|
|
1929
|
+
checkHealth: responder.rig.checkHealth
|
|
1930
|
+
} : null,
|
|
1931
|
+
variableSets: [...responder.variableSets].map((set) => ({
|
|
1932
|
+
name: set.name,
|
|
1933
|
+
variables: set.variables.map((variable) => variable.name).sort()
|
|
1934
|
+
})).sort((left, right) => left.name.localeCompare(right.name))
|
|
1935
|
+
};
|
|
1936
|
+
return {
|
|
1937
|
+
version: 2,
|
|
1938
|
+
toolPolicyVersion: responder.toolPolicyVersion,
|
|
1939
|
+
responderDigest: createHash("sha256").update(stableJson(normalized)).digest("hex"),
|
|
1940
|
+
taskDigest: createHash("sha256").update(stableJson(task)).digest("hex"),
|
|
1941
|
+
alertSelectorDigest: createHash("sha256").update(stableJson(selectorEntries)).digest("hex")
|
|
1942
|
+
};
|
|
1943
|
+
}
|
|
1944
|
+
function parseIncidentTelemetryAuthorityFence(lineage) {
|
|
1945
|
+
const raw = lineage[INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY];
|
|
1946
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
1947
|
+
const value = raw;
|
|
1948
|
+
return value.version === 2 && typeof value.toolPolicyVersion === "number" && Number.isInteger(value.toolPolicyVersion) && value.toolPolicyVersion > 0 && typeof value.responderDigest === "string" && /^[0-9a-f]{64}$/.test(value.responderDigest) && typeof value.taskDigest === "string" && /^[0-9a-f]{64}$/.test(value.taskDigest) && typeof value.alertSelectorDigest === "string" && /^[0-9a-f]{64}$/.test(value.alertSelectorDigest) ? {
|
|
1949
|
+
version: 2,
|
|
1950
|
+
toolPolicyVersion: value.toolPolicyVersion,
|
|
1951
|
+
responderDigest: value.responderDigest,
|
|
1952
|
+
taskDigest: value.taskDigest,
|
|
1953
|
+
alertSelectorDigest: value.alertSelectorDigest
|
|
1954
|
+
} : null;
|
|
1955
|
+
}
|
|
1956
|
+
function blocked(reason) {
|
|
1957
|
+
return { action: "blocked", reason };
|
|
1958
|
+
}
|
|
1959
|
+
function rigSatisfies(rig, required) {
|
|
1960
|
+
if (!rig || rig.name !== required.name || rig.checkHealth !== "passing") {
|
|
1961
|
+
return false;
|
|
1962
|
+
}
|
|
1963
|
+
const hookIds = new Set(rig.credentialHooks);
|
|
1964
|
+
return required.credentialHookIds.every((hookId) => hookIds.has(hookId));
|
|
1965
|
+
}
|
|
1966
|
+
function seriesMetadataSatisfies(dataSource) {
|
|
1967
|
+
if (dataSource.queryPath !== "/api/v1/query" && dataSource.queryPath !== "/api/v1/query_range") {
|
|
1968
|
+
return false;
|
|
1969
|
+
}
|
|
1970
|
+
return dataSource.requiredSeries.every(
|
|
1971
|
+
(required) => dataSource.availableSeries.some((available) => {
|
|
1972
|
+
if (available.metric !== required.metric) return false;
|
|
1973
|
+
const labels = new Set(available.labels);
|
|
1974
|
+
return labels.has(dataSource.workspaceLabel) && required.labels.every((label) => labels.has(label));
|
|
1975
|
+
})
|
|
1976
|
+
);
|
|
1977
|
+
}
|
|
1978
|
+
function alertSelectorsSatisfy(dataSource, occurrenceLabels) {
|
|
1979
|
+
if (!occurrenceLabels) return false;
|
|
1980
|
+
return dataSource.alertSelectorLabels.every(
|
|
1981
|
+
(label) => label !== dataSource.workspaceLabel && typeof occurrenceLabels[label] === "string" && occurrenceLabels[label].length > 0 && dataSource.requiredSeries.every((series) => series.labels.includes(label)) && dataSource.availableSeries.some(
|
|
1982
|
+
(series) => dataSource.requiredSeries.some((required) => required.metric === series.metric) && series.labels.includes(dataSource.workspaceLabel) && series.labels.includes(label)
|
|
1983
|
+
)
|
|
1984
|
+
);
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
// src/activities/incident-telemetry-authority.ts
|
|
1988
|
+
import {
|
|
1989
|
+
DEFAULT_FIRST_PARTY_MCP_PERMISSIONS
|
|
1990
|
+
} from "@opengeni/contracts";
|
|
1991
|
+
import { defaultSessionMcpServerIds, resolveSessionToolPolicy } from "@opengeni/core";
|
|
1992
|
+
import {
|
|
1993
|
+
getRig,
|
|
1994
|
+
getScheduledScopedRigVersionMetadata,
|
|
1995
|
+
getScheduledTaskRunAcceptedExecution,
|
|
1996
|
+
getVariableSet,
|
|
1997
|
+
listEnabledMcpCapabilityServerIds,
|
|
1998
|
+
listInstalledApiIntegrationServerIdsForDelegations,
|
|
1999
|
+
markScheduledTaskRunAuthorityRejectedInTransaction,
|
|
2000
|
+
requireSession as requireSession2
|
|
2001
|
+
} from "@opengeni/db";
|
|
2002
|
+
import {
|
|
2003
|
+
allowedFirstPartyMcpToolsForSession as allowedFirstPartyMcpToolsForSession2,
|
|
2004
|
+
resolveFirstPartyMcpToolPolicy
|
|
2005
|
+
} from "@opengeni/config";
|
|
2006
|
+
async function resolveIncidentTelemetryResponderMetadata(input) {
|
|
2007
|
+
const [capabilityServerIds, apiIntegrationServerIds] = await Promise.all([
|
|
2008
|
+
listEnabledMcpCapabilityServerIds(input.db, input.task.workspaceId),
|
|
2009
|
+
listInstalledApiIntegrationServerIdsForDelegations(
|
|
2010
|
+
input.db,
|
|
2011
|
+
input.task.workspaceId,
|
|
2012
|
+
input.personalConnectionDelegations
|
|
2013
|
+
)
|
|
2014
|
+
]);
|
|
2015
|
+
const sessionTools = input.executionPolicy ? input.executionPolicy.tools : input.session ? input.session.tools : withFirstPartyTools(input.settings, input.task.agentConfig.tools);
|
|
2016
|
+
const resolvedToolPolicy = resolveIncidentTelemetryResponderToolPolicy({
|
|
2017
|
+
settingsMcpServerIds: input.settings.mcpServers.map((server) => server.id),
|
|
2018
|
+
capabilityServerIds,
|
|
2019
|
+
apiIntegrationServerIds,
|
|
2020
|
+
...input.executionPolicy ? { acceptedSessionMcpServerIds: input.executionPolicy.mcpServerIds } : {},
|
|
2021
|
+
session: input.executionPolicy ? {
|
|
2022
|
+
mcpServers: input.executionPolicy.mcpServerIds.map((id) => ({ id })),
|
|
2023
|
+
tools: input.executionPolicy.tools,
|
|
2024
|
+
toolPolicy: input.executionPolicy.toolPolicy
|
|
2025
|
+
} : input.session,
|
|
2026
|
+
plannedTools: sessionTools
|
|
2027
|
+
});
|
|
2028
|
+
let metadataComplete = true;
|
|
2029
|
+
let rig = null;
|
|
2030
|
+
const variableSetIds = /* @__PURE__ */ new Set();
|
|
2031
|
+
if (input.executionPolicy ?? input.session) {
|
|
2032
|
+
const variableSetId = input.executionPolicy?.variableSetId ?? input.session?.variableSetId;
|
|
2033
|
+
const rigId = input.executionPolicy?.rigId ?? input.session?.rigId ?? null;
|
|
2034
|
+
const rigVersionId = input.executionPolicy?.rigVersionId ?? input.session?.rigVersionId ?? null;
|
|
2035
|
+
if (variableSetId) variableSetIds.add(variableSetId);
|
|
2036
|
+
if (rigId === null !== (rigVersionId === null)) {
|
|
2037
|
+
metadataComplete = false;
|
|
2038
|
+
} else if (rigId && rigVersionId) {
|
|
2039
|
+
const selected = await getScheduledScopedRigVersionMetadata(
|
|
2040
|
+
input.db,
|
|
2041
|
+
{
|
|
2042
|
+
accountId: input.task.accountId,
|
|
2043
|
+
workspaceId: input.task.workspaceId,
|
|
2044
|
+
subjectId: input.personalResourceAuthoritySubjectId ?? input.task.createdBy.subjectId
|
|
2045
|
+
},
|
|
2046
|
+
rigId,
|
|
2047
|
+
rigVersionId
|
|
2048
|
+
);
|
|
2049
|
+
if (!selected) {
|
|
2050
|
+
metadataComplete = false;
|
|
2051
|
+
} else {
|
|
2052
|
+
rig = {
|
|
2053
|
+
name: selected.name,
|
|
2054
|
+
credentialHooks: selected.version.credentialHooks,
|
|
2055
|
+
checkHealth: selected.health.checkHealth
|
|
2056
|
+
};
|
|
2057
|
+
for (const id of selected.version.defaultVariableSetIds) variableSetIds.add(id);
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
2060
|
+
} else {
|
|
2061
|
+
if (input.task.variableSetId) variableSetIds.add(input.task.variableSetId);
|
|
2062
|
+
if (input.task.rigId) {
|
|
2063
|
+
const selectedRig = await getRig(
|
|
2064
|
+
input.db,
|
|
2065
|
+
{
|
|
2066
|
+
accountId: input.task.accountId,
|
|
2067
|
+
workspaceId: input.task.workspaceId,
|
|
2068
|
+
subjectId: input.personalResourceAuthoritySubjectId ?? input.task.createdBy.subjectId
|
|
2069
|
+
},
|
|
2070
|
+
input.task.rigId
|
|
2071
|
+
);
|
|
2072
|
+
if (!selectedRig?.activeVersion || !selectedRig.activeVersionHealth) {
|
|
2073
|
+
metadataComplete = false;
|
|
2074
|
+
} else {
|
|
2075
|
+
rig = {
|
|
2076
|
+
name: selectedRig.name,
|
|
2077
|
+
credentialHooks: selectedRig.activeVersion.credentialHooks,
|
|
2078
|
+
checkHealth: selectedRig.activeVersionHealth.checkHealth
|
|
2079
|
+
};
|
|
2080
|
+
for (const id of selectedRig.activeVersion.defaultVariableSetIds) variableSetIds.add(id);
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
const variableSets = await Promise.all(
|
|
2085
|
+
[...variableSetIds].map(
|
|
2086
|
+
async (id) => await getVariableSet(
|
|
2087
|
+
input.db,
|
|
2088
|
+
{
|
|
2089
|
+
accountId: input.task.accountId,
|
|
2090
|
+
workspaceId: input.task.workspaceId,
|
|
2091
|
+
subjectId: input.personalResourceAuthoritySubjectId ?? input.task.createdBy.subjectId
|
|
2092
|
+
},
|
|
2093
|
+
id
|
|
2094
|
+
)
|
|
2095
|
+
)
|
|
2096
|
+
);
|
|
2097
|
+
if (variableSets.some((variableSet) => variableSet === null)) {
|
|
2098
|
+
metadataComplete = false;
|
|
2099
|
+
}
|
|
2100
|
+
return {
|
|
2101
|
+
metadataComplete,
|
|
2102
|
+
resources: input.executionPolicy ? input.task.agentConfig.resources : input.session?.resources ?? input.task.agentConfig.resources,
|
|
2103
|
+
mcpServerIds: resolvedToolPolicy.toolRefs.map((tool) => tool.id),
|
|
2104
|
+
firstPartyMcpTools: input.executionPolicy ? input.executionPolicy.firstPartyMcpTools : input.session ? allowedFirstPartyMcpToolsForSession2(input.settings, input.session.firstPartyMcpTools) : resolveFirstPartyMcpToolPolicy(input.settings).default,
|
|
2105
|
+
firstPartyMcpPermissions: input.executionPolicy ? input.executionPolicy.firstPartyMcpPermissions ?? DEFAULT_FIRST_PARTY_MCP_PERMISSIONS : input.session?.firstPartyMcpPermissions ?? DEFAULT_FIRST_PARTY_MCP_PERMISSIONS,
|
|
2106
|
+
rig,
|
|
2107
|
+
variableSets: variableSets.flatMap(
|
|
2108
|
+
(variableSet) => variableSet ? [
|
|
2109
|
+
{
|
|
2110
|
+
name: variableSet.name,
|
|
2111
|
+
variables: variableSet.variables.map((variable) => ({
|
|
2112
|
+
name: variable.name
|
|
2113
|
+
}))
|
|
2114
|
+
}
|
|
2115
|
+
] : []
|
|
2116
|
+
),
|
|
2117
|
+
toolPolicyVersion: input.executionPolicy?.toolPolicyVersion ?? input.session?.toolPolicyVersion ?? null
|
|
2118
|
+
};
|
|
2119
|
+
}
|
|
2120
|
+
function resolveIncidentTelemetryResponderToolPolicy(input) {
|
|
2121
|
+
const workspaceDefaultMcpServerIds = new Set(input.settingsMcpServerIds);
|
|
2122
|
+
for (const id of input.capabilityServerIds) workspaceDefaultMcpServerIds.add(id);
|
|
2123
|
+
for (const id of input.apiIntegrationServerIds) workspaceDefaultMcpServerIds.add(id);
|
|
2124
|
+
for (const id of input.acceptedSessionMcpServerIds ?? []) {
|
|
2125
|
+
workspaceDefaultMcpServerIds.add(id);
|
|
2126
|
+
}
|
|
2127
|
+
const availableMcpServerIds = new Set(workspaceDefaultMcpServerIds);
|
|
2128
|
+
for (const server of input.session?.mcpServers ?? []) {
|
|
2129
|
+
availableMcpServerIds.add(server.id);
|
|
2130
|
+
}
|
|
2131
|
+
return resolveSessionToolPolicy({
|
|
2132
|
+
toolPolicy: input.session?.toolPolicy ?? {
|
|
2133
|
+
mode: "explicit",
|
|
2134
|
+
inheritedFromSessionId: null
|
|
2135
|
+
},
|
|
2136
|
+
sessionTools: input.plannedTools,
|
|
2137
|
+
availableMcpServerIds,
|
|
2138
|
+
defaultMcpServerIds: defaultSessionMcpServerIds(
|
|
2139
|
+
[...workspaceDefaultMcpServerIds].map((id) => ({ id }))
|
|
2140
|
+
)
|
|
2141
|
+
});
|
|
2142
|
+
}
|
|
2143
|
+
async function validateIncidentTelemetrySystemUpdateAuthority(input) {
|
|
2144
|
+
if (!(INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY in input.update.lineage)) {
|
|
2145
|
+
return { action: "accept" };
|
|
2146
|
+
}
|
|
2147
|
+
const fence = parseIncidentTelemetryAuthorityFence(input.update.lineage);
|
|
2148
|
+
const payload = input.update.payload;
|
|
2149
|
+
const taskId = input.update.lineage.scheduledTaskId;
|
|
2150
|
+
const runId = payload.type === "scheduled_occurrence" ? payload.scheduledTaskRunId : null;
|
|
2151
|
+
const reject = async () => {
|
|
2152
|
+
if (runId) {
|
|
2153
|
+
await markScheduledTaskRunAuthorityRejectedInTransaction(input.db, {
|
|
2154
|
+
workspaceId: input.workspaceId,
|
|
2155
|
+
sessionId: input.sessionId,
|
|
2156
|
+
runId
|
|
2157
|
+
});
|
|
2158
|
+
}
|
|
2159
|
+
return {
|
|
2160
|
+
action: "reject",
|
|
2161
|
+
reason: "incident_responder_under_capable"
|
|
2162
|
+
};
|
|
2163
|
+
};
|
|
2164
|
+
if (!fence || typeof taskId !== "string" || !runId) return await reject();
|
|
2165
|
+
const session = await requireSession2(input.db, input.workspaceId, input.sessionId);
|
|
2166
|
+
const acceptedExecution = await getScheduledTaskRunAcceptedExecution(input.db, {
|
|
2167
|
+
workspaceId: input.workspaceId,
|
|
2168
|
+
runId
|
|
2169
|
+
});
|
|
2170
|
+
if (!acceptedExecution || acceptedExecution.task.id !== taskId || !acceptedExecution.incidentPreflightRequired) {
|
|
2171
|
+
return await reject();
|
|
2172
|
+
}
|
|
2173
|
+
const task = acceptedExecution.task;
|
|
2174
|
+
const personalConnectionDelegations = acceptedExecution.personalConnectionDelegations;
|
|
2175
|
+
const responder = await resolveIncidentTelemetryResponderMetadata({
|
|
2176
|
+
db: input.db,
|
|
2177
|
+
settings: input.settings,
|
|
2178
|
+
task,
|
|
2179
|
+
session,
|
|
2180
|
+
personalConnectionDelegations,
|
|
2181
|
+
personalResourceAuthoritySubjectId: acceptedExecution.personalResourceAuthoritySubjectId,
|
|
2182
|
+
executionPolicy: acceptedExecution.targetSessionExecution ? {
|
|
2183
|
+
tools: acceptedExecution.targetSessionExecution.tools,
|
|
2184
|
+
firstPartyMcpTools: acceptedExecution.targetSessionExecution.firstPartyMcpTools,
|
|
2185
|
+
firstPartyMcpPermissions: acceptedExecution.targetSessionExecution.firstPartyMcpPermissions,
|
|
2186
|
+
variableSetId: acceptedExecution.targetSessionExecution.variableSetId,
|
|
2187
|
+
rigId: acceptedExecution.targetSessionExecution.rigId,
|
|
2188
|
+
rigVersionId: acceptedExecution.targetSessionExecution.rigVersionId,
|
|
2189
|
+
toolPolicy: acceptedExecution.targetSessionExecution.toolPolicy,
|
|
2190
|
+
mcpServerIds: acceptedExecution.targetSessionExecution.mcpServerIds,
|
|
2191
|
+
toolPolicyVersion: acceptedExecution.targetSessionExecution.toolPolicyVersion
|
|
2192
|
+
} : {
|
|
2193
|
+
tools: acceptedExecution.resolvedTools,
|
|
2194
|
+
firstPartyMcpTools: acceptedExecution.resolvedFirstPartyMcpTools,
|
|
2195
|
+
firstPartyMcpPermissions: acceptedExecution.resolvedFirstPartyMcpPermissions,
|
|
2196
|
+
variableSetId: acceptedExecution.resolvedVariableSet?.id ?? null,
|
|
2197
|
+
rigId: acceptedExecution.resolvedRig?.id ?? null,
|
|
2198
|
+
rigVersionId: acceptedExecution.resolvedRig?.versionId ?? null,
|
|
2199
|
+
toolPolicy: session.toolPolicy,
|
|
2200
|
+
mcpServerIds: [],
|
|
2201
|
+
toolPolicyVersion: session.toolPolicyVersion
|
|
2202
|
+
}
|
|
2203
|
+
});
|
|
2204
|
+
const preflight = evaluateIncidentTelemetryPreflight({
|
|
2205
|
+
agentConfig: task.agentConfig,
|
|
2206
|
+
incidentTriggered: acceptedExecution.alertOccurrenceLabels !== null,
|
|
2207
|
+
alertOccurrenceLabels: acceptedExecution.alertOccurrenceLabels,
|
|
2208
|
+
responder
|
|
2209
|
+
});
|
|
2210
|
+
if (preflight.action !== "ready") return await reject();
|
|
2211
|
+
const current = incidentTelemetryAuthorityFence({
|
|
2212
|
+
task,
|
|
2213
|
+
responder,
|
|
2214
|
+
alertOccurrenceLabels: acceptedExecution.alertOccurrenceLabels
|
|
2215
|
+
});
|
|
2216
|
+
return current && current.toolPolicyVersion === fence.toolPolicyVersion && current.responderDigest === fence.responderDigest && current.taskDigest === fence.taskDigest && current.alertSelectorDigest === fence.alertSelectorDigest ? { action: "accept" } : await reject();
|
|
2217
|
+
}
|
|
2218
|
+
|
|
2219
|
+
// src/activities/agent-turn/sandbox-provision.ts
|
|
2220
|
+
import {
|
|
2221
|
+
SandboxLeaseRecoveryBlockedError as SandboxLeaseRecoveryBlockedError2,
|
|
2222
|
+
SandboxLeaseSupersededError as SandboxLeaseSupersededError2,
|
|
2223
|
+
SandboxLeaseTransitionError as SandboxLeaseTransitionError2,
|
|
2224
|
+
SandboxImageConflictError,
|
|
2225
|
+
SandboxRigConflictError
|
|
2226
|
+
} from "@opengeni/db";
|
|
2227
|
+
import {
|
|
2228
|
+
SandboxConfigError,
|
|
2229
|
+
SandboxExactResumeInstanceUnavailableError,
|
|
2230
|
+
SandboxExactResumeReplacedError,
|
|
2231
|
+
SandboxExecReadinessError as SandboxExecReadinessError2,
|
|
2232
|
+
SandboxProviderContinuityUnavailableError as SandboxProviderContinuityUnavailableError2,
|
|
2233
|
+
SandboxProviderUnavailableError,
|
|
2234
|
+
SandboxResumeIdentityMismatchError,
|
|
2235
|
+
SandboxResumeIdentityUnavailableError,
|
|
2236
|
+
SandboxResumeStateUnavailableError,
|
|
2237
|
+
WorkspaceArchiveIntegrityError as WorkspaceArchiveIntegrityError2,
|
|
2238
|
+
classifyProviderSandboxFailure
|
|
2239
|
+
} from "@opengeni/runtime";
|
|
2240
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
2241
|
+
|
|
2242
|
+
// src/sandbox-resume.ts
|
|
2243
|
+
import {
|
|
2244
|
+
effectiveSandboxLifecycle,
|
|
2245
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
2246
|
+
sandboxLifecycleTransitionWaitMs
|
|
2247
|
+
} from "@opengeni/config";
|
|
2248
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
2249
|
+
import {
|
|
2250
|
+
acquireLease,
|
|
2251
|
+
adoptLegacyModalCheckpointArtifact,
|
|
2252
|
+
beginSandboxRematerialization,
|
|
2253
|
+
claimWorkspaceArchiveCapture,
|
|
2254
|
+
commitWarmingToWarm,
|
|
2255
|
+
failSandboxRematerialization,
|
|
2256
|
+
failWarmingToCold,
|
|
2257
|
+
getSandboxSessionEnvelope,
|
|
2258
|
+
heartbeatLeaseHolderStatus,
|
|
2259
|
+
markSandboxRestoreVerifying,
|
|
2260
|
+
markWarmLeaseInstanceLost,
|
|
2261
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
2262
|
+
persistWarmSnapshot,
|
|
2263
|
+
readLease,
|
|
2264
|
+
registerSandboxCheckpointArtifact,
|
|
2265
|
+
recordWarmingSandboxCreated,
|
|
2266
|
+
reinstateTurnLeaseHolder,
|
|
2267
|
+
releaseWorkspaceArchiveCapture,
|
|
2268
|
+
releaseLeaseHolder,
|
|
2269
|
+
touchLeaseHolder,
|
|
2270
|
+
SandboxLeaseRecoveryBlockedError,
|
|
2271
|
+
SandboxLeaseSupersededError,
|
|
2272
|
+
SandboxLeaseTransitionError
|
|
2273
|
+
} from "@opengeni/db";
|
|
2274
|
+
import {
|
|
2275
|
+
captureVerifiedWorkspaceArchive,
|
|
2276
|
+
describeLegacyNativeSnapshotArchive,
|
|
2277
|
+
inlineWorkspaceArchiveForRestore,
|
|
2278
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
2279
|
+
SandboxExecReadinessError,
|
|
2280
|
+
WorkspaceArchiveIntegrityError,
|
|
2281
|
+
establishSandboxSessionFromEnvelope,
|
|
2282
|
+
isProviderSandboxNotFoundError,
|
|
2283
|
+
parseWorkspaceArchiveDescriptor,
|
|
2284
|
+
providerWorkspaceCapturePolicy,
|
|
2285
|
+
SandboxProviderContinuityUnavailableError,
|
|
2286
|
+
requirePersistableReplacementSandboxEnvelope,
|
|
2287
|
+
renewSandboxProviderExpiration,
|
|
2288
|
+
modalSessionMatchesCheckpointProviderBinding,
|
|
2289
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
2290
|
+
serializeReplacementSandboxEnvelope,
|
|
2291
|
+
tagModalSandbox,
|
|
2292
|
+
terminateUnpublishedSandboxSession,
|
|
2293
|
+
verifySandboxExecReadiness,
|
|
2294
|
+
withoutSandboxProviderIdentity
|
|
2295
|
+
} from "@opengeni/runtime";
|
|
2296
|
+
import { parseWorkspaceArchiveObjectRef as parseWorkspaceArchiveObjectRef2 } from "@opengeni/contracts";
|
|
2297
|
+
function sandboxLeaseHolderIdForAttempt(attemptId) {
|
|
2298
|
+
const normalized = attemptId.trim();
|
|
2299
|
+
if (!normalized) {
|
|
2300
|
+
throw new Error("Sandbox lease holder requires a turn attempt id");
|
|
2301
|
+
}
|
|
2302
|
+
return `turn-attempt:${normalized}`;
|
|
2303
|
+
}
|
|
2304
|
+
function isRetryableDegradedRestore(restore) {
|
|
2305
|
+
return restore.status === "degraded" && restore.retryable === true;
|
|
2306
|
+
}
|
|
2307
|
+
var SandboxProvisionStageError = class extends Error {
|
|
2308
|
+
constructor(stage, source) {
|
|
2309
|
+
super(source instanceof Error ? source.message : String(source), { cause: source });
|
|
2310
|
+
this.stage = stage;
|
|
2311
|
+
this.source = source;
|
|
2312
|
+
}
|
|
2313
|
+
name = "SandboxProvisionStageError";
|
|
2314
|
+
};
|
|
2315
|
+
var sandboxProvisionFailureStages = /* @__PURE__ */ new WeakMap();
|
|
2316
|
+
function sandboxProvisionFailureStage(error) {
|
|
2317
|
+
if (!error || typeof error !== "object") return null;
|
|
2318
|
+
return sandboxProvisionFailureStages.get(error) ?? (error instanceof SandboxProvisionStageError ? error.stage : null);
|
|
2319
|
+
}
|
|
2320
|
+
function sandboxProvisionStageError(stage, error) {
|
|
2321
|
+
if (error && typeof error === "object") {
|
|
2322
|
+
sandboxProvisionFailureStages.set(error, stage);
|
|
2323
|
+
return error;
|
|
2324
|
+
}
|
|
2325
|
+
return new SandboxProvisionStageError(stage, error);
|
|
2326
|
+
}
|
|
2327
|
+
var SandboxWarmingTimeoutError = class extends Error {
|
|
2328
|
+
constructor(backend, timeoutMs, stage, sandboxGroupId, instanceId) {
|
|
2329
|
+
const timeoutSeconds = Math.ceil(timeoutMs / 1e3);
|
|
2330
|
+
const identity = [
|
|
2331
|
+
sandboxGroupId ? `group ${sandboxGroupId}` : null,
|
|
2332
|
+
instanceId ? `sandbox ${instanceId}` : null
|
|
2333
|
+
].filter((part) => part !== null).join(", ");
|
|
2334
|
+
const target = identity ? ` (${identity})` : "";
|
|
2335
|
+
super(
|
|
2336
|
+
stage === "exec_readiness" ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.` : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`
|
|
2337
|
+
);
|
|
2338
|
+
this.backend = backend;
|
|
2339
|
+
this.timeoutMs = timeoutMs;
|
|
2340
|
+
this.stage = stage;
|
|
2341
|
+
this.sandboxGroupId = sandboxGroupId;
|
|
2342
|
+
this.instanceId = instanceId;
|
|
2343
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
2344
|
+
}
|
|
2345
|
+
};
|
|
2346
|
+
var SandboxExecReadinessTimeoutError = class extends SandboxWarmingTimeoutError {
|
|
2347
|
+
code = "sandbox_exec_readiness_timeout";
|
|
2348
|
+
constructor(backend, timeoutMs, identity = {}) {
|
|
2349
|
+
super(
|
|
2350
|
+
backend,
|
|
2351
|
+
timeoutMs,
|
|
2352
|
+
"exec_readiness",
|
|
2353
|
+
identity.sandboxGroupId ?? null,
|
|
2354
|
+
identity.instanceId ?? null
|
|
2355
|
+
);
|
|
2356
|
+
this.name = "SandboxExecReadinessTimeoutError";
|
|
2357
|
+
}
|
|
2358
|
+
};
|
|
2359
|
+
var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError {
|
|
2360
|
+
code = "sandbox_sibling_warming_timeout";
|
|
2361
|
+
constructor(backend, timeoutMs, identity) {
|
|
2362
|
+
super(
|
|
2363
|
+
backend,
|
|
2364
|
+
timeoutMs,
|
|
2365
|
+
"sibling_warming",
|
|
2366
|
+
identity.sandboxGroupId,
|
|
2367
|
+
identity.instanceId ?? null
|
|
2368
|
+
);
|
|
2369
|
+
this.name = "SandboxSiblingWarmingTimeoutError";
|
|
2370
|
+
}
|
|
2371
|
+
};
|
|
2372
|
+
var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
|
|
2373
|
+
constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
|
|
2374
|
+
super(sandboxGroupId, leaseEpoch);
|
|
2375
|
+
this.lostInstanceId = lostInstanceId;
|
|
2376
|
+
this.name = "SandboxLeaseInstanceLostError";
|
|
2377
|
+
}
|
|
2378
|
+
};
|
|
2379
|
+
var WARMING_POLL_INTERVAL_MS = 250;
|
|
2380
|
+
async function sleep(ms) {
|
|
2381
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
2382
|
+
}
|
|
2383
|
+
async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}) {
|
|
2384
|
+
try {
|
|
2385
|
+
await verifySandboxExecReadiness(established, timeoutMs);
|
|
2386
|
+
} catch (error) {
|
|
2387
|
+
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
2388
|
+
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
2389
|
+
sandboxGroupId: identity.sandboxGroupId ?? null,
|
|
2390
|
+
instanceId: established.instanceId
|
|
2391
|
+
});
|
|
2392
|
+
}
|
|
2393
|
+
throw error;
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
var SnapshotTimeoutError = class extends Error {
|
|
2397
|
+
constructor(timeoutMs) {
|
|
2398
|
+
super(`workspace snapshot timed out after ${timeoutMs}ms`);
|
|
2399
|
+
this.name = "SnapshotTimeoutError";
|
|
2400
|
+
}
|
|
2401
|
+
};
|
|
2402
|
+
function safeSnapshotError(error) {
|
|
2403
|
+
const fields = {
|
|
2404
|
+
errorClass: "SnapshotOperationError",
|
|
2405
|
+
errorCode: "snapshot_operation_failed",
|
|
2406
|
+
origin: "sandbox-resume"
|
|
2407
|
+
};
|
|
2408
|
+
try {
|
|
2409
|
+
if (error && typeof error === "object") {
|
|
2410
|
+
const candidate = error;
|
|
2411
|
+
if (typeof candidate.name === "string" && /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)) {
|
|
2412
|
+
fields.causeName = candidate.name;
|
|
2413
|
+
}
|
|
2414
|
+
if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
|
|
2415
|
+
fields.integrityCode = candidate.code;
|
|
2416
|
+
}
|
|
2417
|
+
const rawStatus = candidate.status ?? candidate.statusCode;
|
|
2418
|
+
const status = Number(rawStatus);
|
|
2419
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
2420
|
+
}
|
|
2421
|
+
} catch {
|
|
2422
|
+
}
|
|
2423
|
+
return fields;
|
|
2424
|
+
}
|
|
2425
|
+
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
2426
|
+
if (signal?.aborted) return false;
|
|
2427
|
+
let timeout;
|
|
2428
|
+
let cancelListener;
|
|
2429
|
+
try {
|
|
2430
|
+
await Promise.race([
|
|
2431
|
+
snapshot,
|
|
2432
|
+
...signal ? [
|
|
2433
|
+
new Promise((_resolve, reject) => {
|
|
2434
|
+
cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
|
|
2435
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
2436
|
+
if (signal.aborted) cancelListener();
|
|
2437
|
+
})
|
|
2438
|
+
] : [],
|
|
2439
|
+
new Promise((_, reject) => {
|
|
2440
|
+
timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
|
|
2441
|
+
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
2442
|
+
timeout.unref();
|
|
2443
|
+
}
|
|
2444
|
+
})
|
|
2445
|
+
]);
|
|
2446
|
+
return true;
|
|
2447
|
+
} catch (error) {
|
|
2448
|
+
if (signal?.aborted) return false;
|
|
2449
|
+
if (error instanceof SnapshotTimeoutError) {
|
|
2450
|
+
console.error(
|
|
2451
|
+
"mid-session workspace snapshot wait timed out (turn unaffected)",
|
|
2452
|
+
safeSnapshotError(error)
|
|
2453
|
+
);
|
|
2454
|
+
return false;
|
|
2455
|
+
}
|
|
2456
|
+
return true;
|
|
2457
|
+
} finally {
|
|
2458
|
+
if (timeout) {
|
|
2459
|
+
clearTimeout(timeout);
|
|
2460
|
+
}
|
|
2461
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
async function resolveModalCheckpointBindingBeforeCapture(settings, session, signal) {
|
|
2465
|
+
if (signal?.aborted) {
|
|
2466
|
+
throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
|
|
2467
|
+
}
|
|
2468
|
+
const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
|
|
2469
|
+
let timeout;
|
|
2470
|
+
let cancelListener;
|
|
2471
|
+
try {
|
|
2472
|
+
return await Promise.race([
|
|
2473
|
+
pending,
|
|
2474
|
+
...signal ? [
|
|
2475
|
+
new Promise((_resolve, reject) => {
|
|
2476
|
+
cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
|
|
2477
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
2478
|
+
if (signal.aborted) cancelListener();
|
|
2479
|
+
})
|
|
2480
|
+
] : [],
|
|
2481
|
+
new Promise((_resolve, reject) => {
|
|
2482
|
+
timeout = setTimeout(
|
|
2483
|
+
() => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
|
|
2484
|
+
settings.sandboxSnapshotTimeoutMs
|
|
2485
|
+
);
|
|
2486
|
+
timeout.unref?.();
|
|
2487
|
+
})
|
|
2488
|
+
]);
|
|
2489
|
+
} finally {
|
|
2490
|
+
if (timeout) clearTimeout(timeout);
|
|
2491
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
async function terminateEstablishedSandbox(established) {
|
|
2495
|
+
if (!established) return true;
|
|
2496
|
+
try {
|
|
2497
|
+
await terminateUnpublishedSandboxSession(established);
|
|
2498
|
+
return true;
|
|
2499
|
+
} catch (error) {
|
|
2500
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
2501
|
+
return true;
|
|
2502
|
+
}
|
|
2503
|
+
return false;
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
2506
|
+
function recordSandboxWarmingTimeout(metrics, error) {
|
|
2507
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
2508
|
+
return;
|
|
2509
|
+
}
|
|
2510
|
+
try {
|
|
2511
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
|
|
2512
|
+
} catch {
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
function workspaceArchiveFieldsFromEnvelope(envelope) {
|
|
2516
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
2517
|
+
const archive = sessionState?.workspaceArchive;
|
|
2518
|
+
const archiveRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchiveRef);
|
|
2519
|
+
const hasInline = typeof archive === "string" && archive.length > 0;
|
|
2520
|
+
if (!hasInline && !archiveRef) {
|
|
2521
|
+
return null;
|
|
2522
|
+
}
|
|
2523
|
+
const previous = sessionState?.workspaceArchivePrev;
|
|
2524
|
+
const previousRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchivePrevRef);
|
|
2525
|
+
const metadata = sessionState?.workspaceArchiveMeta;
|
|
2526
|
+
const previousMetadata = sessionState?.workspaceArchivePrevMeta;
|
|
2527
|
+
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
2528
|
+
return {
|
|
2529
|
+
...hasInline && !archiveRef ? { workspaceArchive: archive } : {},
|
|
2530
|
+
...archiveRef ? { workspaceArchiveRef: archiveRef } : {},
|
|
2531
|
+
...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
|
|
2532
|
+
...typeof previous === "string" && previous.length > 0 && !previousRef ? { workspaceArchivePrev: previous } : {},
|
|
2533
|
+
...previousRef ? { workspaceArchivePrevRef: previousRef } : {},
|
|
2534
|
+
...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
|
|
2535
|
+
...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
|
|
2536
|
+
};
|
|
2537
|
+
}
|
|
2538
|
+
async function materializeSpawnEnvelopeArchive(envelope, objectStorage) {
|
|
2539
|
+
if (!envelope || typeof envelope !== "object") return envelope;
|
|
2540
|
+
const record = envelope;
|
|
2541
|
+
const sessionState = record.sessionState && typeof record.sessionState === "object" && !Array.isArray(record.sessionState) ? record.sessionState : null;
|
|
2542
|
+
if (!sessionState || !parseWorkspaceArchiveObjectRef2(sessionState.workspaceArchiveRef)) {
|
|
2543
|
+
return envelope;
|
|
2544
|
+
}
|
|
2545
|
+
if (!objectStorage) {
|
|
2546
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2547
|
+
"archive_base64_invalid",
|
|
2548
|
+
"workspace archive object storage is not configured"
|
|
2549
|
+
);
|
|
2550
|
+
}
|
|
2551
|
+
return {
|
|
2552
|
+
...record,
|
|
2553
|
+
sessionState: await inlineWorkspaceArchiveForRestore(
|
|
2554
|
+
sessionState,
|
|
2555
|
+
(key) => objectStorage.getObjectBytes(key)
|
|
2556
|
+
)
|
|
2557
|
+
};
|
|
2558
|
+
}
|
|
2559
|
+
function legacyNativeArchiveFromEnvelope(envelope) {
|
|
2560
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
2561
|
+
if (!sessionState) return null;
|
|
2562
|
+
const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
2563
|
+
if (existing?.version === 2) return null;
|
|
2564
|
+
return describeLegacyNativeSnapshotArchive(
|
|
2565
|
+
sessionState.workspaceArchive,
|
|
2566
|
+
existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
|
|
2567
|
+
);
|
|
2568
|
+
}
|
|
2569
|
+
async function persistSandboxDeadlineRotationCheckpoint(services, ids, session, expected, signal) {
|
|
2570
|
+
const { db, settings } = services;
|
|
2571
|
+
if (signal?.aborted) {
|
|
2572
|
+
return { checkpointed: false, holder: "attempt_fenced", lease: null };
|
|
2573
|
+
}
|
|
2574
|
+
const reinstated = await reinstateTurnLeaseHolder(db, {
|
|
2575
|
+
accountId: ids.accountId,
|
|
2576
|
+
workspaceId: ids.workspaceId,
|
|
2577
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2578
|
+
sessionId: ids.sessionId,
|
|
2579
|
+
attemptId: ids.attemptId,
|
|
2580
|
+
expectedEpoch: expected.leaseEpoch,
|
|
2581
|
+
expectedInstanceId: expected.instanceId,
|
|
2582
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs
|
|
2583
|
+
});
|
|
2584
|
+
if (reinstated.status === "reinstated") {
|
|
2585
|
+
const message = "sandbox rotation checkpoint reinstated the live turn holder";
|
|
2586
|
+
const fields = {
|
|
2587
|
+
workspaceId: ids.workspaceId,
|
|
2588
|
+
sessionId: ids.sessionId,
|
|
2589
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2590
|
+
leaseEpoch: expected.leaseEpoch,
|
|
2591
|
+
liveness: reinstated.lease.liveness
|
|
2592
|
+
};
|
|
2593
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
2594
|
+
else console.warn(message, fields);
|
|
2595
|
+
}
|
|
2596
|
+
if (reinstated.status !== "present" && reinstated.status !== "reinstated") {
|
|
2597
|
+
return { checkpointed: false, holder: reinstated.status, lease: null };
|
|
2598
|
+
}
|
|
2599
|
+
await maybePersistWarmWorkspaceSnapshot(
|
|
2600
|
+
services,
|
|
2601
|
+
ids,
|
|
2602
|
+
session,
|
|
2603
|
+
expected.leaseEpoch,
|
|
2604
|
+
signal,
|
|
2605
|
+
true
|
|
2606
|
+
);
|
|
2607
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
2608
|
+
const checkpointed = lease?.leaseEpoch === expected.leaseEpoch && lease.instanceId === expected.instanceId && lease.rotationRequestedAt !== null && lease.archiveComplete && signal?.aborted !== true;
|
|
2609
|
+
return { checkpointed, holder: reinstated.status, lease };
|
|
2610
|
+
}
|
|
2611
|
+
async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
|
|
2612
|
+
const { db, settings } = services;
|
|
2613
|
+
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
2614
|
+
if (intervalMs <= 0 && !force) {
|
|
2615
|
+
return false;
|
|
2616
|
+
}
|
|
2617
|
+
const persistable = session;
|
|
2618
|
+
if (typeof persistable.persistWorkspace !== "function" && typeof persistable.persistWorkspaceTar !== "function") {
|
|
2619
|
+
console.error("mid-session workspace snapshot skipped (no persist primitive)", {
|
|
2620
|
+
backendId: typeof persistable.backendId === "string" ? persistable.backendId : null
|
|
2621
|
+
});
|
|
2622
|
+
return false;
|
|
2623
|
+
}
|
|
2624
|
+
const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
|
|
2625
|
+
if (signal?.aborted) {
|
|
2626
|
+
return false;
|
|
2627
|
+
}
|
|
2628
|
+
try {
|
|
2629
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
2630
|
+
const canWarmCapture = lease?.liveness === "warm" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
|
|
2631
|
+
const canForceDrainingCapture = force === true && lease?.liveness === "draining" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
|
|
2632
|
+
if (!lease || !canWarmCapture && !canForceDrainingCapture) {
|
|
2633
|
+
console.error("mid-session workspace snapshot skipped (lease not warm)", {
|
|
2634
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2635
|
+
expectedEpoch: leaseEpoch,
|
|
2636
|
+
liveness: lease?.liveness ?? null,
|
|
2637
|
+
leaseEpoch: lease?.leaseEpoch ?? null,
|
|
2638
|
+
hasInstance: lease?.instanceId !== null && lease?.instanceId !== void 0
|
|
2639
|
+
});
|
|
2640
|
+
return false;
|
|
2641
|
+
}
|
|
2642
|
+
if (lease.archiveComplete) {
|
|
2643
|
+
return false;
|
|
2644
|
+
}
|
|
2645
|
+
if (lease.instanceId === null) {
|
|
2646
|
+
return false;
|
|
2647
|
+
}
|
|
2648
|
+
const instanceId = lease.instanceId;
|
|
2649
|
+
const captureLiveness = canWarmCapture ? "warm" : "draining";
|
|
2650
|
+
const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
|
|
2651
|
+
if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
|
|
2652
|
+
console.error("mid-session workspace snapshot skipped (capture policy)", {
|
|
2653
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2654
|
+
backend: lease.backend,
|
|
2655
|
+
liveInstance: capturePolicy?.liveInstance ?? null
|
|
2656
|
+
});
|
|
2657
|
+
return false;
|
|
2658
|
+
}
|
|
2659
|
+
const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
|
|
2660
|
+
const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
|
|
2661
|
+
if (signal?.aborted) return false;
|
|
2662
|
+
const captureId = randomUUID2();
|
|
2663
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
|
|
2664
|
+
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
2665
|
+
accountId: ids.accountId,
|
|
2666
|
+
workspaceId: ids.workspaceId,
|
|
2667
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2668
|
+
captureId,
|
|
2669
|
+
expectedEpoch: leaseEpoch,
|
|
2670
|
+
expectedInstanceId: instanceId,
|
|
2671
|
+
liveness: captureLiveness,
|
|
2672
|
+
captureTimeoutMs,
|
|
2673
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
2674
|
+
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
2675
|
+
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
2676
|
+
...captureLiveness === "warm" ? {
|
|
2677
|
+
warmAttempt: {
|
|
2678
|
+
sessionId: ids.sessionId,
|
|
2679
|
+
turnId: ids.turnId,
|
|
2680
|
+
attemptId: ids.attemptId,
|
|
2681
|
+
holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId)
|
|
2682
|
+
}
|
|
2683
|
+
} : {}
|
|
2684
|
+
});
|
|
2685
|
+
if (claimed.status !== "claimed") {
|
|
2686
|
+
console.error("mid-session workspace snapshot skipped (capture claim)", {
|
|
2687
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2688
|
+
status: claimed.status
|
|
2689
|
+
});
|
|
2690
|
+
return false;
|
|
2691
|
+
}
|
|
2692
|
+
let timeout;
|
|
2693
|
+
const capturedAtMs = Date.now();
|
|
2694
|
+
const registerCandidate = async (archive) => {
|
|
2695
|
+
if (archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
|
|
2696
|
+
return null;
|
|
2697
|
+
}
|
|
2698
|
+
if (!checkpointBinding) {
|
|
2699
|
+
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
2700
|
+
}
|
|
2701
|
+
return await registerSandboxCheckpointArtifact(db, {
|
|
2702
|
+
accountId: ids.accountId,
|
|
2703
|
+
workspaceId: ids.workspaceId,
|
|
2704
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2705
|
+
sourceLeaseId: claimed.claim.leaseId,
|
|
2706
|
+
sourceLeaseEpoch: leaseEpoch,
|
|
2707
|
+
sourceInstanceId: instanceId,
|
|
2708
|
+
sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
2709
|
+
providerBindingKey: checkpointBinding.key,
|
|
2710
|
+
providerBinding: checkpointBinding.binding,
|
|
2711
|
+
workspaceArchive: archive.base64,
|
|
2712
|
+
workspaceArchiveMeta: archive.descriptor
|
|
2713
|
+
});
|
|
2714
|
+
};
|
|
2715
|
+
const abandonCandidate = async (artifactId, reason) => {
|
|
2716
|
+
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
2717
|
+
accountId: ids.accountId,
|
|
2718
|
+
workspaceId: ids.workspaceId,
|
|
2719
|
+
artifactId,
|
|
2720
|
+
reason
|
|
2721
|
+
}).catch(() => void 0);
|
|
2722
|
+
};
|
|
2723
|
+
const captureAndPublish = (async () => {
|
|
2724
|
+
let candidate = null;
|
|
2725
|
+
let workspaceArchiveRef;
|
|
2726
|
+
try {
|
|
2727
|
+
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
|
|
2728
|
+
requestId: claimed.claim.providerRequestId,
|
|
2729
|
+
strategy: capturePolicy.strategy
|
|
2730
|
+
});
|
|
2731
|
+
candidate = await registerCandidate(archive);
|
|
2732
|
+
const priorLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
2733
|
+
const priorKeys = collectWorkspaceArchiveObjectKeys(
|
|
2734
|
+
priorLease?.resumeState ?? null
|
|
2735
|
+
);
|
|
2736
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
2737
|
+
backend: lease.backend,
|
|
2738
|
+
objectStorage: services.objectStorage,
|
|
2739
|
+
accountId: ids.accountId,
|
|
2740
|
+
workspaceId: ids.workspaceId,
|
|
2741
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2742
|
+
archive: {
|
|
2743
|
+
bytes: archive.bytes,
|
|
2744
|
+
descriptor: archive.descriptor,
|
|
2745
|
+
base64: archive.base64
|
|
2746
|
+
},
|
|
2747
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
2748
|
+
});
|
|
2749
|
+
workspaceArchiveRef = published.workspaceArchiveRef;
|
|
2750
|
+
const { wrote } = await persistWarmSnapshot(db, {
|
|
2751
|
+
accountId: ids.accountId,
|
|
2752
|
+
workspaceId: ids.workspaceId,
|
|
2753
|
+
sessionId: ids.sessionId,
|
|
2754
|
+
turnId: ids.turnId,
|
|
2755
|
+
attemptId: ids.attemptId,
|
|
2756
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2757
|
+
expectedEpoch: leaseEpoch,
|
|
2758
|
+
expectedInstanceId: instanceId,
|
|
2759
|
+
expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
2760
|
+
captureId,
|
|
2761
|
+
workspaceArchiveMeta: archive.descriptor,
|
|
2762
|
+
...published,
|
|
2763
|
+
checkpointArtifactId: candidate?.id ?? null,
|
|
2764
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
2765
|
+
capturedAtMs
|
|
2766
|
+
});
|
|
2767
|
+
if (published.workspaceArchiveRef && services.objectStorage) {
|
|
2768
|
+
if (!wrote) {
|
|
2769
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
2770
|
+
services.objectStorage,
|
|
2771
|
+
published.workspaceArchiveRef,
|
|
2772
|
+
services.sandboxMetrics
|
|
2773
|
+
);
|
|
2774
|
+
} else {
|
|
2775
|
+
const afterLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
2776
|
+
const afterKeys = collectWorkspaceArchiveObjectKeys(
|
|
2777
|
+
afterLease?.resumeState ?? null
|
|
2778
|
+
);
|
|
2779
|
+
await deleteWorkspaceArchiveObjectKeys(
|
|
2780
|
+
services.objectStorage,
|
|
2781
|
+
[...priorKeys].filter((key) => !afterKeys.has(key))
|
|
2782
|
+
).catch(() => void 0);
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
if (!wrote && candidate) {
|
|
2786
|
+
await abandonCandidate(candidate.id, "snapshot_publication_fenced");
|
|
2787
|
+
}
|
|
2788
|
+
return wrote;
|
|
2789
|
+
} catch (error) {
|
|
2790
|
+
await deleteUnpublishedWorkspaceArchiveObject(
|
|
2791
|
+
services.objectStorage,
|
|
2792
|
+
workspaceArchiveRef,
|
|
2793
|
+
services.sandboxMetrics
|
|
2794
|
+
);
|
|
2795
|
+
if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
|
|
2796
|
+
throw error;
|
|
2797
|
+
} finally {
|
|
2798
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
2799
|
+
accountId: ids.accountId,
|
|
2800
|
+
workspaceId: ids.workspaceId,
|
|
2801
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2802
|
+
captureId,
|
|
2803
|
+
expectedEpoch: leaseEpoch,
|
|
2804
|
+
expectedInstanceId: instanceId
|
|
2805
|
+
}).catch((error) => {
|
|
2806
|
+
console.error(
|
|
2807
|
+
"mid-session workspace capture gate release failed",
|
|
2808
|
+
safeSnapshotError(error)
|
|
2809
|
+
);
|
|
2810
|
+
});
|
|
2811
|
+
}
|
|
2812
|
+
})();
|
|
2813
|
+
const settled = captureAndPublish.then(
|
|
2814
|
+
(persisted) => ({ kind: "settled", persisted }),
|
|
2815
|
+
(error) => {
|
|
2816
|
+
console.error(
|
|
2817
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
2818
|
+
safeSnapshotError(error)
|
|
2819
|
+
);
|
|
2820
|
+
return { kind: "settled", persisted: false };
|
|
2821
|
+
}
|
|
2822
|
+
);
|
|
2823
|
+
let cancelListener;
|
|
2824
|
+
const outcome = await Promise.race([
|
|
2825
|
+
settled,
|
|
2826
|
+
...signal ? [
|
|
2827
|
+
new Promise((resolve) => {
|
|
2828
|
+
cancelListener = () => resolve({ kind: "cancelled" });
|
|
2829
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
2830
|
+
if (signal.aborted) cancelListener();
|
|
2831
|
+
})
|
|
2832
|
+
] : [],
|
|
2833
|
+
new Promise((resolve) => {
|
|
2834
|
+
timeout = setTimeout(
|
|
2835
|
+
() => resolve({ kind: "timed_out" }),
|
|
2836
|
+
settings.sandboxSnapshotTimeoutMs
|
|
2837
|
+
);
|
|
2838
|
+
})
|
|
2839
|
+
]).finally(() => {
|
|
2840
|
+
if (timeout) clearTimeout(timeout);
|
|
2841
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
2842
|
+
});
|
|
2843
|
+
return outcome.kind === "settled" ? outcome.persisted : false;
|
|
2844
|
+
} catch (error) {
|
|
2845
|
+
console.error(
|
|
2846
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
2847
|
+
safeSnapshotError(error)
|
|
2848
|
+
);
|
|
2849
|
+
return false;
|
|
2850
|
+
}
|
|
2851
|
+
}
|
|
2852
|
+
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
2853
|
+
const { db, settings } = services;
|
|
2854
|
+
const os = ids.os ?? "linux";
|
|
2855
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
2856
|
+
const cancellationSignal = services.cancellationSignal;
|
|
2857
|
+
let releaseStarted = false;
|
|
2858
|
+
let holderRelease = null;
|
|
2859
|
+
let quiescedRelease = null;
|
|
2860
|
+
let holderLivenessTimer;
|
|
2861
|
+
let cancellationListener;
|
|
2862
|
+
const release = (options) => {
|
|
2863
|
+
const writersQuiesced = options?.workspaceWritersQuiesced === true;
|
|
2864
|
+
if (!releaseStarted) {
|
|
2865
|
+
releaseStarted = true;
|
|
2866
|
+
if (cancellationListener) {
|
|
2867
|
+
cancellationSignal?.removeEventListener("abort", cancellationListener);
|
|
2868
|
+
cancellationListener = void 0;
|
|
2869
|
+
}
|
|
2870
|
+
if (holderLivenessTimer) {
|
|
2871
|
+
clearInterval(holderLivenessTimer);
|
|
2872
|
+
holderLivenessTimer = void 0;
|
|
2873
|
+
}
|
|
2874
|
+
}
|
|
2875
|
+
const persistRelease = async (workspaceWritersQuiesced) => {
|
|
2876
|
+
await releaseLeaseHolder(db, {
|
|
2877
|
+
accountId: ids.accountId,
|
|
2878
|
+
workspaceId: ids.workspaceId,
|
|
2879
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2880
|
+
kind,
|
|
2881
|
+
holderId,
|
|
2882
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
2883
|
+
...workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}
|
|
2884
|
+
});
|
|
2885
|
+
};
|
|
2886
|
+
if (writersQuiesced) {
|
|
2887
|
+
if (!quiescedRelease) {
|
|
2888
|
+
const predecessor = holderRelease;
|
|
2889
|
+
const attempt = (predecessor ? predecessor.catch(() => void 0) : Promise.resolve()).then(
|
|
2890
|
+
async () => await persistRelease(true)
|
|
2891
|
+
);
|
|
2892
|
+
const tracked = attempt.catch((error) => {
|
|
2893
|
+
if (quiescedRelease === tracked) quiescedRelease = null;
|
|
2894
|
+
throw error;
|
|
2895
|
+
});
|
|
2896
|
+
quiescedRelease = tracked;
|
|
2897
|
+
holderRelease = tracked;
|
|
2898
|
+
}
|
|
2899
|
+
return quiescedRelease;
|
|
2900
|
+
}
|
|
2901
|
+
if (!holderRelease) {
|
|
2902
|
+
const attempt = persistRelease(false);
|
|
2903
|
+
const tracked = attempt.catch((error) => {
|
|
2904
|
+
if (holderRelease === tracked) holderRelease = null;
|
|
2905
|
+
throw error;
|
|
2906
|
+
});
|
|
2907
|
+
holderRelease = tracked;
|
|
2908
|
+
}
|
|
2909
|
+
return holderRelease;
|
|
2910
|
+
};
|
|
2911
|
+
const cancellationError = () => cancellationSignal?.reason instanceof Error ? cancellationSignal.reason : new Error("Sandbox resume was cancelled with its owning turn attempt");
|
|
2912
|
+
const throwIfReleasedOrCancelled = () => {
|
|
2913
|
+
if (releaseStarted || cancellationSignal?.aborted) {
|
|
2914
|
+
throw cancellationError();
|
|
2915
|
+
}
|
|
2916
|
+
};
|
|
2917
|
+
const acquired = await acquireLease(db, {
|
|
2918
|
+
accountId: ids.accountId,
|
|
2919
|
+
workspaceId: ids.workspaceId,
|
|
2920
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2921
|
+
kind,
|
|
2922
|
+
holderId,
|
|
2923
|
+
subjectId: ids.sessionId,
|
|
2924
|
+
backend: ids.backend,
|
|
2925
|
+
os,
|
|
2926
|
+
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
2927
|
+
// conflicts on a live box already running a different image. A
|
|
2928
|
+
// SandboxImageConflictError propagates while another holder is active; a
|
|
2929
|
+
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
2930
|
+
// after the cold successor can safely stamp the new image.
|
|
2931
|
+
...ids.image ? { image: ids.image } : {},
|
|
2932
|
+
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
2933
|
+
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
2934
|
+
// propagates while another holder is active; a solo change uses the same
|
|
2935
|
+
// durable capture-and-drain rotation as an image change.
|
|
2936
|
+
...ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {},
|
|
2937
|
+
leaseTtlMs,
|
|
2938
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
2939
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
2940
|
+
...cancellationSignal ? { waitSignal: cancellationSignal } : {}
|
|
2941
|
+
});
|
|
2942
|
+
if (cancellationSignal) {
|
|
2943
|
+
cancellationListener = () => {
|
|
2944
|
+
void release().catch(() => void 0);
|
|
2945
|
+
};
|
|
2946
|
+
cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
|
|
2947
|
+
}
|
|
2948
|
+
if (cancellationSignal?.aborted) {
|
|
2949
|
+
await release();
|
|
2950
|
+
throw cancellationError();
|
|
2951
|
+
}
|
|
2952
|
+
let holderLeaseHeartbeat = acquired.role === "spawner" ? {
|
|
2953
|
+
expectedEpoch: acquired.lease.leaseEpoch,
|
|
2954
|
+
leaseTtlMs: settings.sandboxLeaseWarmingTtlMs
|
|
2955
|
+
} : (acquired.role === "attached" || acquired.role === "rearmed") && acquired.lease.liveness === "warm" ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs } : null;
|
|
2956
|
+
let providerRenewalTarget = acquired.lease.liveness === "warm" && acquired.lease.instanceId ? {
|
|
2957
|
+
backend: ids.backend,
|
|
2958
|
+
instanceId: acquired.lease.instanceId
|
|
2959
|
+
} : null;
|
|
2960
|
+
let providerRenewedAtMs = Date.now();
|
|
2961
|
+
let providerRenewalInFlight = null;
|
|
2962
|
+
const maybeRenewProviderExpiration = async (force = false) => {
|
|
2963
|
+
const target = providerRenewalTarget;
|
|
2964
|
+
if (!target || providerRenewalInFlight) return;
|
|
2965
|
+
const ttlSeconds = effectiveSandboxLifecycle(settings, target.backend).renewableTtlSeconds;
|
|
2966
|
+
if (ttlSeconds === null) return;
|
|
2967
|
+
const intervalMs = Math.max(1e4, Math.floor(ttlSeconds * 1e3 / 3));
|
|
2968
|
+
if (!force && Date.now() - providerRenewedAtMs < intervalMs) return;
|
|
2969
|
+
providerRenewalInFlight = renewSandboxProviderExpiration({
|
|
2970
|
+
backend: target.backend,
|
|
2971
|
+
settings,
|
|
2972
|
+
instanceId: target.instanceId,
|
|
2973
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
2974
|
+
}).then(() => {
|
|
2975
|
+
providerRenewedAtMs = Date.now();
|
|
2976
|
+
}).finally(() => {
|
|
2977
|
+
providerRenewalInFlight = null;
|
|
2978
|
+
});
|
|
2979
|
+
await providerRenewalInFlight;
|
|
2980
|
+
};
|
|
2981
|
+
const releaseDeadHolder = (reason) => {
|
|
2982
|
+
const message = "sandbox resume holder released: holder no longer live";
|
|
2983
|
+
const fields = {
|
|
2984
|
+
workspaceId: ids.workspaceId,
|
|
2985
|
+
sessionId: ids.sessionId,
|
|
2986
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2987
|
+
holderKind: kind,
|
|
2988
|
+
reason
|
|
2989
|
+
};
|
|
2990
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
2991
|
+
else console.warn(message, fields);
|
|
2992
|
+
void release().catch(() => void 0);
|
|
2993
|
+
};
|
|
2994
|
+
holderLivenessTimer = setInterval(() => {
|
|
2995
|
+
const heartbeat = holderLeaseHeartbeat;
|
|
2996
|
+
if (heartbeat) {
|
|
2997
|
+
void heartbeatLeaseHolderStatus(db, {
|
|
2998
|
+
accountId: ids.accountId,
|
|
2999
|
+
workspaceId: ids.workspaceId,
|
|
3000
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3001
|
+
kind,
|
|
3002
|
+
holderId,
|
|
3003
|
+
expectedEpoch: heartbeat.expectedEpoch,
|
|
3004
|
+
leaseTtlMs: heartbeat.leaseTtlMs
|
|
3005
|
+
}).then((status) => {
|
|
3006
|
+
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
3007
|
+
if (!status.holderAlive) {
|
|
3008
|
+
releaseDeadHolder(status.fence);
|
|
3009
|
+
return;
|
|
3010
|
+
}
|
|
3011
|
+
if (status.fence === "epoch") {
|
|
3012
|
+
releaseDeadHolder(status.fence);
|
|
3013
|
+
return;
|
|
3014
|
+
}
|
|
3015
|
+
if (status.leaseExtended || status.fence === "rotation_requested") {
|
|
3016
|
+
void maybeRenewProviderExpiration().catch(() => void 0);
|
|
3017
|
+
}
|
|
3018
|
+
}).catch(() => void 0);
|
|
3019
|
+
return;
|
|
3020
|
+
}
|
|
3021
|
+
void touchLeaseHolder(db, {
|
|
3022
|
+
accountId: ids.accountId,
|
|
3023
|
+
workspaceId: ids.workspaceId,
|
|
3024
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3025
|
+
kind,
|
|
3026
|
+
holderId
|
|
3027
|
+
}).then((touched) => {
|
|
3028
|
+
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
3029
|
+
if (!touched) {
|
|
3030
|
+
releaseDeadHolder("holder_gone");
|
|
3031
|
+
return;
|
|
3032
|
+
}
|
|
3033
|
+
void maybeRenewProviderExpiration().catch(() => void 0);
|
|
3034
|
+
}).catch(() => void 0);
|
|
3035
|
+
}, services.holderLivenessIntervalMs ?? 1e4);
|
|
3036
|
+
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
3037
|
+
holderLivenessTimer.unref();
|
|
3038
|
+
}
|
|
3039
|
+
if (acquired.role === "blocked") {
|
|
3040
|
+
await release();
|
|
3041
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3042
|
+
ids.sandboxGroupId,
|
|
3043
|
+
acquired.lease.leaseEpoch,
|
|
3044
|
+
acquired.code,
|
|
3045
|
+
acquired.lease.recovery
|
|
3046
|
+
);
|
|
3047
|
+
}
|
|
3048
|
+
if (acquired.role === "fenced") {
|
|
3049
|
+
await release();
|
|
3050
|
+
if (acquired.reason !== "superseded") {
|
|
3051
|
+
throw new SandboxLeaseTransitionError(
|
|
3052
|
+
ids.sandboxGroupId,
|
|
3053
|
+
acquired.lease.leaseEpoch,
|
|
3054
|
+
acquired.reason,
|
|
3055
|
+
acquired.lease.backend,
|
|
3056
|
+
acquired.lease.instanceId,
|
|
3057
|
+
acquired.lease.liveness
|
|
3058
|
+
);
|
|
3059
|
+
}
|
|
3060
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
|
|
3061
|
+
backend: acquired.lease.backend,
|
|
3062
|
+
instanceId: acquired.lease.instanceId,
|
|
3063
|
+
liveness: acquired.lease.liveness
|
|
3064
|
+
});
|
|
3065
|
+
}
|
|
3066
|
+
if (acquired.role === "spawner") {
|
|
3067
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
3068
|
+
let createdEstablished = null;
|
|
3069
|
+
let rematerialization = null;
|
|
3070
|
+
try {
|
|
3071
|
+
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
3072
|
+
const fallbackArchiveEnvelope = acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
|
|
3073
|
+
let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
3074
|
+
const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
|
|
3075
|
+
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
3076
|
+
if (acquired.lease.recovery.archive.status === "available" && acquired.lease.archiveComplete || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
|
|
3077
|
+
const rematerializationId = crypto.randomUUID();
|
|
3078
|
+
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
3079
|
+
const begun = await beginSandboxRematerialization(db, {
|
|
3080
|
+
accountId: ids.accountId,
|
|
3081
|
+
workspaceId: ids.workspaceId,
|
|
3082
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3083
|
+
expectedEpoch,
|
|
3084
|
+
rematerializationId,
|
|
3085
|
+
archiveSource,
|
|
3086
|
+
legacyNativeArchive
|
|
3087
|
+
});
|
|
3088
|
+
if (begun.status !== "started") {
|
|
3089
|
+
if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
|
|
3090
|
+
throw new SandboxLeaseSupersededError(
|
|
3091
|
+
ids.sandboxGroupId,
|
|
3092
|
+
begun.lease?.leaseEpoch ?? expectedEpoch
|
|
3093
|
+
);
|
|
3094
|
+
}
|
|
3095
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3096
|
+
ids.sandboxGroupId,
|
|
3097
|
+
begun.lease?.leaseEpoch ?? expectedEpoch,
|
|
3098
|
+
begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
|
|
3099
|
+
begun.lease?.recovery ?? acquired.lease.recovery
|
|
3100
|
+
);
|
|
3101
|
+
}
|
|
3102
|
+
spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
|
|
3103
|
+
const selectedRevision = begun.lease.recovery.restore.selectedRevision;
|
|
3104
|
+
if (!selectedRevision) {
|
|
3105
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3106
|
+
"archive_metadata_invalid",
|
|
3107
|
+
"sandbox rematerialization selected no durable archive revision"
|
|
3108
|
+
);
|
|
3109
|
+
}
|
|
3110
|
+
rematerialization = {
|
|
3111
|
+
id: rematerializationId,
|
|
3112
|
+
selectedRevision,
|
|
3113
|
+
workspaceGeneration: begun.lease.workspaceGeneration,
|
|
3114
|
+
providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
|
|
3115
|
+
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
3116
|
+
legacyProviderBinding: null
|
|
3117
|
+
};
|
|
3118
|
+
} else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
3119
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3120
|
+
ids.sandboxGroupId,
|
|
3121
|
+
expectedEpoch,
|
|
3122
|
+
"restore_degraded",
|
|
3123
|
+
acquired.lease.recovery
|
|
3124
|
+
);
|
|
3125
|
+
}
|
|
3126
|
+
const providerCreateStartedAt = /* @__PURE__ */ new Date();
|
|
3127
|
+
const persistArchiveSource = spawnEnvelope && typeof spawnEnvelope === "object" ? spawnEnvelope : null;
|
|
3128
|
+
const hydrateEnvelopeRaw = await materializeSpawnEnvelopeArchive(
|
|
3129
|
+
persistArchiveSource,
|
|
3130
|
+
services.objectStorage
|
|
3131
|
+
);
|
|
3132
|
+
const hydrateEnvelope = hydrateEnvelopeRaw && typeof hydrateEnvelopeRaw === "object" ? hydrateEnvelopeRaw : null;
|
|
3133
|
+
const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
|
|
3134
|
+
sessionId: ids.sessionId,
|
|
3135
|
+
recovery: "create-or-restore",
|
|
3136
|
+
backendOverride: ids.backend,
|
|
3137
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
3138
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
3139
|
+
...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
|
|
3140
|
+
onSandboxCreated: async (created) => {
|
|
3141
|
+
createdEstablished = created;
|
|
3142
|
+
providerRenewalTarget = {
|
|
3143
|
+
backend: ids.backend,
|
|
3144
|
+
instanceId: created.instanceId
|
|
3145
|
+
};
|
|
3146
|
+
providerRenewedAtMs = Date.now();
|
|
3147
|
+
throwIfReleasedOrCancelled();
|
|
3148
|
+
if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
|
|
3149
|
+
if (created.backendId !== "modal") {
|
|
3150
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3151
|
+
"native_snapshot_reference_invalid",
|
|
3152
|
+
"Modal checkpoint restore resolved a non-Modal sandbox backend"
|
|
3153
|
+
);
|
|
3154
|
+
}
|
|
3155
|
+
if (rematerialization.providerBindingKey) {
|
|
3156
|
+
if (!await modalSessionMatchesCheckpointProviderBinding(
|
|
3157
|
+
settings,
|
|
3158
|
+
created.session,
|
|
3159
|
+
rematerialization.providerBindingKey
|
|
3160
|
+
)) {
|
|
3161
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3162
|
+
"native_snapshot_reference_invalid",
|
|
3163
|
+
"Modal checkpoint restore refused because the authenticated provider workspace changed"
|
|
3164
|
+
);
|
|
3165
|
+
}
|
|
3166
|
+
} else {
|
|
3167
|
+
const resolved = await resolveModalCheckpointProviderBindingForSession(
|
|
3168
|
+
settings,
|
|
3169
|
+
created.session
|
|
3170
|
+
);
|
|
3171
|
+
if (rematerialization.legacyProviderBinding && rematerialization.legacyProviderBinding.key !== resolved.key) {
|
|
3172
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3173
|
+
"native_snapshot_reference_invalid",
|
|
3174
|
+
"Legacy Modal checkpoint restore crossed authenticated provider workspaces"
|
|
3175
|
+
);
|
|
3176
|
+
}
|
|
3177
|
+
rematerialization.legacyProviderBinding = resolved;
|
|
3178
|
+
}
|
|
3179
|
+
}
|
|
3180
|
+
const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
|
|
3181
|
+
await serializeReplacementSandboxEnvelope(created, persistArchiveSource),
|
|
3182
|
+
created.backendId
|
|
3183
|
+
);
|
|
3184
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
3185
|
+
accountId: ids.accountId,
|
|
3186
|
+
workspaceId: ids.workspaceId,
|
|
3187
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3188
|
+
expectedEpoch,
|
|
3189
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
3190
|
+
...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
|
|
3191
|
+
instanceId: created.instanceId,
|
|
3192
|
+
resumeBackendId: created.backendId,
|
|
3193
|
+
resumeState: resumeEnvelope2,
|
|
3194
|
+
...created.backendId === "modal" ? {
|
|
3195
|
+
providerCreatedAt: providerCreateStartedAt,
|
|
3196
|
+
providerDeadlineAt: new Date(
|
|
3197
|
+
providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1e3
|
|
3198
|
+
)
|
|
3199
|
+
} : {},
|
|
3200
|
+
leaseTtlMs,
|
|
3201
|
+
// Keep the warming budget after create(): manifest setup and
|
|
3202
|
+
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
3203
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs
|
|
3204
|
+
});
|
|
3205
|
+
if (!recorded.recorded) {
|
|
3206
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3207
|
+
}
|
|
3208
|
+
if (created.backendId === "modal") {
|
|
3209
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
3210
|
+
leaseId: acquired.lease.id,
|
|
3211
|
+
workspaceId: ids.workspaceId,
|
|
3212
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
3213
|
+
}).catch(() => void 0);
|
|
3214
|
+
}
|
|
3215
|
+
throwIfReleasedOrCancelled();
|
|
3216
|
+
},
|
|
3217
|
+
onWorkspaceRestoreVerifying: async (descriptor) => {
|
|
3218
|
+
if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
|
|
3219
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3220
|
+
"archive_metadata_invalid",
|
|
3221
|
+
`hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
|
|
3222
|
+
);
|
|
3223
|
+
}
|
|
3224
|
+
if (rematerialization.legacyCheckpoint) {
|
|
3225
|
+
const binding = rematerialization.legacyProviderBinding;
|
|
3226
|
+
if (!binding) {
|
|
3227
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3228
|
+
"native_snapshot_reference_invalid",
|
|
3229
|
+
"Legacy Modal checkpoint restore produced no authenticated provider identity"
|
|
3230
|
+
);
|
|
3231
|
+
}
|
|
3232
|
+
const adopted = await adoptLegacyModalCheckpointArtifact(db, {
|
|
3233
|
+
accountId: ids.accountId,
|
|
3234
|
+
workspaceId: ids.workspaceId,
|
|
3235
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3236
|
+
leaseId: acquired.lease.id,
|
|
3237
|
+
leaseEpoch: expectedEpoch,
|
|
3238
|
+
workspaceGeneration: rematerialization.workspaceGeneration,
|
|
3239
|
+
slot: "current",
|
|
3240
|
+
archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
|
|
3241
|
+
descriptor: rematerialization.legacyCheckpoint.descriptor,
|
|
3242
|
+
providerBindingKey: binding.key,
|
|
3243
|
+
providerBinding: binding.binding,
|
|
3244
|
+
rematerializationId: rematerialization.id
|
|
3245
|
+
});
|
|
3246
|
+
if (!adopted) {
|
|
3247
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3248
|
+
}
|
|
3249
|
+
rematerialization.providerBindingKey = binding.key;
|
|
3250
|
+
}
|
|
3251
|
+
const verifying = await markSandboxRestoreVerifying(db, {
|
|
3252
|
+
accountId: ids.accountId,
|
|
3253
|
+
workspaceId: ids.workspaceId,
|
|
3254
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3255
|
+
expectedEpoch,
|
|
3256
|
+
rematerializationId: rematerialization.id
|
|
3257
|
+
});
|
|
3258
|
+
if (!verifying.wrote) {
|
|
3259
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3262
|
+
});
|
|
3263
|
+
createdEstablished = established;
|
|
3264
|
+
throwIfReleasedOrCancelled();
|
|
3265
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
3266
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
3267
|
+
});
|
|
3268
|
+
await maybeRenewProviderExpiration(true);
|
|
3269
|
+
throwIfReleasedOrCancelled();
|
|
3270
|
+
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
3271
|
+
await serializeReplacementSandboxEnvelope(established, persistArchiveSource),
|
|
3272
|
+
established.backendId
|
|
3273
|
+
);
|
|
3274
|
+
throwIfReleasedOrCancelled();
|
|
3275
|
+
if (rematerialization && !established.providerContinuity && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
|
|
3276
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
3277
|
+
"workspace_fingerprint_mismatch",
|
|
3278
|
+
"sandbox restore completed without the exact selected durable archive revision"
|
|
3279
|
+
);
|
|
3280
|
+
}
|
|
3281
|
+
const committed = await commitWarmingToWarm(db, {
|
|
3282
|
+
accountId: ids.accountId,
|
|
3283
|
+
workspaceId: ids.workspaceId,
|
|
3284
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3285
|
+
expectedEpoch,
|
|
3286
|
+
instanceId: established.instanceId,
|
|
3287
|
+
// Viewer attach resolves and records the tunnel URL lazily. A headless
|
|
3288
|
+
// turn must not touch the optional desktop data plane.
|
|
3289
|
+
dataPlaneUrl: null,
|
|
3290
|
+
resumeBackendId: established.backendId,
|
|
3291
|
+
resumeState: resumeEnvelope,
|
|
3292
|
+
...established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
|
|
3293
|
+
rematerialization: {
|
|
3294
|
+
id: rematerialization.id,
|
|
3295
|
+
verifiedRevision: rematerialization.selectedRevision
|
|
3296
|
+
}
|
|
3297
|
+
} : {},
|
|
3298
|
+
leaseTtlMs
|
|
3299
|
+
});
|
|
3300
|
+
if (!committed.committed || !committed.lease) {
|
|
3301
|
+
const terminated = await terminateEstablishedSandbox(established);
|
|
3302
|
+
if (terminated && rematerialization && !established.providerContinuity) {
|
|
3303
|
+
await failSandboxRematerialization(db, {
|
|
3304
|
+
accountId: ids.accountId,
|
|
3305
|
+
workspaceId: ids.workspaceId,
|
|
3306
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3307
|
+
expectedEpoch,
|
|
3308
|
+
rematerializationId: rematerialization.id,
|
|
3309
|
+
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
3310
|
+
retryable: false
|
|
3311
|
+
});
|
|
3312
|
+
} else if (terminated) {
|
|
3313
|
+
await failWarmingToCold(db, {
|
|
3314
|
+
accountId: ids.accountId,
|
|
3315
|
+
workspaceId: ids.workspaceId,
|
|
3316
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3317
|
+
expectedEpoch
|
|
3318
|
+
});
|
|
3319
|
+
}
|
|
3320
|
+
await release();
|
|
3321
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
3322
|
+
}
|
|
3323
|
+
holderLeaseHeartbeat = {
|
|
3324
|
+
expectedEpoch: committed.lease.leaseEpoch,
|
|
3325
|
+
leaseTtlMs
|
|
3326
|
+
};
|
|
3327
|
+
throwIfReleasedOrCancelled();
|
|
3328
|
+
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
3329
|
+
} catch (error) {
|
|
3330
|
+
if (error instanceof SandboxLeaseSupersededError) {
|
|
3331
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
3332
|
+
await release();
|
|
3333
|
+
throw error;
|
|
3334
|
+
}
|
|
3335
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
3336
|
+
if (terminated) {
|
|
3337
|
+
const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
|
|
3338
|
+
if (rematerialization && !createdEstablished?.providerContinuity) {
|
|
3339
|
+
await failSandboxRematerialization(db, {
|
|
3340
|
+
accountId: ids.accountId,
|
|
3341
|
+
workspaceId: ids.workspaceId,
|
|
3342
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3343
|
+
expectedEpoch,
|
|
3344
|
+
rematerializationId: rematerialization.id,
|
|
3345
|
+
failureCode: continuityUnavailable ? "provider_continuity_unavailable" : error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
|
|
3346
|
+
retryable: continuityUnavailable ? error.retryable : error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
|
|
3347
|
+
...continuityUnavailable ? { discardContinuity: true } : {}
|
|
3348
|
+
});
|
|
3349
|
+
} else {
|
|
3350
|
+
await failWarmingToCold(db, {
|
|
3351
|
+
accountId: ids.accountId,
|
|
3352
|
+
workspaceId: ids.workspaceId,
|
|
3353
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3354
|
+
expectedEpoch,
|
|
3355
|
+
...continuityUnavailable ? { discardContinuity: true } : {}
|
|
3356
|
+
});
|
|
3357
|
+
}
|
|
3358
|
+
}
|
|
3359
|
+
await release();
|
|
3360
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
3361
|
+
throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
|
|
3362
|
+
}
|
|
3363
|
+
}
|
|
3364
|
+
try {
|
|
3365
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
3366
|
+
if (acquired.lease.liveness === "warming") {
|
|
3367
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
3368
|
+
holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
|
|
3369
|
+
}
|
|
3370
|
+
throwIfReleasedOrCancelled();
|
|
3371
|
+
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
3372
|
+
if (!live || live.liveness !== "warm" || live.leaseEpoch !== leaseEpoch || live.instanceId === null) {
|
|
3373
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
|
|
3374
|
+
}
|
|
3375
|
+
let established;
|
|
3376
|
+
try {
|
|
3377
|
+
const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
|
|
3378
|
+
established = await establish(settings, live.resumeState, {
|
|
3379
|
+
sessionId: ids.sessionId,
|
|
3380
|
+
recovery: "resume-only",
|
|
3381
|
+
backendOverride: ids.backend,
|
|
3382
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
3383
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
3384
|
+
});
|
|
3385
|
+
throwIfReleasedOrCancelled();
|
|
3386
|
+
if (services.verifyAttachedSandboxReadiness) {
|
|
3387
|
+
await services.verifyAttachedSandboxReadiness(established);
|
|
3388
|
+
} else {
|
|
3389
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
3390
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
3391
|
+
});
|
|
3392
|
+
}
|
|
3393
|
+
providerRenewalTarget = {
|
|
3394
|
+
backend: ids.backend,
|
|
3395
|
+
instanceId: established.instanceId
|
|
3396
|
+
};
|
|
3397
|
+
await maybeRenewProviderExpiration(true);
|
|
3398
|
+
throwIfReleasedOrCancelled();
|
|
3399
|
+
} catch (error) {
|
|
3400
|
+
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
3401
|
+
throw sandboxProvisionStageError("resume", error);
|
|
3402
|
+
}
|
|
3403
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
3404
|
+
accountId: ids.accountId,
|
|
3405
|
+
workspaceId: ids.workspaceId,
|
|
3406
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3407
|
+
expectedEpoch: leaseEpoch,
|
|
3408
|
+
expectedInstanceId: live.instanceId
|
|
3409
|
+
});
|
|
3410
|
+
if (marked.status === "marked") {
|
|
3411
|
+
await services.onSandboxLost?.({
|
|
3412
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3413
|
+
instanceId: live.instanceId,
|
|
3414
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
3415
|
+
});
|
|
3416
|
+
throw new SandboxLeaseInstanceLostError(
|
|
3417
|
+
ids.sandboxGroupId,
|
|
3418
|
+
marked.lease.leaseEpoch,
|
|
3419
|
+
live.instanceId
|
|
3420
|
+
);
|
|
3421
|
+
}
|
|
3422
|
+
throw new SandboxLeaseSupersededError(
|
|
3423
|
+
ids.sandboxGroupId,
|
|
3424
|
+
marked.lease?.leaseEpoch ?? leaseEpoch
|
|
3425
|
+
);
|
|
3426
|
+
}
|
|
3427
|
+
throwIfReleasedOrCancelled();
|
|
3428
|
+
return { established, leaseEpoch, release };
|
|
3429
|
+
} catch (error) {
|
|
3430
|
+
await release();
|
|
3431
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
3432
|
+
throw error;
|
|
3433
|
+
}
|
|
3434
|
+
}
|
|
3435
|
+
async function waitForWarm(services, ids) {
|
|
3436
|
+
const { db, settings } = services;
|
|
3437
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
3438
|
+
let instanceId = null;
|
|
3439
|
+
while (Date.now() < deadline) {
|
|
3440
|
+
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
3441
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
3442
|
+
if (!lease) {
|
|
3443
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
3444
|
+
}
|
|
3445
|
+
instanceId = lease.instanceId;
|
|
3446
|
+
if (lease.liveness === "warm") {
|
|
3447
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
3448
|
+
}
|
|
3449
|
+
if (lease.liveness === "draining") {
|
|
3450
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3451
|
+
}
|
|
3452
|
+
if (lease.liveness === "cold") {
|
|
3453
|
+
if (isRetryableDegradedRestore(lease.recovery.restore)) {
|
|
3454
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3455
|
+
}
|
|
3456
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
|
|
3457
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
3458
|
+
ids.sandboxGroupId,
|
|
3459
|
+
lease.leaseEpoch,
|
|
3460
|
+
lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
|
|
3461
|
+
lease.recovery
|
|
3462
|
+
);
|
|
3463
|
+
}
|
|
3464
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
3465
|
+
}
|
|
3466
|
+
}
|
|
3467
|
+
throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
|
|
3468
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
3469
|
+
instanceId
|
|
3470
|
+
});
|
|
3471
|
+
}
|
|
3472
|
+
|
|
3473
|
+
// src/activities/agent-turn/sandbox-provision.ts
|
|
3474
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
3475
|
+
var TurnOperationCancelledError = class extends Error {
|
|
3476
|
+
constructor(reason) {
|
|
3477
|
+
super("Turn operation was cancelled with its owning turn", {
|
|
3478
|
+
...reason instanceof Error ? { cause: reason } : {}
|
|
3479
|
+
});
|
|
3480
|
+
this.reason = reason;
|
|
3481
|
+
}
|
|
3482
|
+
name = "TurnOperationCancelledError";
|
|
3483
|
+
};
|
|
3484
|
+
var SandboxDeadlineRotationError = class extends Error {
|
|
3485
|
+
constructor(sandboxGroupId, leaseEpoch) {
|
|
3486
|
+
super(
|
|
3487
|
+
`Sandbox ${sandboxGroupId} reached its provider rotation boundary at lease epoch ${leaseEpoch}`
|
|
3488
|
+
);
|
|
3489
|
+
this.sandboxGroupId = sandboxGroupId;
|
|
3490
|
+
this.leaseEpoch = leaseEpoch;
|
|
3491
|
+
}
|
|
3492
|
+
name = "SandboxDeadlineRotationError";
|
|
3493
|
+
};
|
|
3494
|
+
function turnOperationCancellationFailure(error) {
|
|
3495
|
+
if (error instanceof CancelledFailure) return error;
|
|
3496
|
+
if (!(error instanceof TurnOperationCancelledError)) return null;
|
|
3497
|
+
return error.reason instanceof CancelledFailure ? error.reason : new CancelledFailure("TURN_SANDBOX_PROVISION_CANCELLED", [], error);
|
|
3498
|
+
}
|
|
3499
|
+
function throwIfTurnOperationCancelled(signal) {
|
|
3500
|
+
if (signal?.aborted) {
|
|
3501
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
3502
|
+
}
|
|
3503
|
+
}
|
|
3504
|
+
async function waitForTurnOperation(operation, signal, disposeLateResult) {
|
|
3505
|
+
if (!signal) return await operation;
|
|
3506
|
+
let rejectCancellation;
|
|
3507
|
+
const cancelled = new Promise((_resolve, reject) => {
|
|
3508
|
+
rejectCancellation = reject;
|
|
3509
|
+
});
|
|
3510
|
+
const cancel = () => {
|
|
3511
|
+
rejectCancellation?.(new TurnOperationCancelledError(signal.reason));
|
|
3512
|
+
};
|
|
3513
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
3514
|
+
if (signal.aborted) cancel();
|
|
3515
|
+
try {
|
|
3516
|
+
const result = await Promise.race([operation, cancelled]);
|
|
3517
|
+
if (signal.aborted) {
|
|
3518
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
3519
|
+
}
|
|
3520
|
+
return result;
|
|
3521
|
+
} catch (error) {
|
|
3522
|
+
if (signal.aborted) {
|
|
3523
|
+
void operation.then(async (result) => await disposeLateResult?.(result)).catch(() => void 0);
|
|
3524
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
3525
|
+
throw new TurnOperationCancelledError(signal.reason);
|
|
3526
|
+
}
|
|
3527
|
+
}
|
|
3528
|
+
throw error;
|
|
3529
|
+
} finally {
|
|
3530
|
+
signal.removeEventListener("abort", cancel);
|
|
3531
|
+
}
|
|
3532
|
+
}
|
|
3533
|
+
function sleep2(ms) {
|
|
3534
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3535
|
+
}
|
|
3536
|
+
function sandboxProvisionErrorChain(error) {
|
|
3537
|
+
const chain = [];
|
|
3538
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3539
|
+
let current = error;
|
|
3540
|
+
for (let depth = 0; depth < 6; depth += 1) {
|
|
3541
|
+
chain.push(current);
|
|
3542
|
+
if (!current || typeof current !== "object" || seen.has(current)) break;
|
|
3543
|
+
seen.add(current);
|
|
3544
|
+
if (current instanceof SandboxProvisionStageError) {
|
|
3545
|
+
current = current.source;
|
|
3546
|
+
continue;
|
|
3547
|
+
}
|
|
3548
|
+
const cause = Reflect.get(current, "cause");
|
|
3549
|
+
if (cause === void 0 || cause === current) break;
|
|
3550
|
+
current = cause;
|
|
3551
|
+
}
|
|
3552
|
+
return chain;
|
|
3553
|
+
}
|
|
3554
|
+
function firstProvisionError(chain, constructor) {
|
|
3555
|
+
return chain.find((value) => value instanceof constructor) ?? null;
|
|
3556
|
+
}
|
|
3557
|
+
function classifySandboxLogicalProvisionFailure(backend, error) {
|
|
3558
|
+
const chain = sandboxProvisionErrorChain(error);
|
|
3559
|
+
const wrappedStage = chain.map((value) => sandboxProvisionFailureStage(value)).find((stage) => stage !== null) ?? null;
|
|
3560
|
+
const warming = chain.find(
|
|
3561
|
+
(value) => value instanceof SandboxWarmingTimeoutError
|
|
3562
|
+
) ?? null;
|
|
3563
|
+
if (warming) {
|
|
3564
|
+
const execReadiness = warming.stage === "exec_readiness";
|
|
3565
|
+
return {
|
|
3566
|
+
category: execReadiness ? "exec_readiness" : "sibling_warming",
|
|
3567
|
+
stage: execReadiness ? "exec_readiness" : "sibling_warming",
|
|
3568
|
+
code: warming.code,
|
|
3569
|
+
expected: false,
|
|
3570
|
+
retryable: false
|
|
3571
|
+
};
|
|
3572
|
+
}
|
|
3573
|
+
const readiness = firstProvisionError(chain, SandboxExecReadinessError2);
|
|
3574
|
+
if (readiness) {
|
|
3575
|
+
return {
|
|
3576
|
+
category: "exec_readiness",
|
|
3577
|
+
stage: "exec_readiness",
|
|
3578
|
+
code: readiness.code,
|
|
3579
|
+
expected: false,
|
|
3580
|
+
retryable: false
|
|
3581
|
+
};
|
|
3582
|
+
}
|
|
3583
|
+
const superseded = firstProvisionError(chain, SandboxLeaseSupersededError2);
|
|
3584
|
+
if (superseded) {
|
|
3585
|
+
return {
|
|
3586
|
+
category: "lease_superseded",
|
|
3587
|
+
stage: "lease_admission",
|
|
3588
|
+
code: "lease_superseded",
|
|
3589
|
+
expected: true,
|
|
3590
|
+
retryable: true
|
|
3591
|
+
};
|
|
3592
|
+
}
|
|
3593
|
+
const transition = firstProvisionError(chain, SandboxLeaseTransitionError2);
|
|
3594
|
+
if (transition) {
|
|
3595
|
+
return {
|
|
3596
|
+
category: "drain_capture_wait",
|
|
3597
|
+
stage: "lifecycle_wait",
|
|
3598
|
+
code: transition.reason,
|
|
3599
|
+
expected: true,
|
|
3600
|
+
retryable: true
|
|
3601
|
+
};
|
|
3602
|
+
}
|
|
3603
|
+
const deadlineRotation = firstProvisionError(chain, SandboxDeadlineRotationError);
|
|
3604
|
+
if (deadlineRotation) {
|
|
3605
|
+
return {
|
|
3606
|
+
category: "drain_capture_wait",
|
|
3607
|
+
stage: "lifecycle_wait",
|
|
3608
|
+
code: "provider_deadline_rotation",
|
|
3609
|
+
expected: true,
|
|
3610
|
+
retryable: false
|
|
3611
|
+
};
|
|
3612
|
+
}
|
|
3613
|
+
const recoveryBlocked = firstProvisionError(chain, SandboxLeaseRecoveryBlockedError2);
|
|
3614
|
+
if (recoveryBlocked) {
|
|
3615
|
+
return {
|
|
3616
|
+
category: "archive_recovery",
|
|
3617
|
+
stage: "archive_recovery",
|
|
3618
|
+
code: recoveryBlocked.code,
|
|
3619
|
+
expected: false,
|
|
3620
|
+
retryable: false
|
|
3621
|
+
};
|
|
3622
|
+
}
|
|
3623
|
+
const archive = firstProvisionError(chain, WorkspaceArchiveIntegrityError2);
|
|
3624
|
+
if (archive) {
|
|
3625
|
+
return {
|
|
3626
|
+
category: "archive_recovery",
|
|
3627
|
+
stage: "archive_recovery",
|
|
3628
|
+
code: archive.code,
|
|
3629
|
+
expected: false,
|
|
3630
|
+
retryable: archive.retryable
|
|
3631
|
+
};
|
|
3632
|
+
}
|
|
3633
|
+
const continuity = firstProvisionError(chain, SandboxProviderContinuityUnavailableError2);
|
|
3634
|
+
if (continuity) {
|
|
3635
|
+
return {
|
|
3636
|
+
category: "archive_recovery",
|
|
3637
|
+
stage: "archive_recovery",
|
|
3638
|
+
code: "provider_continuity_unavailable",
|
|
3639
|
+
expected: false,
|
|
3640
|
+
retryable: continuity.retryable
|
|
3641
|
+
};
|
|
3642
|
+
}
|
|
3643
|
+
const configuration = chain.find(
|
|
3644
|
+
(value) => value instanceof SandboxConfigError || value instanceof SandboxProviderUnavailableError || value instanceof SandboxImageConflictError || value instanceof SandboxRigConflictError
|
|
3645
|
+
);
|
|
3646
|
+
if (configuration) {
|
|
3647
|
+
return {
|
|
3648
|
+
category: "configuration",
|
|
3649
|
+
stage: wrappedStage ?? "configuration",
|
|
3650
|
+
code: configuration instanceof SandboxImageConflictError ? "image_conflict" : configuration instanceof SandboxRigConflictError ? "rig_conflict" : configuration instanceof SandboxProviderUnavailableError ? "provider_unavailable" : "sandbox_config",
|
|
3651
|
+
expected: false,
|
|
3652
|
+
retryable: false
|
|
3653
|
+
};
|
|
3654
|
+
}
|
|
3655
|
+
const resumeFailure = chain.find(
|
|
3656
|
+
(value) => value instanceof SandboxResumeStateUnavailableError || value instanceof SandboxResumeIdentityMismatchError || value instanceof SandboxResumeIdentityUnavailableError || value instanceof SandboxExactResumeReplacedError || value instanceof SandboxExactResumeInstanceUnavailableError
|
|
3657
|
+
);
|
|
3658
|
+
if (resumeFailure) {
|
|
3659
|
+
return {
|
|
3660
|
+
category: "resume",
|
|
3661
|
+
stage: "resume",
|
|
3662
|
+
code: resumeFailure instanceof SandboxResumeStateUnavailableError ? "resume_state_unavailable" : resumeFailure instanceof SandboxResumeIdentityMismatchError ? "resume_identity_mismatch" : resumeFailure instanceof SandboxResumeIdentityUnavailableError ? "resume_identity_unavailable" : resumeFailure instanceof SandboxExactResumeReplacedError ? "exact_resume_replaced" : "exact_resume_instance_unavailable",
|
|
3663
|
+
expected: false,
|
|
3664
|
+
retryable: false
|
|
3665
|
+
};
|
|
3666
|
+
}
|
|
3667
|
+
const providerFailure = classifyProviderSandboxFailure(backend, error);
|
|
3668
|
+
if (providerFailure.kind === "transient_transport") {
|
|
3669
|
+
return {
|
|
3670
|
+
category: "provider_transport",
|
|
3671
|
+
stage: wrappedStage ?? "provider_transport",
|
|
3672
|
+
code: providerFailure.diagnostic,
|
|
3673
|
+
expected: false,
|
|
3674
|
+
// A typed transport fault is attributable, but a create outcome may be
|
|
3675
|
+
// ambiguous. Classification never licenses replay on its own.
|
|
3676
|
+
retryable: false
|
|
3677
|
+
};
|
|
3678
|
+
}
|
|
3679
|
+
if (wrappedStage) {
|
|
3680
|
+
return {
|
|
3681
|
+
category: wrappedStage,
|
|
3682
|
+
stage: wrappedStage,
|
|
3683
|
+
code: `${wrappedStage}_failed`,
|
|
3684
|
+
expected: false,
|
|
3685
|
+
retryable: false
|
|
3686
|
+
};
|
|
3687
|
+
}
|
|
3688
|
+
return {
|
|
3689
|
+
category: "unknown",
|
|
3690
|
+
stage: "unknown",
|
|
3691
|
+
code: "unknown",
|
|
3692
|
+
expected: false,
|
|
3693
|
+
retryable: false
|
|
3694
|
+
};
|
|
3695
|
+
}
|
|
3696
|
+
function isLazySandboxProvisionRetryable(error) {
|
|
3697
|
+
const chain = sandboxProvisionErrorChain(error);
|
|
3698
|
+
if (chain.some(
|
|
3699
|
+
(value) => value instanceof SandboxImageConflictError || value instanceof SandboxRigConflictError || value instanceof SandboxLeaseRecoveryBlockedError2
|
|
3700
|
+
)) {
|
|
3701
|
+
return false;
|
|
3702
|
+
}
|
|
3703
|
+
const archive = firstProvisionError(chain, WorkspaceArchiveIntegrityError2);
|
|
3704
|
+
if (archive) {
|
|
3705
|
+
return archive.retryable;
|
|
3706
|
+
}
|
|
3707
|
+
if (chain.some(
|
|
3708
|
+
(value) => value instanceof SandboxLeaseSupersededError2 || value instanceof SandboxLeaseTransitionError2
|
|
3709
|
+
)) {
|
|
3710
|
+
return true;
|
|
3711
|
+
}
|
|
3712
|
+
if (chain.some((value) => value instanceof SandboxWarmingTimeoutError)) {
|
|
3713
|
+
return false;
|
|
3714
|
+
}
|
|
3715
|
+
return false;
|
|
3716
|
+
}
|
|
3717
|
+
function sandboxDeadlineRotationRecoveryDelayMs(settings) {
|
|
3718
|
+
return Math.min(5e3, settings.sandboxLeaseReaperPeriodMs);
|
|
3719
|
+
}
|
|
3720
|
+
function createTurnSandboxProvisioner(establish, options = {}) {
|
|
3721
|
+
const maxRetries = options.maxRetries ?? 2;
|
|
3722
|
+
const backoffMs = options.backoffMs ?? 250;
|
|
3723
|
+
let memo = null;
|
|
3724
|
+
const run = async (provisionId, onAttempt) => {
|
|
3725
|
+
let internalAttempts = 0;
|
|
3726
|
+
while (true) {
|
|
3727
|
+
internalAttempts += 1;
|
|
3728
|
+
onAttempt(internalAttempts);
|
|
3729
|
+
const startedAt = performance.now();
|
|
3730
|
+
try {
|
|
3731
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
3732
|
+
const operation = establish();
|
|
3733
|
+
const result = await waitForTurnOperation(operation, options.signal, options.disposeResult);
|
|
3734
|
+
await options.onAttemptSettled?.({
|
|
3735
|
+
provisionId,
|
|
3736
|
+
attempt: internalAttempts,
|
|
3737
|
+
outcome: "completed",
|
|
3738
|
+
durationMs: performance.now() - startedAt,
|
|
3739
|
+
result
|
|
3740
|
+
});
|
|
3741
|
+
return { result, internalAttempts };
|
|
3742
|
+
} catch (error) {
|
|
3743
|
+
const retrying = !(error instanceof TurnOperationCancelledError || internalAttempts > maxRetries || !isLazySandboxProvisionRetryable(error));
|
|
3744
|
+
await options.onAttemptSettled?.({
|
|
3745
|
+
provisionId,
|
|
3746
|
+
attempt: internalAttempts,
|
|
3747
|
+
outcome: retrying ? "retrying" : "failed",
|
|
3748
|
+
durationMs: performance.now() - startedAt,
|
|
3749
|
+
error
|
|
3750
|
+
});
|
|
3751
|
+
if (!retrying) {
|
|
3752
|
+
throw error;
|
|
3753
|
+
}
|
|
3754
|
+
await sleep2(backoffMs * internalAttempts);
|
|
3755
|
+
}
|
|
3756
|
+
}
|
|
3757
|
+
};
|
|
3758
|
+
return {
|
|
3759
|
+
get() {
|
|
3760
|
+
if (!memo) {
|
|
3761
|
+
const provisionId = options.provisionIdFactory?.() ?? randomUUID3();
|
|
3762
|
+
memo = (async () => {
|
|
3763
|
+
const startedAt = performance.now();
|
|
3764
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
3765
|
+
await options.onStarted?.({ provisionId });
|
|
3766
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
3767
|
+
let result;
|
|
3768
|
+
let internalAttempts = 0;
|
|
3769
|
+
let hasResult = false;
|
|
3770
|
+
try {
|
|
3771
|
+
const runResult = await run(provisionId, (attempt) => {
|
|
3772
|
+
internalAttempts = attempt;
|
|
3773
|
+
});
|
|
3774
|
+
result = runResult.result;
|
|
3775
|
+
internalAttempts = runResult.internalAttempts;
|
|
3776
|
+
hasResult = true;
|
|
3777
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
3778
|
+
await options.beforeCompleted?.(result, {
|
|
3779
|
+
provisionId,
|
|
3780
|
+
internalAttempts,
|
|
3781
|
+
durationMs: performance.now() - startedAt
|
|
3782
|
+
});
|
|
3783
|
+
throwIfTurnOperationCancelled(options.signal);
|
|
3784
|
+
await options.onCompleted?.(result, {
|
|
3785
|
+
provisionId,
|
|
3786
|
+
internalAttempts,
|
|
3787
|
+
durationMs: performance.now() - startedAt
|
|
3788
|
+
});
|
|
3789
|
+
return result;
|
|
3790
|
+
} catch (error) {
|
|
3791
|
+
if (hasResult) {
|
|
3792
|
+
await options.disposeResult?.(result);
|
|
3793
|
+
}
|
|
3794
|
+
if (!(error instanceof TurnOperationCancelledError)) {
|
|
3795
|
+
await options.onFailed?.(error, {
|
|
3796
|
+
provisionId,
|
|
3797
|
+
internalAttempts: Math.max(1, internalAttempts),
|
|
3798
|
+
durationMs: performance.now() - startedAt
|
|
3799
|
+
});
|
|
3800
|
+
}
|
|
3801
|
+
throw error;
|
|
3802
|
+
}
|
|
3803
|
+
})().catch((error) => {
|
|
3804
|
+
if (isLazySandboxProvisionRetryable(error)) {
|
|
3805
|
+
memo = null;
|
|
3806
|
+
}
|
|
3807
|
+
throw error;
|
|
3808
|
+
});
|
|
3809
|
+
}
|
|
3810
|
+
return memo;
|
|
3811
|
+
},
|
|
3812
|
+
hasStarted() {
|
|
3813
|
+
return memo !== null;
|
|
3814
|
+
},
|
|
3815
|
+
async waitForSettled(timeoutMs) {
|
|
3816
|
+
if (!memo) {
|
|
3817
|
+
return null;
|
|
3818
|
+
}
|
|
3819
|
+
let timer;
|
|
3820
|
+
try {
|
|
3821
|
+
return await Promise.race([
|
|
3822
|
+
memo.catch(() => null),
|
|
3823
|
+
new Promise((resolve) => {
|
|
3824
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
3825
|
+
})
|
|
3826
|
+
]);
|
|
3827
|
+
} finally {
|
|
3828
|
+
if (timer) {
|
|
3829
|
+
clearTimeout(timer);
|
|
3830
|
+
}
|
|
3831
|
+
}
|
|
3832
|
+
}
|
|
3833
|
+
};
|
|
3834
|
+
}
|
|
3835
|
+
|
|
3836
|
+
// src/activities/agent-turn/quiescence.ts
|
|
3837
|
+
function shouldRunTurnEndWorkspacePersistence(input) {
|
|
3838
|
+
return input.activityStatus !== "cancelled" && !input.cancellationRequested;
|
|
3839
|
+
}
|
|
3840
|
+
function shouldStartPeriodicWorkspaceSnapshot(input) {
|
|
3841
|
+
return input.firstProviderRequestStarted && !input.snapshotInFlight && !input.turnEndCaptureInProgress;
|
|
3842
|
+
}
|
|
3843
|
+
function assertPhysicalToolQuiescenceForCancellation(input) {
|
|
3844
|
+
if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
|
|
3845
|
+
if (input.failure instanceof Error) throw input.failure;
|
|
3846
|
+
throw new Error("Physical sandbox-tool quiescence could not be confirmed", {
|
|
3847
|
+
cause: input.failure
|
|
3848
|
+
});
|
|
3849
|
+
}
|
|
3850
|
+
function assertSessionAttemptQuiescenceRecoveryDurable(input) {
|
|
3851
|
+
if (!input.acknowledgeQuiescence || !input.physicalToolQuiescenceConfirmed || input.receiptOrProofDurable) {
|
|
3852
|
+
return;
|
|
3853
|
+
}
|
|
3854
|
+
if (input.failure instanceof Error) throw input.failure;
|
|
3855
|
+
throw new Error("Physical quiescence had no durable receipt or recovery proof", {
|
|
3856
|
+
cause: input.failure
|
|
3857
|
+
});
|
|
3858
|
+
}
|
|
3859
|
+
var QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
3860
|
+
var QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5e3;
|
|
3861
|
+
async function persistOrSignalSessionAttemptQuiescence(input) {
|
|
3862
|
+
let receipt;
|
|
3863
|
+
try {
|
|
3864
|
+
receipt = await input.persistReceipt();
|
|
3865
|
+
} catch (receiptError) {
|
|
3866
|
+
input.onReceiptFailure?.(receiptError);
|
|
3867
|
+
if (!input.signalProof) {
|
|
3868
|
+
throw new Error("Session-attempt quiescence proof signaler is unavailable", {
|
|
3869
|
+
cause: receiptError
|
|
3870
|
+
});
|
|
3871
|
+
}
|
|
3872
|
+
const delay = input.sleep ?? sleep2;
|
|
3873
|
+
let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
|
|
3874
|
+
let attempt = 1;
|
|
3875
|
+
for (; ; ) {
|
|
3876
|
+
try {
|
|
3877
|
+
await input.signalProof(input.proof);
|
|
3878
|
+
return "signal";
|
|
3879
|
+
} catch (signalError) {
|
|
3880
|
+
input.onSignalFailure?.(signalError, attempt, retryMs);
|
|
3881
|
+
try {
|
|
3882
|
+
input.heartbeat?.(attempt, retryMs);
|
|
3883
|
+
} catch {
|
|
3884
|
+
}
|
|
3885
|
+
await delay(retryMs);
|
|
3886
|
+
retryMs = Math.min(retryMs * 2, QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS);
|
|
3887
|
+
attempt += 1;
|
|
3888
|
+
}
|
|
3889
|
+
}
|
|
3890
|
+
}
|
|
3891
|
+
if (receipt.workflowWake && input.deliverWorkflowWake) {
|
|
3892
|
+
try {
|
|
3893
|
+
await input.deliverWorkflowWake(receipt.workflowWake);
|
|
3894
|
+
} catch (wakeError) {
|
|
3895
|
+
input.onWakeFailure?.(wakeError);
|
|
3896
|
+
}
|
|
3897
|
+
}
|
|
3898
|
+
try {
|
|
3899
|
+
await input.publishEvents(receipt.events);
|
|
3900
|
+
} catch (publishError) {
|
|
3901
|
+
input.onPublishFailure?.(publishError);
|
|
3902
|
+
}
|
|
3903
|
+
return "receipt";
|
|
3904
|
+
}
|
|
3905
|
+
async function drainAttemptOwnedSandboxWriters(input) {
|
|
3906
|
+
if (input.toolCancellationFence) {
|
|
3907
|
+
input.toolCancellationFence.cancel(
|
|
3908
|
+
input.cancellationReason ?? new Error("TURN_ATTEMPT_FENCED")
|
|
3909
|
+
);
|
|
3910
|
+
await input.toolCancellationFence.waitForQuiescence();
|
|
3911
|
+
}
|
|
3912
|
+
await Promise.all(input.gitCredentialRenewals.map(async (renewal) => await renewal.stop()));
|
|
3913
|
+
await input.codemodeTokenRenewal?.stop();
|
|
3914
|
+
await input.runCredentialRenewal?.stop();
|
|
3915
|
+
}
|
|
3916
|
+
async function releaseTurnSandboxAfterWriterDrain(sandbox, options = {}) {
|
|
3917
|
+
const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
|
|
3918
|
+
const retryDelayMs = Math.max(0, options.retryDelayMs ?? 50);
|
|
3919
|
+
const wait = options.wait ?? sleep2;
|
|
3920
|
+
let lastError;
|
|
3921
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
3922
|
+
try {
|
|
3923
|
+
await sandbox.release({ workspaceWritersQuiesced: true });
|
|
3924
|
+
return;
|
|
3925
|
+
} catch (error) {
|
|
3926
|
+
lastError = error;
|
|
3927
|
+
if (attempt < maxAttempts) {
|
|
3928
|
+
await wait(retryDelayMs * attempt);
|
|
3929
|
+
}
|
|
3930
|
+
}
|
|
3931
|
+
}
|
|
3932
|
+
throw lastError;
|
|
3933
|
+
}
|
|
3934
|
+
async function waitForTurnFinalizerStep(operation, signal) {
|
|
3935
|
+
if (!signal) return await operation;
|
|
3936
|
+
if (signal.aborted) {
|
|
3937
|
+
void operation.catch(() => void 0);
|
|
3938
|
+
return void 0;
|
|
3939
|
+
}
|
|
3940
|
+
let resolveCancellation;
|
|
3941
|
+
const cancelled = new Promise((resolve) => {
|
|
3942
|
+
resolveCancellation = () => resolve(void 0);
|
|
3943
|
+
});
|
|
3944
|
+
const cancel = () => {
|
|
3945
|
+
void operation.catch(() => void 0);
|
|
3946
|
+
resolveCancellation?.();
|
|
3947
|
+
};
|
|
3948
|
+
signal.addEventListener("abort", cancel, { once: true });
|
|
3949
|
+
try {
|
|
3950
|
+
return await Promise.race([operation, cancelled]);
|
|
3951
|
+
} finally {
|
|
3952
|
+
signal.removeEventListener("abort", cancel);
|
|
3953
|
+
}
|
|
3954
|
+
}
|
|
3955
|
+
async function waitForTurnStreamCleanup(batcherFlush, providerCompleted, signal) {
|
|
3956
|
+
await waitForTurnFinalizerStep(batcherFlush, signal);
|
|
3957
|
+
await waitForTurnFinalizerStep(providerCompleted, signal);
|
|
3958
|
+
}
|
|
3959
|
+
async function clearAttemptCredentialsWithSettledFence(input) {
|
|
3960
|
+
try {
|
|
3961
|
+
await input.runWorkspaceFencedClear();
|
|
3962
|
+
} catch (error) {
|
|
3963
|
+
const settledAttemptFence = error instanceof Error && error.name === "SandboxWorkspaceMutationFencedError" && error.code === "attempt_fenced" && (input.activityStatus === "idle" || input.activityStatus === "failed" || input.activityStatus === "cancelled");
|
|
3964
|
+
if (!settledAttemptFence) throw error;
|
|
3965
|
+
input.onSettledAttemptFence();
|
|
3966
|
+
await input.clearExactAttempt();
|
|
3967
|
+
}
|
|
3968
|
+
}
|
|
3969
|
+
function turnFinalizerCancellationSignal(temporalSignal, activityStatus) {
|
|
3970
|
+
if (activityStatus !== "cancelled" || temporalSignal?.aborted) return temporalSignal;
|
|
3971
|
+
const fenced = new AbortController();
|
|
3972
|
+
fenced.abort(new Error("TURN_ATTEMPT_FENCED"));
|
|
3973
|
+
return fenced.signal;
|
|
3974
|
+
}
|
|
3975
|
+
var MandatoryHistoryPersistenceError = class extends Error {
|
|
3976
|
+
constructor(stage, cause) {
|
|
3977
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
3978
|
+
this.stage = stage;
|
|
3979
|
+
this.cause = cause;
|
|
3980
|
+
}
|
|
3981
|
+
name = "MandatoryHistoryPersistenceError";
|
|
3982
|
+
cause;
|
|
3983
|
+
};
|
|
3984
|
+
async function runMandatoryHistoryPersistenceStep(stage, operation) {
|
|
3985
|
+
try {
|
|
3986
|
+
return await operation();
|
|
3987
|
+
} catch (cause) {
|
|
3988
|
+
throw new MandatoryHistoryPersistenceError(stage, cause);
|
|
3989
|
+
}
|
|
3990
|
+
}
|
|
3991
|
+
async function finalizeDurableTurnOpStreams(sessions, fallback) {
|
|
3992
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
3993
|
+
for (const session of sessions) {
|
|
3994
|
+
const candidate = session;
|
|
3995
|
+
if (typeof candidate?.finalizeOpStreamOps === "function") {
|
|
3996
|
+
candidates.add(candidate);
|
|
3997
|
+
}
|
|
3998
|
+
}
|
|
3999
|
+
if (candidates.size === 0 && fallback) {
|
|
4000
|
+
candidates.add(fallback);
|
|
4001
|
+
}
|
|
4002
|
+
for (const candidate of candidates) {
|
|
4003
|
+
try {
|
|
4004
|
+
await candidate.finalizeOpStreamOps();
|
|
4005
|
+
} catch {
|
|
4006
|
+
}
|
|
4007
|
+
}
|
|
4008
|
+
}
|
|
4009
|
+
|
|
4010
|
+
// src/activities/agent-turn/errors.ts
|
|
4011
|
+
import {
|
|
4012
|
+
ActiveSessionHistoryLimitExceededError,
|
|
4013
|
+
ApprovalRunStateLimitExceededError,
|
|
4014
|
+
isRetryableDatabaseTransportFailure,
|
|
4015
|
+
isSessionEventPersistenceError
|
|
4016
|
+
} from "@opengeni/db";
|
|
4017
|
+
import {
|
|
4018
|
+
ActiveBackendUnresolvableError,
|
|
4019
|
+
CompactionProviderResponseError,
|
|
4020
|
+
EmptyCompactionSummaryError,
|
|
4021
|
+
isMcpRequestTimeoutError,
|
|
4022
|
+
isMcpTransportConnectivityError,
|
|
4023
|
+
RoutingWorkspaceRootChangedError,
|
|
4024
|
+
SelfhostedWorkspaceRootChangedError,
|
|
4025
|
+
UNKNOWN_MODEL_FINISH_REASON_CODE
|
|
4026
|
+
} from "@opengeni/runtime";
|
|
4027
|
+
import {
|
|
4028
|
+
mcpTransportRequestFailureDiagnostic
|
|
4029
|
+
} from "@opengeni/runtime/mcp-network";
|
|
4030
|
+
import { ApplicationFailure, CancelledFailure as CancelledFailure2 } from "@temporalio/activity";
|
|
4031
|
+
import {
|
|
4032
|
+
CodexReloginRequired,
|
|
4033
|
+
classifyCodexResponseTimeoutError,
|
|
4034
|
+
classifyCodexUsageLimitError,
|
|
4035
|
+
isCodexTransportError
|
|
4036
|
+
} from "@opengeni/codex";
|
|
4037
|
+
import {
|
|
4038
|
+
classifyXaiSubscriptionStreamingTerminalError,
|
|
4039
|
+
classifyXaiSubscriptionStreamIdleTimeoutError,
|
|
4040
|
+
isXaiSubscriptionHostedToolContinuationError,
|
|
4041
|
+
isXaiSubscriptionRateLimitDiagnostic,
|
|
4042
|
+
isXaiSubscriptionTransportError,
|
|
4043
|
+
XaiSubscriptionReloginRequired
|
|
4044
|
+
} from "@opengeni/xai-subscription";
|
|
4045
|
+
|
|
4046
|
+
// src/activities/types.ts
|
|
4047
|
+
var PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
|
|
4048
|
+
var PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
|
|
4049
|
+
var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
|
|
4050
|
+
var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
|
|
4051
|
+
|
|
4052
|
+
// src/activities/agent-turn/errors.ts
|
|
4053
|
+
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
4054
|
+
var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
|
|
4055
|
+
var MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
4056
|
+
function providerRecoveryResult(input) {
|
|
4057
|
+
if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
|
|
4058
|
+
return {
|
|
4059
|
+
status: "exhausted",
|
|
4060
|
+
providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
4061
|
+
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES
|
|
4062
|
+
};
|
|
4063
|
+
}
|
|
4064
|
+
const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
|
|
4065
|
+
const continueDelayMs = input.failureCode === "provider_rate_limited" ? providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" ? Math.max(
|
|
4066
|
+
providerDelay ?? 0,
|
|
4067
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
|
|
4068
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
4069
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1
|
|
4070
|
+
)]
|
|
4071
|
+
) : PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
4072
|
+
return {
|
|
4073
|
+
status: "recovering",
|
|
4074
|
+
continueDelayMs
|
|
4075
|
+
};
|
|
4076
|
+
}
|
|
4077
|
+
function providerRecoveryExhaustedFailure(failure, recovery) {
|
|
4078
|
+
return {
|
|
4079
|
+
...failure,
|
|
4080
|
+
error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
|
|
4081
|
+
retryable: false,
|
|
4082
|
+
recoveryExhausted: true,
|
|
4083
|
+
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
4084
|
+
maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
|
|
4085
|
+
lastRetryableError: failure.error
|
|
4086
|
+
};
|
|
4087
|
+
}
|
|
4088
|
+
function providerRecoveryCountFromMetadata(metadata) {
|
|
4089
|
+
const value = metadata.providerRecoveryCount;
|
|
4090
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
4091
|
+
}
|
|
4092
|
+
function headerValue(headers, name) {
|
|
4093
|
+
if (!headers || typeof headers !== "object") return null;
|
|
4094
|
+
const getter = headers.get;
|
|
4095
|
+
if (typeof getter === "function") {
|
|
4096
|
+
const value = getter.call(headers, name);
|
|
4097
|
+
return typeof value === "string" ? value : null;
|
|
4098
|
+
}
|
|
4099
|
+
const target = name.toLowerCase();
|
|
4100
|
+
const entry = Object.entries(headers).find(
|
|
4101
|
+
([key, value]) => key.toLowerCase() === target && typeof value === "string"
|
|
4102
|
+
);
|
|
4103
|
+
return typeof entry?.[1] === "string" ? entry[1] : null;
|
|
4104
|
+
}
|
|
4105
|
+
function providerRetryAfterMs(error, nowMs = Date.now()) {
|
|
4106
|
+
let current = error;
|
|
4107
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
4108
|
+
const value = current;
|
|
4109
|
+
const body = value.error && typeof value.error === "object" ? value.error : null;
|
|
4110
|
+
const directSeconds = Number(
|
|
4111
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
|
|
4112
|
+
);
|
|
4113
|
+
const header = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
|
|
4114
|
+
const headerSeconds = header === null ? Number.NaN : Number(header);
|
|
4115
|
+
const headerDate = header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
|
|
4116
|
+
const seconds = Number.isFinite(directSeconds) ? directSeconds : Number.isFinite(headerSeconds) ? headerSeconds : Number.isFinite(headerDate) ? Math.max(0, (headerDate - nowMs) / 1e3) : Number.NaN;
|
|
4117
|
+
if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1e3);
|
|
4118
|
+
current = value.cause;
|
|
4119
|
+
}
|
|
4120
|
+
return null;
|
|
4121
|
+
}
|
|
4122
|
+
function unavailableMcpOperationalContext(input) {
|
|
4123
|
+
if (input.droppedCount <= 0) {
|
|
4124
|
+
return void 0;
|
|
4125
|
+
}
|
|
4126
|
+
const omittedCount = Math.max(0, input.droppedCount - input.droppedIds.length);
|
|
4127
|
+
const listed = input.droppedIds.map((id) => `"${id}"`).join(", ");
|
|
4128
|
+
const inventory = listed ? `${listed}${omittedCount > 0 ? `, plus ${omittedCount} additional unavailable server(s)` : ""}` : `${input.droppedCount} unavailable server(s)`;
|
|
4129
|
+
return `MCP capability availability for this turn: the following session-selected server(s) are disconnected or no longer registered and were skipped: ${inventory}. Do not claim to have read or updated those systems. If the task depends on one as a source of truth, explain the limitation and ask the user to reconnect it or select another authoritative source; continue with unaffected work only when safe.`;
|
|
4130
|
+
}
|
|
4131
|
+
function escapedMcpTimeoutRecoveryFailure(input) {
|
|
4132
|
+
if (input.failureCode !== "mcp_transport_timeout" || input.modelRequestStarted || !Number.isSafeInteger(input.detail.executionGeneration) || input.detail.executionGeneration <= 1 || !Number.isSafeInteger(input.detail.providerRecoveryCount) || input.detail.providerRecoveryCount <= 0 || input.detail.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || !Number.isSafeInteger(input.detail.continueDelayMs) || input.detail.continueDelayMs <= 0) {
|
|
4133
|
+
return null;
|
|
4134
|
+
}
|
|
4135
|
+
return ApplicationFailure.create({
|
|
4136
|
+
message: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
4137
|
+
type: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
4138
|
+
nonRetryable: true,
|
|
4139
|
+
details: [input.detail]
|
|
4140
|
+
});
|
|
4141
|
+
}
|
|
4142
|
+
function preClaimAdmissionFailure(error) {
|
|
4143
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
4144
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
4145
|
+
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
4146
|
+
const retryable = Boolean(
|
|
4147
|
+
rawTransportFailure || persistenceFailure && (sqlState === null || sqlState.startsWith("08") || sqlState.startsWith("40") || sqlState.startsWith("53") || sqlState === "55P03" || sqlState === "57014" || sqlState === "57P01" || sqlState === "57P02" || sqlState === "57P03" || sqlState.startsWith("58"))
|
|
4148
|
+
);
|
|
4149
|
+
const detail = {
|
|
4150
|
+
disposition: retryable ? "retryable" : "permanent",
|
|
4151
|
+
code: persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant")
|
|
4152
|
+
};
|
|
4153
|
+
return ApplicationFailure.create({
|
|
4154
|
+
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
4155
|
+
type: PRE_CLAIM_FAILURE_TYPE,
|
|
4156
|
+
nonRetryable: true,
|
|
4157
|
+
details: [detail]
|
|
4158
|
+
});
|
|
4159
|
+
}
|
|
4160
|
+
function isWorkerShutdownCancellation(error) {
|
|
4161
|
+
return error instanceof CancelledFailure2 && error.message === "WORKER_SHUTDOWN";
|
|
4162
|
+
}
|
|
4163
|
+
function sandboxRouteTransitionCode(error) {
|
|
4164
|
+
const pending = [error];
|
|
4165
|
+
const seen = /* @__PURE__ */ new WeakSet();
|
|
4166
|
+
let inspected = 0;
|
|
4167
|
+
while (pending.length > 0 && inspected < 64) {
|
|
4168
|
+
const current = pending.shift();
|
|
4169
|
+
inspected += 1;
|
|
4170
|
+
if (!current || typeof current !== "object" || seen.has(current)) continue;
|
|
4171
|
+
seen.add(current);
|
|
4172
|
+
try {
|
|
4173
|
+
const record = current;
|
|
4174
|
+
if ((current instanceof ActiveBackendUnresolvableError || record.name === "ActiveBackendUnresolvableError") && record.code === "home_unavailable_this_turn") {
|
|
4175
|
+
return "home_unavailable_this_turn";
|
|
4176
|
+
}
|
|
4177
|
+
if (current instanceof RoutingWorkspaceRootChangedError || current instanceof SelfhostedWorkspaceRootChangedError || record.name === "RoutingWorkspaceRootChangedError" || record.name === "SelfhostedWorkspaceRootChangedError") {
|
|
4178
|
+
return "workspace_root_changed_this_turn";
|
|
4179
|
+
}
|
|
4180
|
+
for (const key of ["cause", "error"]) {
|
|
4181
|
+
const nested = record[key];
|
|
4182
|
+
if (nested && typeof nested === "object") pending.push(nested);
|
|
4183
|
+
}
|
|
4184
|
+
if (Array.isArray(record.errors)) {
|
|
4185
|
+
pending.push(...record.errors.slice(0, 32));
|
|
4186
|
+
}
|
|
4187
|
+
} catch {
|
|
4188
|
+
}
|
|
4189
|
+
}
|
|
4190
|
+
return null;
|
|
4191
|
+
}
|
|
4192
|
+
function compactionFailureReason(reason) {
|
|
4193
|
+
return reason.startsWith("compaction summarization failed:") ? reason : `compaction summarization failed: ${reason}`;
|
|
4194
|
+
}
|
|
4195
|
+
function safeErrorDiagnostic(error) {
|
|
4196
|
+
const diagnostic = {
|
|
4197
|
+
errorClass: "WorkerOperationError",
|
|
4198
|
+
errorCode: "worker_operation_failed",
|
|
4199
|
+
origin: "worker"
|
|
4200
|
+
};
|
|
4201
|
+
try {
|
|
4202
|
+
let statusSource = error;
|
|
4203
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
4204
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
4205
|
+
statusSource = error.cause;
|
|
4206
|
+
}
|
|
4207
|
+
if (statusSource && typeof statusSource === "object") {
|
|
4208
|
+
const status = Number(
|
|
4209
|
+
statusSource.status ?? statusSource.statusCode
|
|
4210
|
+
);
|
|
4211
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
4212
|
+
diagnostic.status = status;
|
|
4213
|
+
}
|
|
4214
|
+
}
|
|
4215
|
+
} catch {
|
|
4216
|
+
}
|
|
4217
|
+
return diagnostic;
|
|
4218
|
+
}
|
|
4219
|
+
function safeErrorForTelemetry(error) {
|
|
4220
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
4221
|
+
const safe = new Error("worker operation failed");
|
|
4222
|
+
safe.name = "WorkerOperationError";
|
|
4223
|
+
safe.code = diagnostic.errorCode;
|
|
4224
|
+
if (diagnostic.status !== void 0) safe.status = diagnostic.status;
|
|
4225
|
+
safe.origin = diagnostic.origin;
|
|
4226
|
+
return safe;
|
|
4227
|
+
}
|
|
4228
|
+
function compactionFailureReasonFromError(error) {
|
|
4229
|
+
if (error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError) {
|
|
4230
|
+
return compactionFailureReason(error.message);
|
|
4231
|
+
}
|
|
4232
|
+
const errorName = error instanceof Error && error.name ? error.name : "unknown error";
|
|
4233
|
+
return compactionFailureReason(`unexpected ${errorName}`);
|
|
4234
|
+
}
|
|
4235
|
+
function isCompactionSummaryFailure(error) {
|
|
4236
|
+
return error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError;
|
|
4237
|
+
}
|
|
4238
|
+
function shouldRecoverCompactionProviderFailure(error) {
|
|
4239
|
+
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
4240
|
+
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
4241
|
+
return agentRunFailurePayload(error).retryable === true;
|
|
4242
|
+
}
|
|
4243
|
+
function classifyContextWindowOverflowError(error) {
|
|
4244
|
+
const fields = collectErrorStrings(error);
|
|
4245
|
+
const matched = fields.find(
|
|
4246
|
+
(value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value) || /exceeds?\s+(?:the\s+)?context\s+window/i.test(value) || /maximum\s+context\s+length/i.test(value) || /context\s+window[^.]*exceed/i.test(value)
|
|
4247
|
+
);
|
|
4248
|
+
if (!matched) {
|
|
4249
|
+
return null;
|
|
4250
|
+
}
|
|
4251
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4252
|
+
const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
|
|
4253
|
+
return {
|
|
4254
|
+
message,
|
|
4255
|
+
...code ? { code } : {},
|
|
4256
|
+
...matched && matched !== message ? { detail: matched } : {}
|
|
4257
|
+
};
|
|
4258
|
+
}
|
|
4259
|
+
function classifyMcpTransportTimeoutError(error) {
|
|
4260
|
+
const fields = collectErrorStrings(error);
|
|
4261
|
+
const matchedText = fields.find(
|
|
4262
|
+
(value) => /\bmcp\b/i.test(value) && /(?:request\s+timed\s+out|request\s+timeout|\btimed\s+out\b|\btimeout\b|ETIMEDOUT)/i.test(
|
|
4263
|
+
value
|
|
4264
|
+
) && !/authentication\s+required/i.test(value)
|
|
4265
|
+
);
|
|
4266
|
+
const sanitizedSdkTimeout = isMcpRequestTimeoutError(error);
|
|
4267
|
+
if (!matchedText && !sanitizedSdkTimeout) {
|
|
4268
|
+
return null;
|
|
4269
|
+
}
|
|
4270
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4271
|
+
const matched = matchedText ?? fields.find((value) => /\bmcp\b/i.test(value));
|
|
4272
|
+
return {
|
|
4273
|
+
message,
|
|
4274
|
+
...matched && matched !== message ? { detail: matched } : {}
|
|
4275
|
+
};
|
|
4276
|
+
}
|
|
4277
|
+
function collectErrorStrings(value, seen = /* @__PURE__ */ new WeakSet()) {
|
|
4278
|
+
if (typeof value === "string") {
|
|
4279
|
+
return [value];
|
|
4280
|
+
}
|
|
4281
|
+
if (!value || typeof value !== "object") {
|
|
4282
|
+
return [];
|
|
4283
|
+
}
|
|
4284
|
+
if (seen.has(value)) {
|
|
4285
|
+
return [];
|
|
4286
|
+
}
|
|
4287
|
+
seen.add(value);
|
|
4288
|
+
const out = [];
|
|
4289
|
+
const record = value;
|
|
4290
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
4291
|
+
const field = record[key];
|
|
4292
|
+
if (typeof field === "string" && field.length > 0) {
|
|
4293
|
+
out.push(field);
|
|
4294
|
+
}
|
|
4295
|
+
}
|
|
4296
|
+
for (const key of ["error", "cause", "response", "data"]) {
|
|
4297
|
+
out.push(...collectErrorStrings(record[key], seen));
|
|
4298
|
+
}
|
|
4299
|
+
return out;
|
|
4300
|
+
}
|
|
4301
|
+
var STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE = "unable to connect. is the computer able to access the url?";
|
|
4302
|
+
function isExactStatuslessUpstreamConnectivityMessage(message) {
|
|
4303
|
+
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
4304
|
+
}
|
|
4305
|
+
function isTransientProviderError(error) {
|
|
4306
|
+
const status = typeof error === "object" && error !== null ? Number(
|
|
4307
|
+
error.status ?? error.statusCode
|
|
4308
|
+
) : void 0;
|
|
4309
|
+
if (status !== void 0 && Number.isFinite(status)) {
|
|
4310
|
+
return status >= 500 && status < 600;
|
|
4311
|
+
}
|
|
4312
|
+
const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
4313
|
+
if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
|
|
4314
|
+
return true;
|
|
4315
|
+
}
|
|
4316
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4317
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
4318
|
+
return true;
|
|
4319
|
+
}
|
|
4320
|
+
return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
|
|
4321
|
+
message
|
|
4322
|
+
);
|
|
4323
|
+
}
|
|
4324
|
+
function classifyXaiCredentialFailure(error) {
|
|
4325
|
+
let relogin = error;
|
|
4326
|
+
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
4327
|
+
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
4328
|
+
return { kind: "auth", cooldownMs: null };
|
|
4329
|
+
}
|
|
4330
|
+
relogin = relogin.cause;
|
|
4331
|
+
}
|
|
4332
|
+
if (!isXaiSubscriptionTransportError(error)) return null;
|
|
4333
|
+
let current = error;
|
|
4334
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
4335
|
+
const value = current;
|
|
4336
|
+
const body = value.error && typeof value.error === "object" ? value.error : null;
|
|
4337
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
4338
|
+
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
4339
|
+
if (status === 401 || code === "unauthorized" || code === "invalid_token") {
|
|
4340
|
+
return { kind: "auth", cooldownMs: null };
|
|
4341
|
+
}
|
|
4342
|
+
if (status === 403) {
|
|
4343
|
+
return { kind: "forbidden", cooldownMs: null };
|
|
4344
|
+
}
|
|
4345
|
+
const message = String(value.message ?? body?.message ?? "");
|
|
4346
|
+
if (isXaiSubscriptionRateLimitDiagnostic({
|
|
4347
|
+
code,
|
|
4348
|
+
message,
|
|
4349
|
+
status: Number.isInteger(status) ? status : null
|
|
4350
|
+
})) {
|
|
4351
|
+
return {
|
|
4352
|
+
kind: "rate_limit",
|
|
4353
|
+
cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS
|
|
4354
|
+
};
|
|
4355
|
+
}
|
|
4356
|
+
current = value.cause;
|
|
4357
|
+
}
|
|
4358
|
+
return null;
|
|
4359
|
+
}
|
|
4360
|
+
function agentRunFailurePayload(error, options = {}) {
|
|
4361
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
4362
|
+
const underlying = isSessionEventPersistenceError(error.cause) ? agentRunFailurePayload(error.cause, options) : {
|
|
4363
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause)
|
|
4364
|
+
};
|
|
4365
|
+
return {
|
|
4366
|
+
...underlying,
|
|
4367
|
+
historyPersistenceStage: error.stage
|
|
4368
|
+
};
|
|
4369
|
+
}
|
|
4370
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4371
|
+
const status = typeof error === "object" && error !== null ? Number(
|
|
4372
|
+
error.status ?? error.statusCode
|
|
4373
|
+
) : void 0;
|
|
4374
|
+
const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
4375
|
+
if (error instanceof ActiveSessionHistoryLimitExceededError) {
|
|
4376
|
+
return {
|
|
4377
|
+
error: "The session's active conversation history exceeds the worker's safe materialization envelope. Clear the session context before retrying; an oversized history cannot be compacted safely in a serving worker.",
|
|
4378
|
+
code: error.code,
|
|
4379
|
+
retryable: false,
|
|
4380
|
+
detail: error.message
|
|
4381
|
+
};
|
|
4382
|
+
}
|
|
4383
|
+
if (error instanceof ApprovalRunStateLimitExceededError) {
|
|
4384
|
+
return {
|
|
4385
|
+
error: "The saved approval state exceeds the worker's safe materialization envelope. Clear the pending approval context before retrying.",
|
|
4386
|
+
code: error.code,
|
|
4387
|
+
retryable: false,
|
|
4388
|
+
detail: error.message
|
|
4389
|
+
};
|
|
4390
|
+
}
|
|
4391
|
+
if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
|
|
4392
|
+
return {
|
|
4393
|
+
error: "The Codex response stream ended without a terminal response",
|
|
4394
|
+
code: "invalid_sse_terminal",
|
|
4395
|
+
retryable: false
|
|
4396
|
+
};
|
|
4397
|
+
}
|
|
4398
|
+
if (isXaiSubscriptionHostedToolContinuationError(error)) {
|
|
4399
|
+
return {
|
|
4400
|
+
error: "SuperGrok stopped responding after its hosted search completed. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
|
|
4401
|
+
code: "xai_hosted_tool_continuation_stalled",
|
|
4402
|
+
retryable: false
|
|
4403
|
+
};
|
|
4404
|
+
}
|
|
4405
|
+
const xaiStreamTerminal = classifyXaiSubscriptionStreamingTerminalError(error);
|
|
4406
|
+
if (xaiStreamTerminal) {
|
|
4407
|
+
return {
|
|
4408
|
+
error: xaiStreamTerminal.message,
|
|
4409
|
+
code: xaiStreamTerminal.code,
|
|
4410
|
+
retryable: xaiStreamTerminal.status === 429,
|
|
4411
|
+
lastEventType: xaiStreamTerminal.eventType,
|
|
4412
|
+
...xaiStreamTerminal.requestId ? { requestId: xaiStreamTerminal.requestId } : {}
|
|
4413
|
+
};
|
|
4414
|
+
}
|
|
4415
|
+
const xaiStreamTimeout = classifyXaiSubscriptionStreamIdleTimeoutError(error);
|
|
4416
|
+
if (xaiStreamTimeout) {
|
|
4417
|
+
return {
|
|
4418
|
+
error: "SuperGrok stopped sending valid response events. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
|
|
4419
|
+
code: "xai_response_stream_idle_timeout",
|
|
4420
|
+
retryable: false,
|
|
4421
|
+
responseObserved: xaiStreamTimeout.responseObserved,
|
|
4422
|
+
eventCount: xaiStreamTimeout.eventCount,
|
|
4423
|
+
silenceDurationMs: xaiStreamTimeout.silenceDurationMs,
|
|
4424
|
+
...xaiStreamTimeout.requestId ? { requestId: xaiStreamTimeout.requestId } : {},
|
|
4425
|
+
...xaiStreamTimeout.lastEventType ? { lastEventType: xaiStreamTimeout.lastEventType } : {}
|
|
4426
|
+
};
|
|
4427
|
+
}
|
|
4428
|
+
if (isSessionEventPersistenceError(error)) {
|
|
4429
|
+
const { details } = error;
|
|
4430
|
+
return {
|
|
4431
|
+
error: error.message,
|
|
4432
|
+
code: details.code,
|
|
4433
|
+
detail: details.retryOutcome === "exhausted" ? `The idempotent persistence transaction failed after ${details.attempts} attempts.` : "The database rejected the idempotent persistence transaction.",
|
|
4434
|
+
correlationId: details.correlationId,
|
|
4435
|
+
stage: details.stage,
|
|
4436
|
+
sqlState: details.sqlState,
|
|
4437
|
+
attempts: details.attempts,
|
|
4438
|
+
retryOutcome: details.retryOutcome,
|
|
4439
|
+
...Object.keys(details.database).length > 0 ? { database: details.database } : {}
|
|
4440
|
+
};
|
|
4441
|
+
}
|
|
4442
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
4443
|
+
if (usageLimit) {
|
|
4444
|
+
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
4445
|
+
}
|
|
4446
|
+
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
4447
|
+
allowLegacyRequestTimeout: options.isCodexTurn === true
|
|
4448
|
+
});
|
|
4449
|
+
if (codexTimeout) {
|
|
4450
|
+
return {
|
|
4451
|
+
error: codexTimeout.responseObserved ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active." : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
|
|
4452
|
+
code: "codex_response_timeout",
|
|
4453
|
+
retryable: false,
|
|
4454
|
+
timeoutClass: codexTimeout.timeoutClass,
|
|
4455
|
+
responseObserved: codexTimeout.responseObserved,
|
|
4456
|
+
...codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {},
|
|
4457
|
+
...codexTimeout.message ? { detail: codexTimeout.message } : {}
|
|
4458
|
+
};
|
|
4459
|
+
}
|
|
4460
|
+
const mcpTimeout = classifyMcpTransportTimeoutError(error);
|
|
4461
|
+
if (mcpTimeout) {
|
|
4462
|
+
const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
|
|
4463
|
+
return {
|
|
4464
|
+
error: "An MCP server request timed out. Any completed tool output was checkpointed; the session can continue safely.",
|
|
4465
|
+
code: "mcp_transport_timeout",
|
|
4466
|
+
retryable: true,
|
|
4467
|
+
...mcpTimeout.detail || mcpTimeout.message ? { detail: mcpTimeout.detail ?? mcpTimeout.message } : {},
|
|
4468
|
+
...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
|
|
4469
|
+
};
|
|
4470
|
+
}
|
|
4471
|
+
if (isMcpTransportConnectivityError(error)) {
|
|
4472
|
+
const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
|
|
4473
|
+
return {
|
|
4474
|
+
error: "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
4475
|
+
code: "mcp_transport_unavailable",
|
|
4476
|
+
retryable: true,
|
|
4477
|
+
detail: message,
|
|
4478
|
+
...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
|
|
4479
|
+
};
|
|
4480
|
+
}
|
|
4481
|
+
if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
|
|
4482
|
+
return {
|
|
4483
|
+
error: "The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
|
|
4484
|
+
code: UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
4485
|
+
retryable: true
|
|
4486
|
+
};
|
|
4487
|
+
}
|
|
4488
|
+
if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
|
|
4489
|
+
return {
|
|
4490
|
+
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
4491
|
+
code: "provider_rate_limited",
|
|
4492
|
+
retryable: true,
|
|
4493
|
+
...message && message !== "Too Many Requests" ? { detail: message } : {}
|
|
4494
|
+
};
|
|
4495
|
+
}
|
|
4496
|
+
if (isTransientProviderError(error)) {
|
|
4497
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
4498
|
+
return {
|
|
4499
|
+
error: "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
4500
|
+
code: "upstream_connectivity_unavailable",
|
|
4501
|
+
retryable: true
|
|
4502
|
+
};
|
|
4503
|
+
}
|
|
4504
|
+
return { error: message, code: "provider_unavailable", retryable: true };
|
|
4505
|
+
}
|
|
4506
|
+
return { error: message };
|
|
4507
|
+
}
|
|
4508
|
+
var CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 6e4;
|
|
4509
|
+
function codexCredentialCooldownUntil(failure, account, now) {
|
|
4510
|
+
if (failure.kind === "auth" || failure.kind === "forbidden") {
|
|
4511
|
+
return null;
|
|
4512
|
+
}
|
|
4513
|
+
const providerReset = failure.cooldownSeconds !== null && Number.isFinite(failure.cooldownSeconds) && failure.cooldownSeconds > 0 ? new Date(now.getTime() + Math.ceil(failure.cooldownSeconds) * 1e3) : null;
|
|
4514
|
+
if (failure.kind === "rate_limit") {
|
|
4515
|
+
return providerReset ?? new Date(now.getTime() + PROVIDER_BACKPRESSURE_DELAY_MS);
|
|
4516
|
+
}
|
|
4517
|
+
const blockingResets = account ? [
|
|
4518
|
+
{ used: account.primaryUsedPercent, reset: account.primaryResetAt },
|
|
4519
|
+
{ used: account.secondaryUsedPercent, reset: account.secondaryResetAt }
|
|
4520
|
+
].filter(
|
|
4521
|
+
(window) => (window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && window.reset instanceof Date && window.reset.getTime() > now.getTime()
|
|
4522
|
+
).map((window) => window.reset) : [];
|
|
4523
|
+
const quotaResets = providerReset ? [...blockingResets, providerReset] : blockingResets;
|
|
4524
|
+
if (quotaResets.length === 0) {
|
|
4525
|
+
return new Date(now.getTime() + CODEX_ALLOWANCE_FALLBACK_MS);
|
|
4526
|
+
}
|
|
4527
|
+
return quotaResets.reduce(
|
|
4528
|
+
(latest, reset) => reset.getTime() > latest.getTime() ? reset : latest
|
|
4529
|
+
);
|
|
4530
|
+
}
|
|
4531
|
+
function classifyCodexCredentialFailure(error) {
|
|
4532
|
+
let refreshError = error;
|
|
4533
|
+
for (let depth = 0; depth < 6 && refreshError && typeof refreshError === "object"; depth += 1) {
|
|
4534
|
+
if (refreshError instanceof CodexReloginRequired) {
|
|
4535
|
+
return { kind: "auth", cooldownSeconds: null };
|
|
4536
|
+
}
|
|
4537
|
+
refreshError = refreshError.cause;
|
|
4538
|
+
}
|
|
4539
|
+
if (!isCodexTransportError(error)) {
|
|
4540
|
+
return null;
|
|
4541
|
+
}
|
|
4542
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
4543
|
+
if (usageLimit) {
|
|
4544
|
+
return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
|
|
4545
|
+
}
|
|
4546
|
+
let cur = error;
|
|
4547
|
+
for (let depth = 0; depth < 6 && cur && typeof cur === "object"; depth++) {
|
|
4548
|
+
const value = cur;
|
|
4549
|
+
const body = value.error && typeof value.error === "object" ? value.error : null;
|
|
4550
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
4551
|
+
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
4552
|
+
const directRetryAfter = Number(
|
|
4553
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds
|
|
4554
|
+
);
|
|
4555
|
+
const retryAfterHeader = headerValue(value.headers, "retry-after") ?? headerValue(value.responseHeaders, "retry-after") ?? headerValue(body?.headers, "retry-after");
|
|
4556
|
+
const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
|
|
4557
|
+
const retryAfterDate = retryAfterHeader !== null && !Number.isFinite(retryAfterNumber) ? Date.parse(retryAfterHeader) : Number.NaN;
|
|
4558
|
+
const retryAfter = Number.isFinite(directRetryAfter) ? directRetryAfter : Number.isFinite(retryAfterNumber) ? retryAfterNumber : Number.isFinite(retryAfterDate) ? Math.max(0, (retryAfterDate - Date.now()) / 1e3) : Number.NaN;
|
|
4559
|
+
const cooldownSeconds = Number.isFinite(retryAfter) && retryAfter > 0 ? Math.ceil(retryAfter) : null;
|
|
4560
|
+
if (code === "insufficient_quota" || code === "quota_exceeded" || code === "billing_hard_limit_reached") {
|
|
4561
|
+
return { kind: "quota", cooldownSeconds };
|
|
4562
|
+
}
|
|
4563
|
+
if (status === 401 || code === "unauthorized" || code === "invalid_api_key") {
|
|
4564
|
+
return { kind: "auth", cooldownSeconds };
|
|
4565
|
+
}
|
|
4566
|
+
if (status === 403) {
|
|
4567
|
+
return { kind: "forbidden", cooldownSeconds };
|
|
4568
|
+
}
|
|
4569
|
+
if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
|
|
4570
|
+
return { kind: "rate_limit", cooldownSeconds };
|
|
4571
|
+
}
|
|
4572
|
+
cur = value.cause;
|
|
4573
|
+
}
|
|
4574
|
+
return null;
|
|
4575
|
+
}
|
|
4576
|
+
function humanizeResetWindow(resetsInSeconds) {
|
|
4577
|
+
if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
|
|
4578
|
+
return "shortly";
|
|
4579
|
+
}
|
|
4580
|
+
const total = Math.ceil(resetsInSeconds);
|
|
4581
|
+
if (total < 60) {
|
|
4582
|
+
return "in under a minute";
|
|
4583
|
+
}
|
|
4584
|
+
const hours = Math.floor(total / 3600);
|
|
4585
|
+
const minutes = Math.round(total % 3600 / 60);
|
|
4586
|
+
if (hours > 0) {
|
|
4587
|
+
return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
|
|
4588
|
+
}
|
|
4589
|
+
return `in about ${minutes}m`;
|
|
4590
|
+
}
|
|
4591
|
+
function codexUsageLimitFailurePayload(info, detail, opts) {
|
|
4592
|
+
const error = opts?.allAccounts ? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for a subscription to reset.` : `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
|
|
4593
|
+
return {
|
|
4594
|
+
error,
|
|
4595
|
+
code: "codex_usage_limit_reached",
|
|
4596
|
+
retryable: false,
|
|
4597
|
+
...detail ? { detail } : {}
|
|
4598
|
+
};
|
|
4599
|
+
}
|
|
4600
|
+
var CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 6e4;
|
|
4601
|
+
|
|
4602
|
+
export {
|
|
4603
|
+
createSharedActivityServices,
|
|
4604
|
+
classifyCodexPin,
|
|
4605
|
+
CODEX_USAGE_EXHAUSTED_PCT,
|
|
4606
|
+
isCodexCredentialEligible,
|
|
4607
|
+
shardCredentialForSession,
|
|
4608
|
+
earliestCodexReset,
|
|
4609
|
+
authoritativeCodexCapacityResetAt,
|
|
4610
|
+
computeIdleDelayMs,
|
|
4611
|
+
computeReactiveRotationResume,
|
|
4612
|
+
chooseRotationActive,
|
|
4613
|
+
selectCodexCredentialLeaseForTurn,
|
|
4614
|
+
refreshCodexUsageAndRepairCapacityWaiters,
|
|
4615
|
+
signalCodexCapacityWakeTargets,
|
|
4616
|
+
signalPendingCodexCapacityWakeTargets,
|
|
4617
|
+
createCodexCapacityActivities,
|
|
4618
|
+
createGoalActivities,
|
|
4619
|
+
withFirstPartyTools,
|
|
4620
|
+
rigProviderImageSourceImage,
|
|
4621
|
+
settingsWithPackSandboxImage,
|
|
4622
|
+
settingsWithRigImage,
|
|
4623
|
+
resolveWorkspacePackRuntime,
|
|
4624
|
+
resolveWorkspaceInstalledSkillRuntime,
|
|
4625
|
+
createRuntimeBatcher,
|
|
4626
|
+
currentActivityContext,
|
|
4627
|
+
startActivityHeartbeat,
|
|
4628
|
+
nextStreamEvent,
|
|
4629
|
+
notifyParentOfChildIdle,
|
|
4630
|
+
deliverFailedChildTurnToParent,
|
|
4631
|
+
deliverChildRequiresActionToParent,
|
|
4632
|
+
reconcilePendingParentSystemUpdates,
|
|
4633
|
+
reconcilePendingSessionWorkflowWakes,
|
|
4634
|
+
collectWorkspaceArchiveObjectKeys,
|
|
4635
|
+
putVersion1TarArchiveOrInline,
|
|
4636
|
+
deleteUnpublishedWorkspaceArchiveObject,
|
|
4637
|
+
deleteWorkspaceArchiveObjectKeys,
|
|
4638
|
+
assertReusableSessionRevivable,
|
|
4639
|
+
scheduledUserMessagePayload,
|
|
4640
|
+
workflowIdForSession,
|
|
4641
|
+
mergeResourceRefs,
|
|
4642
|
+
INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
|
|
4643
|
+
incidentTelemetryPreflightDeclaration,
|
|
4644
|
+
evaluateIncidentTelemetryPreflight,
|
|
4645
|
+
incidentTelemetryAuthorityFence,
|
|
4646
|
+
resolveIncidentTelemetryResponderMetadata,
|
|
4647
|
+
validateIncidentTelemetrySystemUpdateAuthority,
|
|
4648
|
+
sandboxLeaseHolderIdForAttempt,
|
|
4649
|
+
waitForWarmSnapshot,
|
|
4650
|
+
persistSandboxDeadlineRotationCheckpoint,
|
|
4651
|
+
maybePersistWarmWorkspaceSnapshot,
|
|
4652
|
+
resumeBoxForTurn,
|
|
4653
|
+
SandboxDeadlineRotationError,
|
|
4654
|
+
turnOperationCancellationFailure,
|
|
4655
|
+
throwIfTurnOperationCancelled,
|
|
4656
|
+
waitForTurnOperation,
|
|
4657
|
+
classifySandboxLogicalProvisionFailure,
|
|
4658
|
+
isLazySandboxProvisionRetryable,
|
|
4659
|
+
sandboxDeadlineRotationRecoveryDelayMs,
|
|
4660
|
+
createTurnSandboxProvisioner,
|
|
4661
|
+
shouldRunTurnEndWorkspacePersistence,
|
|
4662
|
+
shouldStartPeriodicWorkspaceSnapshot,
|
|
4663
|
+
assertPhysicalToolQuiescenceForCancellation,
|
|
4664
|
+
assertSessionAttemptQuiescenceRecoveryDurable,
|
|
4665
|
+
persistOrSignalSessionAttemptQuiescence,
|
|
4666
|
+
drainAttemptOwnedSandboxWriters,
|
|
4667
|
+
releaseTurnSandboxAfterWriterDrain,
|
|
4668
|
+
waitForTurnFinalizerStep,
|
|
4669
|
+
waitForTurnStreamCleanup,
|
|
4670
|
+
clearAttemptCredentialsWithSettledFence,
|
|
4671
|
+
turnFinalizerCancellationSignal,
|
|
4672
|
+
runMandatoryHistoryPersistenceStep,
|
|
4673
|
+
finalizeDurableTurnOpStreams,
|
|
4674
|
+
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
4675
|
+
providerRecoveryResult,
|
|
4676
|
+
providerRecoveryExhaustedFailure,
|
|
4677
|
+
providerRecoveryCountFromMetadata,
|
|
4678
|
+
providerRetryAfterMs,
|
|
4679
|
+
unavailableMcpOperationalContext,
|
|
4680
|
+
escapedMcpTimeoutRecoveryFailure,
|
|
4681
|
+
preClaimAdmissionFailure,
|
|
4682
|
+
isWorkerShutdownCancellation,
|
|
4683
|
+
sandboxRouteTransitionCode,
|
|
4684
|
+
compactionFailureReason,
|
|
4685
|
+
safeErrorDiagnostic,
|
|
4686
|
+
safeErrorForTelemetry,
|
|
4687
|
+
compactionFailureReasonFromError,
|
|
4688
|
+
isCompactionSummaryFailure,
|
|
4689
|
+
shouldRecoverCompactionProviderFailure,
|
|
4690
|
+
classifyContextWindowOverflowError,
|
|
4691
|
+
classifyXaiCredentialFailure,
|
|
4692
|
+
agentRunFailurePayload,
|
|
4693
|
+
codexCredentialCooldownUntil,
|
|
4694
|
+
classifyCodexCredentialFailure,
|
|
4695
|
+
codexUsageLimitFailurePayload,
|
|
4696
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
4697
|
+
};
|
|
4698
|
+
//# sourceMappingURL=chunk-CNYGVRKF.js.map
|