@opengeni/worker-bundle 0.16.29 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +85 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +662 -17374
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +2 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1043 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1470 -246
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +6 -6
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +45 -42
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +77 -4
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +48 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +199 -4
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +19 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +3 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,3091 @@
|
|
|
1
|
+
import {
|
|
2
|
+
observabilityEventLogger
|
|
3
|
+
} from "./chunk-E3RI3I33.js";
|
|
4
|
+
|
|
5
|
+
// src/activity-services.ts
|
|
6
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
7
|
+
import { createDb } from "@opengeni/db";
|
|
8
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
9
|
+
import { createObservability } from "@opengeni/observability";
|
|
10
|
+
import { createObjectStorage } from "@opengeni/storage";
|
|
11
|
+
function createSharedActivityServices(dependencies) {
|
|
12
|
+
let servicesPromise = null;
|
|
13
|
+
return async function services() {
|
|
14
|
+
servicesPromise ??= (async () => {
|
|
15
|
+
const settings = dependencies.settings ?? getSettings();
|
|
16
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
17
|
+
const searchPath = dbSearchPath(settings);
|
|
18
|
+
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
19
|
+
...searchPath ? { searchPath } : {},
|
|
20
|
+
rlsStrategy: settings.rlsStrategy
|
|
21
|
+
});
|
|
22
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
23
|
+
return {
|
|
24
|
+
settings,
|
|
25
|
+
db: dependencies.db ?? dbClient.db,
|
|
26
|
+
bus: dependencies.bus ?? await createNatsEventBus(
|
|
27
|
+
settings.natsUrl,
|
|
28
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
29
|
+
{ logger: observabilityEventLogger(observability) }
|
|
30
|
+
),
|
|
31
|
+
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
32
|
+
observability,
|
|
33
|
+
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
34
|
+
signalSessionAttemptQuiesced: dependencies.signalSessionAttemptQuiesced ?? null,
|
|
35
|
+
inspectSessionAttemptActivity: dependencies.inspectSessionAttemptActivity ?? null,
|
|
36
|
+
signalCodexCapacityWorkflow: dependencies.signalCodexCapacityWorkflow ?? null,
|
|
37
|
+
startSandboxReaperWorkflow: dependencies.startSandboxReaperWorkflow ?? null,
|
|
38
|
+
startVideoGenerationWorkflow: dependencies.startVideoGenerationWorkflow ?? null,
|
|
39
|
+
entitlements: dependencies.entitlements ?? null,
|
|
40
|
+
connectionCredentials: dependencies.connectionCredentials ?? null
|
|
41
|
+
};
|
|
42
|
+
})();
|
|
43
|
+
return await servicesPromise;
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// src/activities/codex-capacity.ts
|
|
48
|
+
import {
|
|
49
|
+
fetchCodexUsageForAccount,
|
|
50
|
+
getCodexCapacityWaitForSession,
|
|
51
|
+
listCodexAccountStatuses,
|
|
52
|
+
listPendingCodexCapacityWakeTargets,
|
|
53
|
+
reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb
|
|
54
|
+
} from "@opengeni/db";
|
|
55
|
+
|
|
56
|
+
// src/activities/codex-rotation.ts
|
|
57
|
+
function effectiveRotationStrategy(_stored) {
|
|
58
|
+
return "sharded";
|
|
59
|
+
}
|
|
60
|
+
function classifyCodexPin(args) {
|
|
61
|
+
const { pinnedCredentialId, pinSource, strategy, rotationEnabled } = args;
|
|
62
|
+
const pinned = pinnedCredentialId != null;
|
|
63
|
+
if (pinned && pinSource !== "policy") {
|
|
64
|
+
return "manual";
|
|
65
|
+
}
|
|
66
|
+
const shardedActive = rotationEnabled && effectiveRotationStrategy(strategy) === "sharded";
|
|
67
|
+
if (shardedActive) {
|
|
68
|
+
return "sharded";
|
|
69
|
+
}
|
|
70
|
+
if (pinned && pinSource === "policy") {
|
|
71
|
+
return "clearStale";
|
|
72
|
+
}
|
|
73
|
+
return "unpinned";
|
|
74
|
+
}
|
|
75
|
+
var MIN_IDLE_MS = 6e4;
|
|
76
|
+
var DEFAULT_RESET_COOLDOWN_MS = 6e4;
|
|
77
|
+
var CODEX_USAGE_EXHAUSTED_PCT = 100;
|
|
78
|
+
function effectiveWindowUsed(usedPercent, resetAt, now) {
|
|
79
|
+
return resetAt && resetAt.getTime() <= now.getTime() ? 0 : usedPercent ?? 0;
|
|
80
|
+
}
|
|
81
|
+
function bindingUsedPct(acct, now) {
|
|
82
|
+
return Math.max(
|
|
83
|
+
effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now),
|
|
84
|
+
effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now)
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
function bindingRemaining(acct, now) {
|
|
88
|
+
const primaryRemaining = 100 - effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now);
|
|
89
|
+
const secondaryRemaining = 100 - effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now);
|
|
90
|
+
return Math.min(primaryRemaining, secondaryRemaining);
|
|
91
|
+
}
|
|
92
|
+
function cooling(acct, now) {
|
|
93
|
+
return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
|
|
94
|
+
}
|
|
95
|
+
function isCodexCredentialHealthy(acct, now) {
|
|
96
|
+
return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct, now) < CODEX_USAGE_EXHAUSTED_PCT;
|
|
97
|
+
}
|
|
98
|
+
function isCodexCredentialEligible(acct, now) {
|
|
99
|
+
return acct.allocatorEnabled && isCodexCredentialHealthy(acct, now);
|
|
100
|
+
}
|
|
101
|
+
function fnv1a32(input) {
|
|
102
|
+
let hash = 2166136261;
|
|
103
|
+
for (let i = 0; i < input.length; i++) {
|
|
104
|
+
hash ^= input.charCodeAt(i);
|
|
105
|
+
hash = Math.imul(hash, 16777619);
|
|
106
|
+
}
|
|
107
|
+
return hash >>> 0;
|
|
108
|
+
}
|
|
109
|
+
function shardCredentialForSession(args) {
|
|
110
|
+
const { sessionId, accounts, now } = args;
|
|
111
|
+
const eligibles = accounts.filter((acct) => isCodexCredentialEligible(acct, now));
|
|
112
|
+
if (eligibles.length === 0) {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
const index = fnv1a32(sessionId) % eligibles.length;
|
|
116
|
+
return eligibles[index].id;
|
|
117
|
+
}
|
|
118
|
+
function earliestCodexReset(accounts, now) {
|
|
119
|
+
return earliestReset(accounts, now);
|
|
120
|
+
}
|
|
121
|
+
function chooseShardedHome(args) {
|
|
122
|
+
const { sessionId, currentPolicyPin, accounts, now } = args;
|
|
123
|
+
const pinRow = currentPolicyPin ? accounts.find((acct) => acct.id === currentPolicyPin) ?? null : null;
|
|
124
|
+
if (pinRow && isCodexCredentialEligible(pinRow, now)) {
|
|
125
|
+
return { kind: "home", credentialId: currentPolicyPin, rewritePin: false };
|
|
126
|
+
}
|
|
127
|
+
const home = shardCredentialForSession({ sessionId, accounts, now });
|
|
128
|
+
if (home == null) {
|
|
129
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, now) };
|
|
130
|
+
}
|
|
131
|
+
return { kind: "home", credentialId: home, rewritePin: true };
|
|
132
|
+
}
|
|
133
|
+
function availableAt(acct, now) {
|
|
134
|
+
const nowMs = now.getTime();
|
|
135
|
+
const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
|
|
136
|
+
const candidates = [];
|
|
137
|
+
if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
|
|
138
|
+
candidates.push(acct.exhaustedUntil);
|
|
139
|
+
}
|
|
140
|
+
const windowClear = (over, resetAt) => {
|
|
141
|
+
if (!over) return;
|
|
142
|
+
candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
|
|
143
|
+
};
|
|
144
|
+
windowClear(
|
|
145
|
+
effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
|
|
146
|
+
acct.primaryResetAt
|
|
147
|
+
);
|
|
148
|
+
windowClear(
|
|
149
|
+
effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now) >= CODEX_USAGE_EXHAUSTED_PCT,
|
|
150
|
+
acct.secondaryResetAt
|
|
151
|
+
);
|
|
152
|
+
if (candidates.length === 0) {
|
|
153
|
+
return defaultClear;
|
|
154
|
+
}
|
|
155
|
+
return candidates.reduce((a, b) => b.getTime() > a.getTime() ? b : a);
|
|
156
|
+
}
|
|
157
|
+
function earliestReset(accounts, now) {
|
|
158
|
+
return accounts.map((acct) => availableAt(acct, now)).reduce((a, b) => b.getTime() < a.getTime() ? b : a);
|
|
159
|
+
}
|
|
160
|
+
function authoritativeCodexCapacityResetAt(accounts, now) {
|
|
161
|
+
const future = [];
|
|
162
|
+
for (const account of accounts) {
|
|
163
|
+
if (!account.allocatorEnabled || account.status !== "active") continue;
|
|
164
|
+
if (account.exhaustedUntil && account.exhaustedUntil.getTime() > now.getTime()) {
|
|
165
|
+
future.push(account.exhaustedUntil);
|
|
166
|
+
}
|
|
167
|
+
if ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.primaryResetAt && account.primaryResetAt.getTime() > now.getTime()) {
|
|
168
|
+
future.push(account.primaryResetAt);
|
|
169
|
+
}
|
|
170
|
+
if ((account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT && account.secondaryResetAt && account.secondaryResetAt.getTime() > now.getTime()) {
|
|
171
|
+
future.push(account.secondaryResetAt);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
if (future.length === 0) return null;
|
|
175
|
+
return future.reduce(
|
|
176
|
+
(earliest, candidate) => candidate.getTime() < earliest.getTime() ? candidate : earliest
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
function computeIdleDelayMs(earliestResetAt, now, maxMs) {
|
|
180
|
+
const delta = earliestResetAt.getTime() - now.getTime();
|
|
181
|
+
return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
|
|
182
|
+
}
|
|
183
|
+
var REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5e3;
|
|
184
|
+
var REACTIVE_ROTATION_MARGIN = 2;
|
|
185
|
+
var REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS;
|
|
186
|
+
function computeReactiveRotationResume(args) {
|
|
187
|
+
const streak = args.priorConsecutiveRotations + 1;
|
|
188
|
+
const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
|
|
189
|
+
if (streak > bound) {
|
|
190
|
+
return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
|
|
191
|
+
}
|
|
192
|
+
if (!args.cooldownPersisted) {
|
|
193
|
+
return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
|
|
194
|
+
}
|
|
195
|
+
return { continueDelayMs: 0, idleUntilReset: false };
|
|
196
|
+
}
|
|
197
|
+
function chooseRotationActive(args) {
|
|
198
|
+
const { activeCredentialId, priorCredentialId, accounts, now } = args;
|
|
199
|
+
const rotationStrategy = effectiveRotationStrategy(args.rotationStrategy);
|
|
200
|
+
const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
|
|
201
|
+
if (allocatableAccounts.length === 0) {
|
|
202
|
+
return { kind: "none" };
|
|
203
|
+
}
|
|
204
|
+
const eligibles = allocatableAccounts.filter((acct) => isCodexCredentialEligible(acct, now));
|
|
205
|
+
const decide = (chosen2) => {
|
|
206
|
+
if (!chosen2) {
|
|
207
|
+
return {
|
|
208
|
+
kind: "allCapped",
|
|
209
|
+
earliestResetAt: earliestReset(allocatableAccounts, now)
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
kind: "active",
|
|
214
|
+
credentialId: chosen2.id,
|
|
215
|
+
moved: chosen2.id !== activeCredentialId
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
if (rotationStrategy === "round_robin") {
|
|
219
|
+
if (eligibles.length === 0) {
|
|
220
|
+
return {
|
|
221
|
+
kind: "allCapped",
|
|
222
|
+
earliestResetAt: earliestReset(allocatableAccounts, now)
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
const priorIdx = priorCredentialId ? allocatableAccounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
|
|
226
|
+
const ordered = priorIdx >= 0 ? [
|
|
227
|
+
...allocatableAccounts.slice(priorIdx + 1),
|
|
228
|
+
...allocatableAccounts.slice(0, priorIdx + 1)
|
|
229
|
+
] : allocatableAccounts;
|
|
230
|
+
const chosen2 = ordered.find((acct) => isCodexCredentialEligible(acct, now));
|
|
231
|
+
return decide(chosen2);
|
|
232
|
+
}
|
|
233
|
+
if (rotationStrategy === "drain_then_next") {
|
|
234
|
+
const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : void 0;
|
|
235
|
+
if (priorRow && isCodexCredentialEligible(priorRow, now)) {
|
|
236
|
+
return decide(priorRow);
|
|
237
|
+
}
|
|
238
|
+
return decide(eligibles[0]);
|
|
239
|
+
}
|
|
240
|
+
const activeLeases = (acct) => "activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
|
|
241
|
+
const selections = (acct) => "selectionCount" in acct ? acct.selectionCount : 0;
|
|
242
|
+
const lastSelected = (acct) => "lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
|
|
243
|
+
const chosen = eligibles.reduce((best, acct) => {
|
|
244
|
+
if (!best) {
|
|
245
|
+
return acct;
|
|
246
|
+
}
|
|
247
|
+
if (activeLeases(acct) !== activeLeases(best)) {
|
|
248
|
+
return activeLeases(acct) < activeLeases(best) ? acct : best;
|
|
249
|
+
}
|
|
250
|
+
if (bindingRemaining(acct, now) !== bindingRemaining(best, now)) {
|
|
251
|
+
return bindingRemaining(acct, now) > bindingRemaining(best, now) ? acct : best;
|
|
252
|
+
}
|
|
253
|
+
if (selections(acct) !== selections(best)) {
|
|
254
|
+
return selections(acct) < selections(best) ? acct : best;
|
|
255
|
+
}
|
|
256
|
+
return lastSelected(acct) < lastSelected(best) ? acct : best;
|
|
257
|
+
}, void 0);
|
|
258
|
+
return decide(chosen);
|
|
259
|
+
}
|
|
260
|
+
function chooseLegacyRotationActive(args) {
|
|
261
|
+
if (args.rotationStrategy === "most_remaining") {
|
|
262
|
+
const active = args.activeCredentialId ? args.accounts.find((account) => account.id === args.activeCredentialId) : void 0;
|
|
263
|
+
if (active && isCodexCredentialEligible(active, args.now)) {
|
|
264
|
+
return { kind: "active", credentialId: active.id, moved: false };
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return chooseRotationActive({
|
|
268
|
+
...args,
|
|
269
|
+
accounts: args.accounts.map((account) => ({
|
|
270
|
+
...account,
|
|
271
|
+
activeLeaseCount: 0,
|
|
272
|
+
selectionCount: 0,
|
|
273
|
+
lastSelectedAt: null
|
|
274
|
+
}))
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
function selectCodexCredentialLeaseForTurn(args) {
|
|
278
|
+
const {
|
|
279
|
+
accounts,
|
|
280
|
+
activeCredentialId,
|
|
281
|
+
rotationEnabled,
|
|
282
|
+
leaseRotationEnabled,
|
|
283
|
+
rotationStrategy,
|
|
284
|
+
existingCredentialId
|
|
285
|
+
} = args.context;
|
|
286
|
+
const leasingEnabled = args.leasingEnabled && leaseRotationEnabled;
|
|
287
|
+
const strategy = effectiveRotationStrategy(rotationStrategy);
|
|
288
|
+
const existing = existingCredentialId ? accounts.find((account) => account.id === existingCredentialId) : void 0;
|
|
289
|
+
if (existing && isCodexCredentialHealthy(existing, args.now)) {
|
|
290
|
+
return {
|
|
291
|
+
credentialId: existing.id,
|
|
292
|
+
advanceActivePointer: false,
|
|
293
|
+
decision: {
|
|
294
|
+
kind: "active",
|
|
295
|
+
credentialId: existing.id,
|
|
296
|
+
moved: existing.id !== activeCredentialId
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
const connectedIds = new Set(
|
|
301
|
+
accounts.filter((account) => account.allocatorEnabled).map((account) => account.id)
|
|
302
|
+
);
|
|
303
|
+
const pinDisposition = classifyCodexPin({
|
|
304
|
+
pinnedCredentialId: args.sessionPinnedCredentialId,
|
|
305
|
+
pinSource: args.sessionPinSource,
|
|
306
|
+
strategy,
|
|
307
|
+
rotationEnabled
|
|
308
|
+
});
|
|
309
|
+
if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
|
|
310
|
+
const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
|
|
311
|
+
if (!pinned?.allocatorEnabled) {
|
|
312
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
313
|
+
}
|
|
314
|
+
if (isCodexCredentialHealthy(pinned, args.now)) {
|
|
315
|
+
return {
|
|
316
|
+
credentialId: pinned.id,
|
|
317
|
+
decision: { kind: "active", credentialId: pinned.id, moved: false },
|
|
318
|
+
advanceActivePointer: false
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
return {
|
|
322
|
+
credentialId: null,
|
|
323
|
+
decision: {
|
|
324
|
+
kind: "allCapped",
|
|
325
|
+
earliestResetAt: availableAt(pinned, args.now)
|
|
326
|
+
},
|
|
327
|
+
advanceActivePointer: false
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
if (pinDisposition === "sharded") {
|
|
331
|
+
if (accounts.length === 0) {
|
|
332
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
333
|
+
}
|
|
334
|
+
const shard = chooseShardedHome({
|
|
335
|
+
sessionId: args.sessionId,
|
|
336
|
+
currentPolicyPin: args.sessionPinSource === "policy" ? args.sessionPinnedCredentialId : null,
|
|
337
|
+
accounts,
|
|
338
|
+
now: args.now
|
|
339
|
+
});
|
|
340
|
+
if (shard.kind === "allCapped") {
|
|
341
|
+
return {
|
|
342
|
+
credentialId: null,
|
|
343
|
+
decision: shard,
|
|
344
|
+
advanceActivePointer: false
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
return {
|
|
348
|
+
credentialId: shard.credentialId,
|
|
349
|
+
decision: {
|
|
350
|
+
kind: "active",
|
|
351
|
+
credentialId: shard.credentialId,
|
|
352
|
+
moved: shard.credentialId !== activeCredentialId
|
|
353
|
+
},
|
|
354
|
+
advanceActivePointer: false
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
if (!rotationEnabled) {
|
|
358
|
+
const active = activeCredentialId ? accounts.find((account) => account.id === activeCredentialId) : void 0;
|
|
359
|
+
if (!active?.allocatorEnabled) {
|
|
360
|
+
return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
|
|
361
|
+
}
|
|
362
|
+
if (!isCodexCredentialHealthy(active, args.now)) {
|
|
363
|
+
return {
|
|
364
|
+
credentialId: null,
|
|
365
|
+
decision: { kind: "allCapped", earliestResetAt: availableAt(active, args.now) },
|
|
366
|
+
advanceActivePointer: false
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
return {
|
|
370
|
+
credentialId: active.id,
|
|
371
|
+
decision: { kind: "active", credentialId: active.id, moved: false },
|
|
372
|
+
advanceActivePointer: false
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
|
|
376
|
+
const credentialId = connectedIds.has(args.sessionPinnedCredentialId) ? args.sessionPinnedCredentialId : activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
|
|
377
|
+
return {
|
|
378
|
+
credentialId,
|
|
379
|
+
decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" }
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
const priorId = args.sessionLastCredentialId ?? activeCredentialId;
|
|
383
|
+
const choose = leasingEnabled ? chooseRotationActive : chooseLegacyRotationActive;
|
|
384
|
+
const decision = choose({
|
|
385
|
+
rotationStrategy: strategy,
|
|
386
|
+
activeCredentialId,
|
|
387
|
+
// Per-session continuity is the round-robin/drain cursor. Falling back to
|
|
388
|
+
// the workspace pointer is only correct when this session has never run.
|
|
389
|
+
priorCredentialId: priorId,
|
|
390
|
+
accounts,
|
|
391
|
+
now: args.now
|
|
392
|
+
});
|
|
393
|
+
return {
|
|
394
|
+
credentialId: decision.kind === "active" ? decision.credentialId : null,
|
|
395
|
+
decision
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// src/activities/codex-capacity.ts
|
|
400
|
+
async function refreshCodexUsageAndRepairCapacityWaiters(refreshes, repairPendingWakes) {
|
|
401
|
+
await Promise.all(refreshes.map((refresh) => refresh().catch(() => void 0)));
|
|
402
|
+
await repairPendingWakes();
|
|
403
|
+
}
|
|
404
|
+
async function signalCodexCapacityWakeTargets(services, targets) {
|
|
405
|
+
await Promise.allSettled(
|
|
406
|
+
targets.map(
|
|
407
|
+
(target) => services.signalCodexCapacityWorkflow ? services.signalCodexCapacityWorkflow({
|
|
408
|
+
accountId: target.accountId,
|
|
409
|
+
workspaceId: target.workspaceId,
|
|
410
|
+
sessionId: target.sessionId,
|
|
411
|
+
workflowId: target.workflowId,
|
|
412
|
+
wakeRevision: target.wakeRevision
|
|
413
|
+
}) : services.wakeSessionWorkflow ? services.wakeSessionWorkflow({
|
|
414
|
+
accountId: target.accountId,
|
|
415
|
+
workspaceId: target.workspaceId,
|
|
416
|
+
sessionId: target.sessionId,
|
|
417
|
+
workflowId: target.workflowId,
|
|
418
|
+
wakeRevision: target.workflowWakeRevision
|
|
419
|
+
}) : Promise.resolve()
|
|
420
|
+
)
|
|
421
|
+
);
|
|
422
|
+
}
|
|
423
|
+
async function signalPendingCodexCapacityWakeTargets(services, workspaceId) {
|
|
424
|
+
const targets = await listPendingCodexCapacityWakeTargets(services.db, workspaceId).catch(
|
|
425
|
+
() => []
|
|
426
|
+
);
|
|
427
|
+
await signalCodexCapacityWakeTargets(services, targets);
|
|
428
|
+
}
|
|
429
|
+
function codexCapacityDecision(context, settings) {
|
|
430
|
+
const now = /* @__PURE__ */ new Date();
|
|
431
|
+
const selected = selectCodexCredentialLeaseForTurn({
|
|
432
|
+
context,
|
|
433
|
+
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
434
|
+
sessionId: context.sessionId,
|
|
435
|
+
sessionPinnedCredentialId: context.sessionPinnedCredentialId,
|
|
436
|
+
sessionPinSource: context.sessionPinSource,
|
|
437
|
+
sessionLastCredentialId: context.sessionLastCredentialId,
|
|
438
|
+
now
|
|
439
|
+
});
|
|
440
|
+
const selectedAccount = selected.credentialId ? context.accounts.find((account) => account.id === selected.credentialId) : void 0;
|
|
441
|
+
const selectedIsAvailable = selectedAccount !== void 0 && (selectedAccount.id === context.existingCredentialId ? isCodexCredentialHealthy(selectedAccount, now) : isCodexCredentialEligible(selectedAccount, now));
|
|
442
|
+
if (selected.credentialId && selectedIsAvailable) {
|
|
443
|
+
return {
|
|
444
|
+
kind: "available",
|
|
445
|
+
credentialId: selected.credentialId,
|
|
446
|
+
diagnostic: {
|
|
447
|
+
connectedCount: context.accounts.length,
|
|
448
|
+
eligibleCount: context.accounts.filter((account) => isCodexCredentialEligible(account, now)).length
|
|
449
|
+
}
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
const authoritativeReset = authoritativeCodexCapacityResetAt(context.accounts, now);
|
|
453
|
+
return {
|
|
454
|
+
kind: "unavailable",
|
|
455
|
+
earliestResetAt: authoritativeReset,
|
|
456
|
+
resetKind: authoritativeReset ? "authoritative" : "bounded_refresh",
|
|
457
|
+
diagnostic: {
|
|
458
|
+
connectedCount: context.accounts.length,
|
|
459
|
+
allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
|
|
460
|
+
policyHash: context.policyHash
|
|
461
|
+
}
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
async function refreshCapacityMetadata(services, workspaceId) {
|
|
465
|
+
const accounts = await listCodexAccountStatuses(services.db, workspaceId).catch(() => []);
|
|
466
|
+
const stale = accounts.filter(
|
|
467
|
+
(account) => account.allocatorEnabled && account.status === "active" && ((account.primaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || (account.secondaryUsedPercent ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT || account.usageCheckedAt === null)
|
|
468
|
+
);
|
|
469
|
+
await refreshCodexUsageAndRepairCapacityWaiters(
|
|
470
|
+
stale.map(
|
|
471
|
+
(account) => () => fetchCodexUsageForAccount(services.db, services.settings, workspaceId, account.id)
|
|
472
|
+
),
|
|
473
|
+
() => signalPendingCodexCapacityWakeTargets(services, workspaceId)
|
|
474
|
+
);
|
|
475
|
+
}
|
|
476
|
+
function createCodexCapacityActivities(services) {
|
|
477
|
+
async function getCodexCapacityWait(input) {
|
|
478
|
+
const { db } = await services();
|
|
479
|
+
const waiter = await getCodexCapacityWaitForSession(db, input.workspaceId, input.sessionId);
|
|
480
|
+
return waiter ? {
|
|
481
|
+
waiterId: waiter.id,
|
|
482
|
+
generation: waiter.generation,
|
|
483
|
+
// A capacity mutation may have committed while its Temporal signal
|
|
484
|
+
// was lost or while the workflow continued-as-new. Reconstruct that
|
|
485
|
+
// outbox edge as an immediate re-evaluation rather than waiting for
|
|
486
|
+
// the older timer.
|
|
487
|
+
nextCheckAt: waiter.wakeRevision > waiter.observedWakeRevision ? (/* @__PURE__ */ new Date(0)).toISOString() : waiter.nextCheckAt.toISOString(),
|
|
488
|
+
wakeRevision: waiter.wakeRevision
|
|
489
|
+
} : null;
|
|
490
|
+
}
|
|
491
|
+
async function reconcileCodexCapacityWait(input) {
|
|
492
|
+
const resolved = await services();
|
|
493
|
+
const current = await getCodexCapacityWaitForSession(
|
|
494
|
+
resolved.db,
|
|
495
|
+
input.workspaceId,
|
|
496
|
+
input.sessionId
|
|
497
|
+
);
|
|
498
|
+
if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
|
|
499
|
+
return { action: "stale" };
|
|
500
|
+
}
|
|
501
|
+
if (input.cause === "timer" && current.nextCheckAt.getTime() <= Date.now()) {
|
|
502
|
+
await refreshCapacityMetadata(resolved, input.workspaceId);
|
|
503
|
+
}
|
|
504
|
+
const result = await reconcileCodexCapacityWaitDb(
|
|
505
|
+
resolved.db,
|
|
506
|
+
{
|
|
507
|
+
accountId: input.accountId,
|
|
508
|
+
workspaceId: input.workspaceId,
|
|
509
|
+
sessionId: input.sessionId,
|
|
510
|
+
waiterId: input.waiterId,
|
|
511
|
+
generation: input.generation
|
|
512
|
+
},
|
|
513
|
+
(context) => codexCapacityDecision(context, resolved.settings)
|
|
514
|
+
);
|
|
515
|
+
if (result.events.length > 0) {
|
|
516
|
+
try {
|
|
517
|
+
await resolved.bus.publish(input.workspaceId, input.sessionId, result.events);
|
|
518
|
+
} catch {
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
if (result.action === "resumed") {
|
|
522
|
+
return { action: "resumed" };
|
|
523
|
+
}
|
|
524
|
+
if (result.action === "waiting") {
|
|
525
|
+
return {
|
|
526
|
+
action: "waiting",
|
|
527
|
+
waiterId: result.waiter.id,
|
|
528
|
+
generation: result.waiter.generation,
|
|
529
|
+
nextCheckAt: result.waiter.nextCheckAt.toISOString(),
|
|
530
|
+
wakeRevision: result.waiter.wakeRevision
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
return { action: result.action };
|
|
534
|
+
}
|
|
535
|
+
return { getCodexCapacityWait, reconcileCodexCapacityWait };
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// src/activities/goals.ts
|
|
539
|
+
import {
|
|
540
|
+
configuredStaticUsageLimits,
|
|
541
|
+
policyProviderIdForModel
|
|
542
|
+
} from "@opengeni/config";
|
|
543
|
+
import {
|
|
544
|
+
evaluateWorkspaceModelPolicy,
|
|
545
|
+
latencyModeForMetadata,
|
|
546
|
+
mergeToolRefs,
|
|
547
|
+
reasoningEffortForMetadata
|
|
548
|
+
} from "@opengeni/contracts";
|
|
549
|
+
import { isCodexBilledModel } from "@opengeni/codex";
|
|
550
|
+
import {
|
|
551
|
+
enqueueSessionWorkflowWakeIfRunnable,
|
|
552
|
+
getBillingBalance,
|
|
553
|
+
getLatestStartedSessionTurn,
|
|
554
|
+
getWorkspaceModelPolicy,
|
|
555
|
+
getSessionGoal,
|
|
556
|
+
isCodexBilledTurn,
|
|
557
|
+
materializeGoalContinuation,
|
|
558
|
+
requireSession,
|
|
559
|
+
sumUsageQuantity
|
|
560
|
+
} from "@opengeni/db";
|
|
561
|
+
function createGoalActivities(services) {
|
|
562
|
+
async function enqueueGoalRetryWake(input) {
|
|
563
|
+
const { db } = await services();
|
|
564
|
+
await enqueueSessionWorkflowWakeIfRunnable(db, {
|
|
565
|
+
accountId: input.accountId,
|
|
566
|
+
workspaceId: input.workspaceId,
|
|
567
|
+
sessionId: input.sessionId,
|
|
568
|
+
temporalWorkflowId: input.workflowId,
|
|
569
|
+
reason: "goal_retry",
|
|
570
|
+
// A permanently invalid goal must not become a tight workflow loop. One
|
|
571
|
+
// durable retry after a short delay preserves liveness without scanning.
|
|
572
|
+
notBefore: new Date(Date.now() + 3e4)
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
async function maybeContinueGoal(input) {
|
|
576
|
+
const { settings, db, bus } = await services();
|
|
577
|
+
const existingGoal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
578
|
+
if (!existingGoal || existingGoal.status !== "active") {
|
|
579
|
+
return { action: "none" };
|
|
580
|
+
}
|
|
581
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
582
|
+
const latestStartedTurn = await getLatestStartedSessionTurn(
|
|
583
|
+
db,
|
|
584
|
+
input.workspaceId,
|
|
585
|
+
input.sessionId
|
|
586
|
+
);
|
|
587
|
+
let continuationModel = latestStartedTurn?.model ?? session.model;
|
|
588
|
+
const continuationReasoningEffort = latestStartedTurn?.reasoningEffort ?? reasoningEffortForMetadata(session.metadata, settings.openaiReasoningEffort);
|
|
589
|
+
const continuationLatencyMode = latestStartedTurn?.latencyMode ?? latencyModeForMetadata(session.metadata, "standard");
|
|
590
|
+
let modelPolicyBlocked = null;
|
|
591
|
+
const workspaceModelPolicy = await getWorkspaceModelPolicy(db, input.workspaceId);
|
|
592
|
+
if (workspaceModelPolicy) {
|
|
593
|
+
const policyBlocks = (modelId) => !evaluateWorkspaceModelPolicy(workspaceModelPolicy, {
|
|
594
|
+
providerId: policyProviderIdForModel(settings, modelId),
|
|
595
|
+
modelId
|
|
596
|
+
}).allowed;
|
|
597
|
+
if (policyBlocks(continuationModel)) {
|
|
598
|
+
if (continuationModel !== session.model && !policyBlocks(session.model)) {
|
|
599
|
+
continuationModel = session.model;
|
|
600
|
+
} else {
|
|
601
|
+
modelPolicyBlocked = `workspace model policy blocks model "${continuationModel}"; pick an allowed model or change the workspace model policy`;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
if (!modelPolicyBlocked && session.codexCompactionMode === "remote_v2" && !isCodexBilledModel(continuationModel)) {
|
|
606
|
+
modelPolicyBlocked = `session is locked to Codex remote compaction v2; model "${continuationModel}" is not a Codex subscription model`;
|
|
607
|
+
}
|
|
608
|
+
const isCodexRun = await isCodexBilledTurn({
|
|
609
|
+
db,
|
|
610
|
+
settings,
|
|
611
|
+
workspaceId: input.workspaceId,
|
|
612
|
+
model: continuationModel
|
|
613
|
+
});
|
|
614
|
+
const budgetBlocked = await goalRunBudgetBlocked(
|
|
615
|
+
settings,
|
|
616
|
+
db,
|
|
617
|
+
input.accountId,
|
|
618
|
+
input.workspaceId,
|
|
619
|
+
isCodexRun
|
|
620
|
+
);
|
|
621
|
+
const decision = await materializeGoalContinuation(db, {
|
|
622
|
+
accountId: input.accountId,
|
|
623
|
+
workspaceId: input.workspaceId,
|
|
624
|
+
sessionId: input.sessionId,
|
|
625
|
+
workflowId: input.workflowId,
|
|
626
|
+
defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
|
|
627
|
+
noProgressLimit: settings.goalNoProgressLimit,
|
|
628
|
+
// A model-policy block takes precedence: it is deterministic (a budget
|
|
629
|
+
// pause can clear on its own; a policy pause needs a model/policy change)
|
|
630
|
+
// and rides the same visible-pause channel.
|
|
631
|
+
budgetBlocked: modelPolicyBlocked ?? budgetBlocked,
|
|
632
|
+
policy: {
|
|
633
|
+
model: continuationModel,
|
|
634
|
+
reasoningEffort: continuationReasoningEffort,
|
|
635
|
+
latencyMode: continuationLatencyMode,
|
|
636
|
+
tools: withFirstPartyTools(settings, session.tools),
|
|
637
|
+
sandboxBackend: session.sandboxBackend
|
|
638
|
+
},
|
|
639
|
+
prompt: goalContinuationPrompt
|
|
640
|
+
});
|
|
641
|
+
if (decision.events.length > 0) {
|
|
642
|
+
await bus.publish(input.workspaceId, input.sessionId, decision.events);
|
|
643
|
+
}
|
|
644
|
+
return { action: decision.action };
|
|
645
|
+
}
|
|
646
|
+
return {
|
|
647
|
+
enqueueGoalRetryWake,
|
|
648
|
+
maybeContinueGoal
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
function goalContinuationPrompt(goal, autoContinuation, cap) {
|
|
652
|
+
const counter = cap === null ? `${autoContinuation}` : `${autoContinuation}/${cap}`;
|
|
653
|
+
return [
|
|
654
|
+
`[GOAL CONTINUATION ${counter}] The session goal is not done. Goal: ${goal.text}.`,
|
|
655
|
+
`Success criteria: ${goal.successCriteria ?? "none specified"}.`,
|
|
656
|
+
"Continue from the existing conversation state. Do not repeat completed session setup, persistent metadata settings, or context checks merely because this is a new continuation turn.",
|
|
657
|
+
"Continue working toward the goal now. If it is actually complete, call opengeni__goal_complete with concrete evidence.",
|
|
658
|
+
"If you are blocked or continuing is not productive, call opengeni__goal_pause with your rationale.",
|
|
659
|
+
"You may revise the goal with opengeni__goal_update. Do not stop without one of these explicit actions."
|
|
660
|
+
].join("\n");
|
|
661
|
+
}
|
|
662
|
+
function withFirstPartyTools(settings, tools) {
|
|
663
|
+
if (!settings.mcpServers.some((server) => server.id === "opengeni")) {
|
|
664
|
+
return tools;
|
|
665
|
+
}
|
|
666
|
+
return mergeToolRefs(tools, [{ kind: "mcp", id: "opengeni" }]);
|
|
667
|
+
}
|
|
668
|
+
async function goalRunBudgetBlocked(settings, db, accountId, workspaceId, isCodexRun) {
|
|
669
|
+
if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
670
|
+
const balance = await getBillingBalance(db, accountId);
|
|
671
|
+
if (balance.balanceMicros <= 0) {
|
|
672
|
+
return "insufficient OpenGeni credits";
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
676
|
+
const limits = configuredStaticUsageLimits(settings);
|
|
677
|
+
if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
|
|
678
|
+
const used = await sumUsageQuantity(db, {
|
|
679
|
+
accountId,
|
|
680
|
+
eventType: "model.cost",
|
|
681
|
+
since: startOfUtcMonth()
|
|
682
|
+
});
|
|
683
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
684
|
+
return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
688
|
+
const used = await sumUsageQuantity(db, {
|
|
689
|
+
workspaceId,
|
|
690
|
+
eventType: "agent_run.created",
|
|
691
|
+
since: startOfUtcMonth()
|
|
692
|
+
});
|
|
693
|
+
if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
694
|
+
return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
return null;
|
|
699
|
+
}
|
|
700
|
+
function startOfUtcMonth() {
|
|
701
|
+
const now = /* @__PURE__ */ new Date();
|
|
702
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// src/activities/sandbox-images.ts
|
|
706
|
+
import {
|
|
707
|
+
rigProviderImageContentHash,
|
|
708
|
+
rigProviderImageMatchesDefinition,
|
|
709
|
+
rigProviderImageProviderBindingKeyHash
|
|
710
|
+
} from "@opengeni/core";
|
|
711
|
+
import { resolveModalCheckpointProviderBinding } from "@opengeni/runtime/sandbox";
|
|
712
|
+
function settingsWithPackSandboxImage(settings, sandboxImage, sandboxProviderImages = null) {
|
|
713
|
+
if (!sandboxImage) return settings;
|
|
714
|
+
return {
|
|
715
|
+
...settings,
|
|
716
|
+
dockerImage: sandboxImage,
|
|
717
|
+
modalImageRef: sandboxImage,
|
|
718
|
+
modalImageId: sandboxProviderImages?.modal?.imageId
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
function settingsWithRigImage(settings, rigImage) {
|
|
722
|
+
if (!rigImage) return settings;
|
|
723
|
+
return {
|
|
724
|
+
...settings,
|
|
725
|
+
dockerImage: rigImage,
|
|
726
|
+
modalImageRef: rigImage,
|
|
727
|
+
// A logical rig image invalidates a lower-precedence pack/deployment
|
|
728
|
+
// provider ID. A verified version-bound provider ID is applied separately
|
|
729
|
+
// after this precedence step.
|
|
730
|
+
modalImageId: void 0
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
function rigProviderImageSourceImage(settings, backend) {
|
|
734
|
+
if (backend === "modal") return settings.modalImageId ?? settings.modalImageRef ?? null;
|
|
735
|
+
if (backend === "docker") return settings.dockerImage ?? null;
|
|
736
|
+
return null;
|
|
737
|
+
}
|
|
738
|
+
function resolveRigProviderImageSelection(settings, version, backend, currentProviderBindingKeyHash) {
|
|
739
|
+
if (!version) {
|
|
740
|
+
return { settings, reason: "missing", contentHash: null, imageId: null };
|
|
741
|
+
}
|
|
742
|
+
if (backend !== "modal") {
|
|
743
|
+
return { settings, reason: "provider_unsupported", contentHash: null, imageId: null };
|
|
744
|
+
}
|
|
745
|
+
const image = version.providerImages[backend];
|
|
746
|
+
if (!image) {
|
|
747
|
+
return { settings, reason: "missing", contentHash: null, imageId: null };
|
|
748
|
+
}
|
|
749
|
+
if (image.status !== "ready" || !image.imageId) {
|
|
750
|
+
return {
|
|
751
|
+
settings,
|
|
752
|
+
reason: "not_ready",
|
|
753
|
+
contentHash: image.contentHash,
|
|
754
|
+
imageId: image.imageId
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
if (image.coldBootValidation?.version !== 1) {
|
|
758
|
+
return {
|
|
759
|
+
settings,
|
|
760
|
+
reason: "not_cold_boot_validated",
|
|
761
|
+
contentHash: image.contentHash,
|
|
762
|
+
imageId: null
|
|
763
|
+
};
|
|
764
|
+
}
|
|
765
|
+
const sourceImage = rigProviderImageSourceImage(settings, backend);
|
|
766
|
+
const expectedContentHash = rigProviderImageContentHash({
|
|
767
|
+
backend,
|
|
768
|
+
sourceImage,
|
|
769
|
+
definition: version
|
|
770
|
+
});
|
|
771
|
+
if (image.sourceImage !== sourceImage || image.contentHash !== expectedContentHash || !rigProviderImageMatchesDefinition(image, version)) {
|
|
772
|
+
return {
|
|
773
|
+
settings,
|
|
774
|
+
reason: "content_mismatch",
|
|
775
|
+
contentHash: expectedContentHash,
|
|
776
|
+
imageId: null
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
if (!image.providerBindingKeyHash || !currentProviderBindingKeyHash) {
|
|
780
|
+
return {
|
|
781
|
+
settings,
|
|
782
|
+
reason: "provider_binding_unavailable",
|
|
783
|
+
contentHash: expectedContentHash,
|
|
784
|
+
imageId: null
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
if (image.providerBindingKeyHash !== currentProviderBindingKeyHash) {
|
|
788
|
+
return {
|
|
789
|
+
settings,
|
|
790
|
+
reason: "provider_binding_mismatch",
|
|
791
|
+
contentHash: expectedContentHash,
|
|
792
|
+
imageId: null
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
return {
|
|
796
|
+
// Keep modalImageRef unchanged: the lease continues to fence the logical
|
|
797
|
+
// rig/pack/deployment image, so a newly available build optimization never
|
|
798
|
+
// rotates an already-warm or archive-backed box. modalImageId affects only
|
|
799
|
+
// genuinely fresh provider creation.
|
|
800
|
+
settings: { ...settings, modalImageId: image.imageId },
|
|
801
|
+
reason: "selected",
|
|
802
|
+
contentHash: expectedContentHash,
|
|
803
|
+
imageId: image.imageId
|
|
804
|
+
};
|
|
805
|
+
}
|
|
806
|
+
async function settingsWithRigProviderImage(settings, version, backend, resolveBinding = resolveModalCheckpointProviderBinding) {
|
|
807
|
+
const image = backend === "modal" ? version?.providerImages.modal : null;
|
|
808
|
+
if (image?.status !== "ready" || !image.providerBindingKeyHash) {
|
|
809
|
+
return resolveRigProviderImageSelection(settings, version, backend, null).settings;
|
|
810
|
+
}
|
|
811
|
+
const structural = resolveRigProviderImageSelection(
|
|
812
|
+
settings,
|
|
813
|
+
version,
|
|
814
|
+
backend,
|
|
815
|
+
image.providerBindingKeyHash
|
|
816
|
+
);
|
|
817
|
+
if (structural.reason !== "selected") return structural.settings;
|
|
818
|
+
let currentProviderBindingKeyHash = null;
|
|
819
|
+
try {
|
|
820
|
+
const binding = await resolveBinding(settings);
|
|
821
|
+
currentProviderBindingKeyHash = rigProviderImageProviderBindingKeyHash(binding.key);
|
|
822
|
+
} catch {
|
|
823
|
+
}
|
|
824
|
+
return resolveRigProviderImageSelection(settings, version, backend, currentProviderBindingKeyHash).settings;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
// src/activities/packs.ts
|
|
828
|
+
import { CapabilityPack } from "@opengeni/contracts";
|
|
829
|
+
import {
|
|
830
|
+
getWorkspace,
|
|
831
|
+
getWorkspacePack,
|
|
832
|
+
listInstalledPortableSkills,
|
|
833
|
+
listCapabilityInstallations,
|
|
834
|
+
listPackInstallations
|
|
835
|
+
} from "@opengeni/db";
|
|
836
|
+
import {
|
|
837
|
+
buildPortableSkillArtifact,
|
|
838
|
+
isSkillLibraryEntryId,
|
|
839
|
+
loadSkillLibrarySkill
|
|
840
|
+
} from "@opengeni/runtime";
|
|
841
|
+
var emptyPackRuntime = {
|
|
842
|
+
sandboxImage: null,
|
|
843
|
+
sandboxProviderImages: null,
|
|
844
|
+
skills: []
|
|
845
|
+
};
|
|
846
|
+
async function resolveWorkspacePackRuntime(db, workspaceId) {
|
|
847
|
+
const installations = await listPackInstallations(db, workspaceId);
|
|
848
|
+
const active = installations.filter((installation) => installation.status === "active");
|
|
849
|
+
if (active.length === 0) {
|
|
850
|
+
return emptyPackRuntime;
|
|
851
|
+
}
|
|
852
|
+
const packs = [];
|
|
853
|
+
for (const installation of active) {
|
|
854
|
+
if (!packInstallationUsesLegacyRuntime(installation)) {
|
|
855
|
+
continue;
|
|
856
|
+
}
|
|
857
|
+
const registration = await getWorkspacePack(db, workspaceId, installation.packId);
|
|
858
|
+
if (!registration) {
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
const parsed = CapabilityPack.safeParse(registration.pack);
|
|
862
|
+
if (parsed.success) {
|
|
863
|
+
packs.push(parsed.data);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
return workspacePackRuntimeFromPacks(packs);
|
|
867
|
+
}
|
|
868
|
+
function packInstallationUsesLegacyRuntime(installation) {
|
|
869
|
+
return installation.manifestSnapshot === null && installation.manifestDigest === null;
|
|
870
|
+
}
|
|
871
|
+
async function resolveWorkspaceSkillLibraryRuntime(db, workspaceId) {
|
|
872
|
+
const installations = await listCapabilityInstallations(db, workspaceId);
|
|
873
|
+
const activeSkills = installations.filter(
|
|
874
|
+
(installation) => installation.status === "active" && installation.kind === "skill" && installation.capabilityId.startsWith("skill:") && isSkillLibraryEntryId(installation.capabilityId.slice("skill:".length))
|
|
875
|
+
).sort((a, b) => a.capabilityId.localeCompare(b.capabilityId));
|
|
876
|
+
const skillLibrarySkills = [];
|
|
877
|
+
const skillLibrarySelections = [];
|
|
878
|
+
for (const installation of activeSkills) {
|
|
879
|
+
const metadata = parseSkillLibraryInstallationMetadata(
|
|
880
|
+
installation.metadata,
|
|
881
|
+
installation.capabilityId
|
|
882
|
+
);
|
|
883
|
+
if (installation.capabilityId !== `skill:${metadata.libraryId}`) {
|
|
884
|
+
throw new Error(
|
|
885
|
+
`Skill installation id does not match its library entry: ${installation.capabilityId} -> ${metadata.libraryId}`
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
const loaded = loadSkillLibrarySkill(metadata.libraryId, metadata.libraryVersion);
|
|
889
|
+
if (installation.config.version !== metadata.libraryVersion) {
|
|
890
|
+
throw new Error(
|
|
891
|
+
`Skill installation version mismatch for ${metadata.libraryId}@${metadata.libraryVersion}: config has ${String(installation.config.version)}`
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
if (loaded.entry.contentSha256 !== metadata.contentSha256) {
|
|
895
|
+
throw new Error(
|
|
896
|
+
`Skill installation hash mismatch for ${metadata.libraryId}@${metadata.libraryVersion}: expected ${metadata.contentSha256}, catalog has ${loaded.entry.contentSha256}`
|
|
897
|
+
);
|
|
898
|
+
}
|
|
899
|
+
if (loaded.entry.sourceCommit !== metadata.sourceCommit || loaded.entry.provenance !== metadata.provenance) {
|
|
900
|
+
throw new Error(
|
|
901
|
+
`Skill installation provenance mismatch for ${metadata.libraryId}@${metadata.libraryVersion}; re-enable the skill from the current catalog`
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
skillLibrarySkills.push({
|
|
905
|
+
name: loaded.skill.name,
|
|
906
|
+
description: loaded.skill.description,
|
|
907
|
+
files: loaded.skill.files.map((file) => ({ path: file.path, content: file.content }))
|
|
908
|
+
});
|
|
909
|
+
skillLibrarySelections.push({
|
|
910
|
+
id: loaded.entry.id,
|
|
911
|
+
name: loaded.entry.name,
|
|
912
|
+
source: "library",
|
|
913
|
+
version: loaded.entry.version,
|
|
914
|
+
contentSha256: loaded.entry.contentSha256,
|
|
915
|
+
reason: "enabled workspace capability installation"
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
const importedSkills = await listInstalledPortableSkills(db, workspaceId);
|
|
919
|
+
for (const imported of importedSkills) {
|
|
920
|
+
const artifact = buildPortableSkillArtifact(imported.files);
|
|
921
|
+
if (artifact.name !== imported.name || artifact.description !== imported.description || artifact.contentSha256 !== imported.contentSha256) {
|
|
922
|
+
throw new Error(
|
|
923
|
+
`Imported Skill artifact verification failed for ${imported.capabilityId}; reinstall it from the pinned source`
|
|
924
|
+
);
|
|
925
|
+
}
|
|
926
|
+
skillLibrarySkills.push({
|
|
927
|
+
name: artifact.name,
|
|
928
|
+
description: artifact.description,
|
|
929
|
+
files: artifact.files.map((file) => ({ path: file.path, content: file.content }))
|
|
930
|
+
});
|
|
931
|
+
skillLibrarySelections.push({
|
|
932
|
+
id: imported.capabilityId,
|
|
933
|
+
name: artifact.name,
|
|
934
|
+
source: "imported",
|
|
935
|
+
version: imported.sourceCommit,
|
|
936
|
+
contentSha256: artifact.contentSha256,
|
|
937
|
+
reason: `installed from ${imported.sourceUrl}`
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
return { skillLibrarySkills, skillLibrarySelections };
|
|
941
|
+
}
|
|
942
|
+
function parseSkillLibraryInstallationMetadata(metadata, capabilityId) {
|
|
943
|
+
const libraryId = stringMetadata(metadata.libraryId);
|
|
944
|
+
const libraryVersion = stringMetadata(metadata.libraryVersion);
|
|
945
|
+
const contentSha256 = stringMetadata(metadata.contentSha256);
|
|
946
|
+
const sourceCommit = stringMetadata(metadata.sourceCommit);
|
|
947
|
+
const provenance = stringMetadata(metadata.provenance);
|
|
948
|
+
if (!libraryId || !libraryVersion || !contentSha256 || !sourceCommit || !provenance) {
|
|
949
|
+
throw new Error(
|
|
950
|
+
`Skill installation ${capabilityId} is missing immutable library metadata; re-enable the skill from the current catalog`
|
|
951
|
+
);
|
|
952
|
+
}
|
|
953
|
+
return { libraryId, libraryVersion, contentSha256, sourceCommit, provenance };
|
|
954
|
+
}
|
|
955
|
+
function stringMetadata(value) {
|
|
956
|
+
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
957
|
+
}
|
|
958
|
+
function workspacePackRuntimeFromPacks(packs) {
|
|
959
|
+
const imagePacks = packs.filter(
|
|
960
|
+
(pack) => typeof pack.sandboxImage === "string" && pack.sandboxImage.trim().length > 0
|
|
961
|
+
);
|
|
962
|
+
if (imagePacks.length > 1) {
|
|
963
|
+
const ids = imagePacks.map((pack) => pack.id).sort().join(", ");
|
|
964
|
+
throw new Error(
|
|
965
|
+
`Multiple enabled packs declare a sandbox image (${ids}). Only one enabled pack per workspace may declare sandboxImage; disable the others and retry.`
|
|
966
|
+
);
|
|
967
|
+
}
|
|
968
|
+
const skills = [];
|
|
969
|
+
const skillOwners = /* @__PURE__ */ new Map();
|
|
970
|
+
for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
971
|
+
for (const skill of pack.skills) {
|
|
972
|
+
const key = skill.name.toLowerCase();
|
|
973
|
+
const existingOwner = skillOwners.get(key);
|
|
974
|
+
if (existingOwner !== void 0 && existingOwner !== pack.id) {
|
|
975
|
+
throw new Error(
|
|
976
|
+
`Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
skillOwners.set(key, pack.id);
|
|
980
|
+
skills.push({
|
|
981
|
+
name: skill.name,
|
|
982
|
+
description: skill.description ?? null,
|
|
983
|
+
files: skill.files.map((file) => ({ path: file.path, content: file.content }))
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
return {
|
|
988
|
+
sandboxImage: imagePacks[0]?.sandboxImage?.trim() ?? null,
|
|
989
|
+
sandboxProviderImages: imagePacks[0]?.sandboxProviderImages ?? null,
|
|
990
|
+
skills
|
|
991
|
+
};
|
|
992
|
+
}
|
|
993
|
+
function mergeRigDefaultVariableSetEnvironment(rigDefaultValues, sessionValues) {
|
|
994
|
+
return { ...rigDefaultValues, ...sessionValues };
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
// src/activities/streaming.ts
|
|
998
|
+
import { TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS } from "@opengeni/core";
|
|
999
|
+
import { Context } from "@temporalio/activity";
|
|
1000
|
+
var TRAILING_FLUSH_MS = 33;
|
|
1001
|
+
var MAX_BATCH_EVENTS = 50;
|
|
1002
|
+
var MAX_BUFFERED_EVENTS = 1e3;
|
|
1003
|
+
function createRuntimeBatcher(flushEvents, options = {}) {
|
|
1004
|
+
const now = options.now ?? (() => performance.now());
|
|
1005
|
+
let pending = [];
|
|
1006
|
+
let lastFlush = Date.now();
|
|
1007
|
+
let trailingTimer = null;
|
|
1008
|
+
let activeBatchSize = 0;
|
|
1009
|
+
let drainPromise = null;
|
|
1010
|
+
let flushFailed = false;
|
|
1011
|
+
let flushFailure;
|
|
1012
|
+
const structural = /* @__PURE__ */ new Set([
|
|
1013
|
+
"terminal.pty.output.delta",
|
|
1014
|
+
"agent.toolCall.created",
|
|
1015
|
+
"agent.toolCall.output",
|
|
1016
|
+
"agent.message.completed",
|
|
1017
|
+
"tool.auth_needed",
|
|
1018
|
+
"credential.auth_needed",
|
|
1019
|
+
"session.requiresAction",
|
|
1020
|
+
"turn.completed",
|
|
1021
|
+
"turn.failed",
|
|
1022
|
+
"turn.cancelled"
|
|
1023
|
+
]);
|
|
1024
|
+
return {
|
|
1025
|
+
push: async (event) => {
|
|
1026
|
+
throwIfFlushFailed();
|
|
1027
|
+
pending.push({ type: event.type, payload: event.payload });
|
|
1028
|
+
const elapsed = Date.now() - lastFlush;
|
|
1029
|
+
if (structural.has(event.type)) {
|
|
1030
|
+
await flush();
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
if (pending.length >= MAX_BATCH_EVENTS || elapsed >= TRAILING_FLUSH_MS) {
|
|
1034
|
+
void startDrain().catch(() => void 0);
|
|
1035
|
+
}
|
|
1036
|
+
if (activeBatchSize + pending.length >= MAX_BUFFERED_EVENTS) {
|
|
1037
|
+
await flush();
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
armTrailingTimer();
|
|
1041
|
+
},
|
|
1042
|
+
flush
|
|
1043
|
+
};
|
|
1044
|
+
function armTrailingTimer() {
|
|
1045
|
+
if (trailingTimer !== null) {
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
trailingTimer = setTimeout(() => {
|
|
1049
|
+
trailingTimer = null;
|
|
1050
|
+
void flush().catch(() => void 0);
|
|
1051
|
+
}, TRAILING_FLUSH_MS);
|
|
1052
|
+
if ("unref" in trailingTimer && typeof trailingTimer.unref === "function") {
|
|
1053
|
+
trailingTimer.unref();
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
function clearTrailingTimer() {
|
|
1057
|
+
if (trailingTimer !== null) {
|
|
1058
|
+
clearTimeout(trailingTimer);
|
|
1059
|
+
trailingTimer = null;
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
async function flush() {
|
|
1063
|
+
throwIfFlushFailed();
|
|
1064
|
+
clearTrailingTimer();
|
|
1065
|
+
await startDrain();
|
|
1066
|
+
throwIfFlushFailed();
|
|
1067
|
+
}
|
|
1068
|
+
function startDrain() {
|
|
1069
|
+
if (drainPromise) {
|
|
1070
|
+
return drainPromise;
|
|
1071
|
+
}
|
|
1072
|
+
if (flushFailed) {
|
|
1073
|
+
return Promise.reject(flushFailure);
|
|
1074
|
+
}
|
|
1075
|
+
if (pending.length === 0) {
|
|
1076
|
+
return Promise.resolve();
|
|
1077
|
+
}
|
|
1078
|
+
clearTrailingTimer();
|
|
1079
|
+
const running = (async () => {
|
|
1080
|
+
while (pending.length > 0) {
|
|
1081
|
+
const events = pending.splice(0, MAX_BATCH_EVENTS);
|
|
1082
|
+
activeBatchSize = events.length;
|
|
1083
|
+
lastFlush = Date.now();
|
|
1084
|
+
const startedAt = now();
|
|
1085
|
+
try {
|
|
1086
|
+
await flushEvents(events);
|
|
1087
|
+
} catch (error) {
|
|
1088
|
+
flushFailed = true;
|
|
1089
|
+
flushFailure = error;
|
|
1090
|
+
throw error;
|
|
1091
|
+
} finally {
|
|
1092
|
+
activeBatchSize = 0;
|
|
1093
|
+
if (options.onFlush) {
|
|
1094
|
+
try {
|
|
1095
|
+
options.onFlush({
|
|
1096
|
+
events: events.length,
|
|
1097
|
+
durationSeconds: Math.max(0, (now() - startedAt) / 1e3)
|
|
1098
|
+
});
|
|
1099
|
+
} catch {
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
})();
|
|
1105
|
+
drainPromise = running.finally(() => {
|
|
1106
|
+
drainPromise = null;
|
|
1107
|
+
});
|
|
1108
|
+
return drainPromise;
|
|
1109
|
+
}
|
|
1110
|
+
function throwIfFlushFailed() {
|
|
1111
|
+
if (flushFailed) {
|
|
1112
|
+
throw flushFailure;
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
function currentActivityContext() {
|
|
1117
|
+
try {
|
|
1118
|
+
return Context.current();
|
|
1119
|
+
} catch {
|
|
1120
|
+
return null;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
function startActivityHeartbeat(context, details, schedule = setInterval) {
|
|
1124
|
+
if (!context) {
|
|
1125
|
+
return null;
|
|
1126
|
+
}
|
|
1127
|
+
const timer = schedule(() => {
|
|
1128
|
+
context.heartbeat({ ...details, at: (/* @__PURE__ */ new Date()).toISOString() });
|
|
1129
|
+
}, TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS);
|
|
1130
|
+
if ("unref" in timer && typeof timer.unref === "function") {
|
|
1131
|
+
timer.unref();
|
|
1132
|
+
}
|
|
1133
|
+
return timer;
|
|
1134
|
+
}
|
|
1135
|
+
async function nextStreamEvent(iterator, context) {
|
|
1136
|
+
if (!context) {
|
|
1137
|
+
return await iterator.next();
|
|
1138
|
+
}
|
|
1139
|
+
const signal = context.cancellationSignal;
|
|
1140
|
+
if (signal.aborted) {
|
|
1141
|
+
throw signal.reason;
|
|
1142
|
+
}
|
|
1143
|
+
return await new Promise((resolve, reject) => {
|
|
1144
|
+
let settled = false;
|
|
1145
|
+
const settle = (complete) => {
|
|
1146
|
+
if (settled) return;
|
|
1147
|
+
settled = true;
|
|
1148
|
+
signal.removeEventListener("abort", onAbort);
|
|
1149
|
+
complete();
|
|
1150
|
+
};
|
|
1151
|
+
const onAbort = () => settle(() => reject(signal.reason));
|
|
1152
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1153
|
+
if (signal.aborted) {
|
|
1154
|
+
onAbort();
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
let next;
|
|
1158
|
+
try {
|
|
1159
|
+
next = iterator.next();
|
|
1160
|
+
} catch (error) {
|
|
1161
|
+
settle(() => reject(error));
|
|
1162
|
+
return;
|
|
1163
|
+
}
|
|
1164
|
+
void Promise.resolve(next).then(
|
|
1165
|
+
(result) => settle(() => resolve(result)),
|
|
1166
|
+
(error) => settle(() => reject(error))
|
|
1167
|
+
);
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
// src/activities/parent-wake.ts
|
|
1172
|
+
import {
|
|
1173
|
+
getSession,
|
|
1174
|
+
getSessionGoal as getSessionGoal2,
|
|
1175
|
+
getSessionParentPersonalConnectionDelegations,
|
|
1176
|
+
addSessionSystemUpdateWithSourceMutation,
|
|
1177
|
+
claimPendingSessionSystemUpdateOutbox,
|
|
1178
|
+
claimPendingSessionWorkflowWakes,
|
|
1179
|
+
getSessionSystemUpdateOutboxByDedupeKey,
|
|
1180
|
+
getOrCreateSessionSystemUpdateOutbox,
|
|
1181
|
+
markSessionWorkflowWakeFailed,
|
|
1182
|
+
markSessionSystemUpdateOutboxDeliveredInTransaction,
|
|
1183
|
+
markSessionSystemUpdateOutboxFailed
|
|
1184
|
+
} from "@opengeni/db";
|
|
1185
|
+
async function notifyParentOfChildIdle(svc, workspaceId, childSessionId, episodeKey) {
|
|
1186
|
+
try {
|
|
1187
|
+
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
1188
|
+
if (!child || !child.parentSessionId) {
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
|
|
1192
|
+
const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
|
|
1193
|
+
const payload = childCompletionPayload(child, goal);
|
|
1194
|
+
const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
|
|
1195
|
+
svc.db,
|
|
1196
|
+
workspaceId,
|
|
1197
|
+
childSessionId
|
|
1198
|
+
);
|
|
1199
|
+
const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
|
|
1200
|
+
accountId: child.accountId,
|
|
1201
|
+
workspaceId,
|
|
1202
|
+
sourceSessionId: child.id,
|
|
1203
|
+
targetSessionId: child.parentSessionId,
|
|
1204
|
+
kind: "child_terminal_result",
|
|
1205
|
+
classification: goal?.status === "paused" ? "action_required" : "success",
|
|
1206
|
+
sourceId: child.id,
|
|
1207
|
+
dedupeKey: clientEventId,
|
|
1208
|
+
summary: childCompletionSummary(child, goal, "idle"),
|
|
1209
|
+
payload,
|
|
1210
|
+
lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
|
|
1211
|
+
personalConnectionDelegations
|
|
1212
|
+
});
|
|
1213
|
+
if (outbox.status === "delivered") {
|
|
1214
|
+
return;
|
|
1215
|
+
}
|
|
1216
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
1217
|
+
} catch (error) {
|
|
1218
|
+
svc.observability.error("Failed to wake parent session on worker idle boundary", {
|
|
1219
|
+
childSessionId,
|
|
1220
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1221
|
+
});
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
async function deliverFailedChildTurnToParent(svc, workspaceId, childSessionId, turnId) {
|
|
1225
|
+
try {
|
|
1226
|
+
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
1227
|
+
if (!child || !child.parentSessionId) return;
|
|
1228
|
+
const dedupeKey = `child-completion:${childSessionId}:turn:${turnId}`;
|
|
1229
|
+
const outbox = await getSessionSystemUpdateOutboxByDedupeKey(svc.db, {
|
|
1230
|
+
accountId: child.accountId,
|
|
1231
|
+
workspaceId,
|
|
1232
|
+
dedupeKey
|
|
1233
|
+
});
|
|
1234
|
+
if (!outbox) {
|
|
1235
|
+
throw new Error(`Committed failed-child outbox disappeared: ${dedupeKey}`);
|
|
1236
|
+
}
|
|
1237
|
+
if (outbox.status === "delivered") return;
|
|
1238
|
+
await deliverParentSystemUpdateOutbox(svc, outbox);
|
|
1239
|
+
} catch (error) {
|
|
1240
|
+
svc.observability.error("Failed to deliver committed child-turn failure", {
|
|
1241
|
+
childSessionId,
|
|
1242
|
+
turnId,
|
|
1243
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
async function deliverParentSystemUpdateOutbox(svc, outbox) {
|
|
1248
|
+
try {
|
|
1249
|
+
const result = await addSessionSystemUpdateWithSourceMutation(
|
|
1250
|
+
svc.db,
|
|
1251
|
+
{
|
|
1252
|
+
accountId: outbox.accountId,
|
|
1253
|
+
workspaceId: outbox.workspaceId,
|
|
1254
|
+
sessionId: outbox.targetSessionId,
|
|
1255
|
+
kind: outbox.kind,
|
|
1256
|
+
classification: outbox.classification,
|
|
1257
|
+
sourceId: outbox.sourceId,
|
|
1258
|
+
dedupeKey: outbox.dedupeKey,
|
|
1259
|
+
summary: outbox.summary,
|
|
1260
|
+
payload: outbox.payload,
|
|
1261
|
+
lineage: outbox.lineage,
|
|
1262
|
+
personalConnectionDelegations: outbox.personalConnectionDelegations
|
|
1263
|
+
},
|
|
1264
|
+
async (tx) => {
|
|
1265
|
+
await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
|
|
1266
|
+
}
|
|
1267
|
+
);
|
|
1268
|
+
if (result.reason === "session_cancelled") {
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
if (result.added && result.events.length > 0) {
|
|
1272
|
+
await svc.bus.publish(outbox.workspaceId, outbox.targetSessionId, result.events);
|
|
1273
|
+
}
|
|
1274
|
+
if (result.shouldWake && svc.wakeSessionWorkflow) {
|
|
1275
|
+
if (result.workflowWakeRevision === null) {
|
|
1276
|
+
throw new Error("Runnable system update has no workflow wake revision");
|
|
1277
|
+
}
|
|
1278
|
+
await svc.wakeSessionWorkflow({
|
|
1279
|
+
accountId: outbox.accountId,
|
|
1280
|
+
workspaceId: outbox.workspaceId,
|
|
1281
|
+
sessionId: outbox.targetSessionId,
|
|
1282
|
+
workflowId: result.temporalWorkflowId ?? `session-${outbox.targetSessionId}`,
|
|
1283
|
+
wakeRevision: result.workflowWakeRevision
|
|
1284
|
+
});
|
|
1285
|
+
}
|
|
1286
|
+
svc.observability.info("Woke parent session on worker completion", {
|
|
1287
|
+
childSessionId: outbox.sourceSessionId,
|
|
1288
|
+
parentSessionId: outbox.targetSessionId,
|
|
1289
|
+
dedupeKey: outbox.dedupeKey
|
|
1290
|
+
});
|
|
1291
|
+
} catch (error) {
|
|
1292
|
+
await markSessionSystemUpdateOutboxFailed(
|
|
1293
|
+
svc.db,
|
|
1294
|
+
outbox,
|
|
1295
|
+
error instanceof Error ? error.message : String(error)
|
|
1296
|
+
).catch(() => void 0);
|
|
1297
|
+
throw error;
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
async function reconcilePendingParentSystemUpdates(svc, limit = 100, overrides = {}) {
|
|
1301
|
+
const claimPendingSessionSystemUpdateOutboxFn = overrides.claimPendingSessionSystemUpdateOutbox ?? claimPendingSessionSystemUpdateOutbox;
|
|
1302
|
+
const rows = await claimPendingSessionSystemUpdateOutboxFn(svc.db, limit);
|
|
1303
|
+
let delivered = 0;
|
|
1304
|
+
let failed = 0;
|
|
1305
|
+
for (const row of rows) {
|
|
1306
|
+
try {
|
|
1307
|
+
await deliverParentSystemUpdateOutbox(svc, row);
|
|
1308
|
+
delivered += 1;
|
|
1309
|
+
} catch {
|
|
1310
|
+
failed += 1;
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
return { claimed: rows.length, delivered, failed };
|
|
1314
|
+
}
|
|
1315
|
+
async function reconcilePendingSessionWorkflowWakes(svc, limit = 1e3, overrides = {}) {
|
|
1316
|
+
const claimPendingSessionWorkflowWakesFn = overrides.claimPendingSessionWorkflowWakes ?? claimPendingSessionWorkflowWakes;
|
|
1317
|
+
if (!svc.wakeSessionWorkflow) {
|
|
1318
|
+
return { claimed: 0, delivered: 0, failed: 0 };
|
|
1319
|
+
}
|
|
1320
|
+
const repairs = await claimPendingSessionWorkflowWakesFn(svc.db, limit);
|
|
1321
|
+
let delivered = 0;
|
|
1322
|
+
let failed = 0;
|
|
1323
|
+
const queue = [...repairs];
|
|
1324
|
+
const workers = Array.from({ length: Math.min(20, queue.length) }, async () => {
|
|
1325
|
+
for (; ; ) {
|
|
1326
|
+
const repair = queue.shift();
|
|
1327
|
+
if (!repair) return;
|
|
1328
|
+
try {
|
|
1329
|
+
await svc.wakeSessionWorkflow({
|
|
1330
|
+
accountId: repair.accountId,
|
|
1331
|
+
workspaceId: repair.workspaceId,
|
|
1332
|
+
sessionId: repair.sessionId,
|
|
1333
|
+
workflowId: repair.temporalWorkflowId,
|
|
1334
|
+
wakeRevision: repair.wakeRevision,
|
|
1335
|
+
...repair.interruptionRequested ? { interruptionRequested: true } : {}
|
|
1336
|
+
});
|
|
1337
|
+
delivered += 1;
|
|
1338
|
+
} catch (error) {
|
|
1339
|
+
failed += 1;
|
|
1340
|
+
await markSessionWorkflowWakeFailed(
|
|
1341
|
+
svc.db,
|
|
1342
|
+
repair,
|
|
1343
|
+
error instanceof Error ? error.message : String(error)
|
|
1344
|
+
).catch(() => void 0);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
});
|
|
1348
|
+
await Promise.all(workers);
|
|
1349
|
+
return { claimed: repairs.length, delivered, failed };
|
|
1350
|
+
}
|
|
1351
|
+
function childCompletionPayload(child, goal) {
|
|
1352
|
+
return {
|
|
1353
|
+
type: "child_terminal_result",
|
|
1354
|
+
childSessionId: child.id,
|
|
1355
|
+
status: "idle",
|
|
1356
|
+
...goal ? {
|
|
1357
|
+
goal: {
|
|
1358
|
+
status: goal.status,
|
|
1359
|
+
text: goal.text,
|
|
1360
|
+
...goal.evidence ? { evidence: goal.evidence } : {},
|
|
1361
|
+
...goal.rationale ? { rationale: goal.rationale } : {},
|
|
1362
|
+
...goal.pausedReason ? { pausedReason: goal.pausedReason } : {}
|
|
1363
|
+
}
|
|
1364
|
+
} : {}
|
|
1365
|
+
};
|
|
1366
|
+
}
|
|
1367
|
+
function childCompletionSummary(child, goal, terminalStatus) {
|
|
1368
|
+
const lines = [];
|
|
1369
|
+
if (terminalStatus === "failed") {
|
|
1370
|
+
lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
|
|
1371
|
+
} else if (goal?.status === "completed") {
|
|
1372
|
+
lines.push(
|
|
1373
|
+
`A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`
|
|
1374
|
+
);
|
|
1375
|
+
} else if (goal?.status === "paused") {
|
|
1376
|
+
lines.push(
|
|
1377
|
+
`A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`
|
|
1378
|
+
);
|
|
1379
|
+
} else {
|
|
1380
|
+
lines.push(
|
|
1381
|
+
`A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`
|
|
1382
|
+
);
|
|
1383
|
+
}
|
|
1384
|
+
if (goal) {
|
|
1385
|
+
lines.push(`Worker goal: ${goal.text}`);
|
|
1386
|
+
if (goal.status === "completed" && goal.evidence) {
|
|
1387
|
+
lines.push(`Completion evidence: ${goal.evidence}`);
|
|
1388
|
+
}
|
|
1389
|
+
if (goal.status === "paused" && goal.rationale) {
|
|
1390
|
+
lines.push(`Pause rationale: ${goal.rationale}`);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
return lines.join("\n");
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
// src/sandbox-resume.ts
|
|
1397
|
+
import {
|
|
1398
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
1399
|
+
sandboxLifecycleTransitionWaitMs
|
|
1400
|
+
} from "@opengeni/config";
|
|
1401
|
+
import { randomUUID } from "crypto";
|
|
1402
|
+
import {
|
|
1403
|
+
acquireLease,
|
|
1404
|
+
adoptLegacyModalCheckpointArtifact,
|
|
1405
|
+
beginSandboxRematerialization,
|
|
1406
|
+
claimWorkspaceArchiveCapture,
|
|
1407
|
+
commitWarmingToWarm,
|
|
1408
|
+
failSandboxRematerialization,
|
|
1409
|
+
failWarmingToCold,
|
|
1410
|
+
getSandboxSessionEnvelope,
|
|
1411
|
+
heartbeatLeaseHolder,
|
|
1412
|
+
markSandboxRestoreVerifying,
|
|
1413
|
+
markWarmLeaseInstanceLost,
|
|
1414
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
1415
|
+
persistWarmSnapshot,
|
|
1416
|
+
readLease,
|
|
1417
|
+
registerSandboxCheckpointArtifact,
|
|
1418
|
+
recordWarmingSandboxCreated,
|
|
1419
|
+
releaseWorkspaceArchiveCapture,
|
|
1420
|
+
releaseLeaseHolder,
|
|
1421
|
+
touchLeaseHolder,
|
|
1422
|
+
SandboxLeaseRecoveryBlockedError,
|
|
1423
|
+
SandboxLeaseSupersededError,
|
|
1424
|
+
SandboxLeaseTransitionError
|
|
1425
|
+
} from "@opengeni/db";
|
|
1426
|
+
import {
|
|
1427
|
+
captureVerifiedWorkspaceArchive,
|
|
1428
|
+
describeLegacyNativeSnapshotArchive,
|
|
1429
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
1430
|
+
SandboxExecReadinessError,
|
|
1431
|
+
WorkspaceArchiveIntegrityError,
|
|
1432
|
+
establishSandboxSessionFromEnvelope,
|
|
1433
|
+
isProviderSandboxNotFoundError,
|
|
1434
|
+
parseWorkspaceArchiveDescriptor,
|
|
1435
|
+
providerWorkspaceCapturePolicy,
|
|
1436
|
+
SandboxProviderContinuityUnavailableError,
|
|
1437
|
+
requirePersistableReplacementSandboxEnvelope,
|
|
1438
|
+
modalSessionMatchesCheckpointProviderBinding,
|
|
1439
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
1440
|
+
serializeReplacementSandboxEnvelope,
|
|
1441
|
+
tagModalSandbox,
|
|
1442
|
+
terminateUnpublishedSandboxSession,
|
|
1443
|
+
verifySandboxExecReadiness,
|
|
1444
|
+
withoutSandboxProviderIdentity
|
|
1445
|
+
} from "@opengeni/runtime";
|
|
1446
|
+
function sandboxLeaseHolderIdForAttempt(attemptId) {
|
|
1447
|
+
const normalized = attemptId.trim();
|
|
1448
|
+
if (!normalized) {
|
|
1449
|
+
throw new Error("Sandbox lease holder requires a turn attempt id");
|
|
1450
|
+
}
|
|
1451
|
+
return `turn-attempt:${normalized}`;
|
|
1452
|
+
}
|
|
1453
|
+
function isRetryableDegradedRestore(restore) {
|
|
1454
|
+
return restore.status === "degraded" && restore.retryable === true;
|
|
1455
|
+
}
|
|
1456
|
+
var SandboxWarmingTimeoutError = class extends Error {
|
|
1457
|
+
constructor(backend, timeoutMs, stage, sandboxGroupId, instanceId) {
|
|
1458
|
+
const timeoutSeconds = Math.ceil(timeoutMs / 1e3);
|
|
1459
|
+
const identity = [
|
|
1460
|
+
sandboxGroupId ? `group ${sandboxGroupId}` : null,
|
|
1461
|
+
instanceId ? `sandbox ${instanceId}` : null
|
|
1462
|
+
].filter((part) => part !== null).join(", ");
|
|
1463
|
+
const target = identity ? ` (${identity})` : "";
|
|
1464
|
+
super(
|
|
1465
|
+
stage === "exec_readiness" ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.` : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`
|
|
1466
|
+
);
|
|
1467
|
+
this.backend = backend;
|
|
1468
|
+
this.timeoutMs = timeoutMs;
|
|
1469
|
+
this.stage = stage;
|
|
1470
|
+
this.sandboxGroupId = sandboxGroupId;
|
|
1471
|
+
this.instanceId = instanceId;
|
|
1472
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
1473
|
+
}
|
|
1474
|
+
};
|
|
1475
|
+
var SandboxExecReadinessTimeoutError = class extends SandboxWarmingTimeoutError {
|
|
1476
|
+
code = "sandbox_exec_readiness_timeout";
|
|
1477
|
+
constructor(backend, timeoutMs, identity = {}) {
|
|
1478
|
+
super(
|
|
1479
|
+
backend,
|
|
1480
|
+
timeoutMs,
|
|
1481
|
+
"exec_readiness",
|
|
1482
|
+
identity.sandboxGroupId ?? null,
|
|
1483
|
+
identity.instanceId ?? null
|
|
1484
|
+
);
|
|
1485
|
+
this.name = "SandboxExecReadinessTimeoutError";
|
|
1486
|
+
}
|
|
1487
|
+
};
|
|
1488
|
+
var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError {
|
|
1489
|
+
code = "sandbox_sibling_warming_timeout";
|
|
1490
|
+
constructor(backend, timeoutMs, identity) {
|
|
1491
|
+
super(
|
|
1492
|
+
backend,
|
|
1493
|
+
timeoutMs,
|
|
1494
|
+
"sibling_warming",
|
|
1495
|
+
identity.sandboxGroupId,
|
|
1496
|
+
identity.instanceId ?? null
|
|
1497
|
+
);
|
|
1498
|
+
this.name = "SandboxSiblingWarmingTimeoutError";
|
|
1499
|
+
}
|
|
1500
|
+
};
|
|
1501
|
+
var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
|
|
1502
|
+
constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
|
|
1503
|
+
super(sandboxGroupId, leaseEpoch);
|
|
1504
|
+
this.lostInstanceId = lostInstanceId;
|
|
1505
|
+
this.name = "SandboxLeaseInstanceLostError";
|
|
1506
|
+
}
|
|
1507
|
+
};
|
|
1508
|
+
var WARMING_POLL_INTERVAL_MS = 250;
|
|
1509
|
+
async function sleep(ms) {
|
|
1510
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
1511
|
+
}
|
|
1512
|
+
async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}) {
|
|
1513
|
+
try {
|
|
1514
|
+
await verifySandboxExecReadiness(established, timeoutMs);
|
|
1515
|
+
} catch (error) {
|
|
1516
|
+
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
1517
|
+
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
1518
|
+
sandboxGroupId: identity.sandboxGroupId ?? null,
|
|
1519
|
+
instanceId: established.instanceId
|
|
1520
|
+
});
|
|
1521
|
+
}
|
|
1522
|
+
throw error;
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
var SnapshotTimeoutError = class extends Error {
|
|
1526
|
+
constructor(timeoutMs) {
|
|
1527
|
+
super(`workspace snapshot timed out after ${timeoutMs}ms`);
|
|
1528
|
+
this.name = "SnapshotTimeoutError";
|
|
1529
|
+
}
|
|
1530
|
+
};
|
|
1531
|
+
function safeSnapshotError(error) {
|
|
1532
|
+
const fields = {
|
|
1533
|
+
errorClass: "SnapshotOperationError",
|
|
1534
|
+
errorCode: "snapshot_operation_failed",
|
|
1535
|
+
origin: "sandbox-resume"
|
|
1536
|
+
};
|
|
1537
|
+
try {
|
|
1538
|
+
const rawStatus = error && typeof error === "object" ? error.status ?? error.statusCode : void 0;
|
|
1539
|
+
const status = Number(rawStatus);
|
|
1540
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
1541
|
+
} catch {
|
|
1542
|
+
}
|
|
1543
|
+
return fields;
|
|
1544
|
+
}
|
|
1545
|
+
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
1546
|
+
if (signal?.aborted) return false;
|
|
1547
|
+
let timeout;
|
|
1548
|
+
let cancelListener;
|
|
1549
|
+
try {
|
|
1550
|
+
await Promise.race([
|
|
1551
|
+
snapshot,
|
|
1552
|
+
...signal ? [
|
|
1553
|
+
new Promise((_resolve, reject) => {
|
|
1554
|
+
cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
|
|
1555
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
1556
|
+
if (signal.aborted) cancelListener();
|
|
1557
|
+
})
|
|
1558
|
+
] : [],
|
|
1559
|
+
new Promise((_, reject) => {
|
|
1560
|
+
timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
|
|
1561
|
+
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
1562
|
+
timeout.unref();
|
|
1563
|
+
}
|
|
1564
|
+
})
|
|
1565
|
+
]);
|
|
1566
|
+
return true;
|
|
1567
|
+
} catch (error) {
|
|
1568
|
+
if (signal?.aborted) return false;
|
|
1569
|
+
if (error instanceof SnapshotTimeoutError) {
|
|
1570
|
+
console.error(
|
|
1571
|
+
"mid-session workspace snapshot wait timed out (turn unaffected)",
|
|
1572
|
+
safeSnapshotError(error)
|
|
1573
|
+
);
|
|
1574
|
+
return false;
|
|
1575
|
+
}
|
|
1576
|
+
return true;
|
|
1577
|
+
} finally {
|
|
1578
|
+
if (timeout) {
|
|
1579
|
+
clearTimeout(timeout);
|
|
1580
|
+
}
|
|
1581
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
async function resolveModalCheckpointBindingBeforeCapture(settings, session, signal) {
|
|
1585
|
+
if (signal?.aborted) {
|
|
1586
|
+
throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
|
|
1587
|
+
}
|
|
1588
|
+
const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
|
|
1589
|
+
let timeout;
|
|
1590
|
+
let cancelListener;
|
|
1591
|
+
try {
|
|
1592
|
+
return await Promise.race([
|
|
1593
|
+
pending,
|
|
1594
|
+
...signal ? [
|
|
1595
|
+
new Promise((_resolve, reject) => {
|
|
1596
|
+
cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
|
|
1597
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
1598
|
+
if (signal.aborted) cancelListener();
|
|
1599
|
+
})
|
|
1600
|
+
] : [],
|
|
1601
|
+
new Promise((_resolve, reject) => {
|
|
1602
|
+
timeout = setTimeout(
|
|
1603
|
+
() => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
|
|
1604
|
+
settings.sandboxSnapshotTimeoutMs
|
|
1605
|
+
);
|
|
1606
|
+
timeout.unref?.();
|
|
1607
|
+
})
|
|
1608
|
+
]);
|
|
1609
|
+
} finally {
|
|
1610
|
+
if (timeout) clearTimeout(timeout);
|
|
1611
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
1614
|
+
async function terminateEstablishedSandbox(established) {
|
|
1615
|
+
if (!established) return true;
|
|
1616
|
+
try {
|
|
1617
|
+
await terminateUnpublishedSandboxSession(established);
|
|
1618
|
+
return true;
|
|
1619
|
+
} catch (error) {
|
|
1620
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
1621
|
+
return true;
|
|
1622
|
+
}
|
|
1623
|
+
return false;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
function recordSandboxWarmingTimeout(metrics, error) {
|
|
1627
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
1628
|
+
return;
|
|
1629
|
+
}
|
|
1630
|
+
try {
|
|
1631
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
|
|
1632
|
+
} catch {
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
function workspaceArchiveFieldsFromEnvelope(envelope) {
|
|
1636
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1637
|
+
const archive = sessionState?.workspaceArchive;
|
|
1638
|
+
if (typeof archive !== "string" || archive.length === 0) {
|
|
1639
|
+
return null;
|
|
1640
|
+
}
|
|
1641
|
+
const previous = sessionState?.workspaceArchivePrev;
|
|
1642
|
+
const metadata = sessionState?.workspaceArchiveMeta;
|
|
1643
|
+
const previousMetadata = sessionState?.workspaceArchivePrevMeta;
|
|
1644
|
+
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
1645
|
+
return {
|
|
1646
|
+
workspaceArchive: archive,
|
|
1647
|
+
...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
|
|
1648
|
+
...typeof previous === "string" && previous.length > 0 ? { workspaceArchivePrev: previous } : {},
|
|
1649
|
+
...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
|
|
1650
|
+
...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
|
|
1651
|
+
};
|
|
1652
|
+
}
|
|
1653
|
+
function legacyNativeArchiveFromEnvelope(envelope) {
|
|
1654
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1655
|
+
if (!sessionState) return null;
|
|
1656
|
+
const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
1657
|
+
if (existing?.version === 2) return null;
|
|
1658
|
+
return describeLegacyNativeSnapshotArchive(
|
|
1659
|
+
sessionState.workspaceArchive,
|
|
1660
|
+
existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
|
|
1661
|
+
);
|
|
1662
|
+
}
|
|
1663
|
+
async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
|
|
1664
|
+
const { db, settings } = services;
|
|
1665
|
+
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
1666
|
+
if (intervalMs <= 0 && !force) {
|
|
1667
|
+
return false;
|
|
1668
|
+
}
|
|
1669
|
+
const persistable = session;
|
|
1670
|
+
if (typeof persistable.persistWorkspace !== "function") {
|
|
1671
|
+
return false;
|
|
1672
|
+
}
|
|
1673
|
+
const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
|
|
1674
|
+
if (signal?.aborted) {
|
|
1675
|
+
return false;
|
|
1676
|
+
}
|
|
1677
|
+
try {
|
|
1678
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1679
|
+
if (!lease || lease.leaseEpoch !== leaseEpoch || lease.liveness !== "warm" || lease.instanceId === null) {
|
|
1680
|
+
return false;
|
|
1681
|
+
}
|
|
1682
|
+
if (lease.archiveComplete) {
|
|
1683
|
+
return false;
|
|
1684
|
+
}
|
|
1685
|
+
const instanceId = lease.instanceId;
|
|
1686
|
+
const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
|
|
1687
|
+
if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
|
|
1688
|
+
return false;
|
|
1689
|
+
}
|
|
1690
|
+
const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
|
|
1691
|
+
const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
|
|
1692
|
+
if (signal?.aborted) return false;
|
|
1693
|
+
const captureId = randomUUID();
|
|
1694
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
|
|
1695
|
+
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
1696
|
+
accountId: ids.accountId,
|
|
1697
|
+
workspaceId: ids.workspaceId,
|
|
1698
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1699
|
+
captureId,
|
|
1700
|
+
expectedEpoch: leaseEpoch,
|
|
1701
|
+
expectedInstanceId: instanceId,
|
|
1702
|
+
liveness: "warm",
|
|
1703
|
+
captureTimeoutMs,
|
|
1704
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
1705
|
+
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
1706
|
+
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
1707
|
+
warmAttempt: {
|
|
1708
|
+
sessionId: ids.sessionId,
|
|
1709
|
+
turnId: ids.turnId,
|
|
1710
|
+
attemptId: ids.attemptId,
|
|
1711
|
+
holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId)
|
|
1712
|
+
}
|
|
1713
|
+
});
|
|
1714
|
+
if (claimed.status !== "claimed") return false;
|
|
1715
|
+
let timeout;
|
|
1716
|
+
const capturedAtMs = Date.now();
|
|
1717
|
+
const registerCandidate = async (archive) => {
|
|
1718
|
+
if (archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
|
|
1719
|
+
return null;
|
|
1720
|
+
}
|
|
1721
|
+
if (!checkpointBinding) {
|
|
1722
|
+
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
1723
|
+
}
|
|
1724
|
+
return await registerSandboxCheckpointArtifact(db, {
|
|
1725
|
+
accountId: ids.accountId,
|
|
1726
|
+
workspaceId: ids.workspaceId,
|
|
1727
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1728
|
+
sourceLeaseId: claimed.claim.leaseId,
|
|
1729
|
+
sourceLeaseEpoch: leaseEpoch,
|
|
1730
|
+
sourceInstanceId: instanceId,
|
|
1731
|
+
sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
1732
|
+
providerBindingKey: checkpointBinding.key,
|
|
1733
|
+
providerBinding: checkpointBinding.binding,
|
|
1734
|
+
workspaceArchive: archive.base64,
|
|
1735
|
+
workspaceArchiveMeta: archive.descriptor
|
|
1736
|
+
});
|
|
1737
|
+
};
|
|
1738
|
+
const abandonCandidate = async (artifactId, reason) => {
|
|
1739
|
+
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
1740
|
+
accountId: ids.accountId,
|
|
1741
|
+
workspaceId: ids.workspaceId,
|
|
1742
|
+
artifactId,
|
|
1743
|
+
reason
|
|
1744
|
+
}).catch(() => void 0);
|
|
1745
|
+
};
|
|
1746
|
+
const captureAndPublish = (async () => {
|
|
1747
|
+
let candidate = null;
|
|
1748
|
+
try {
|
|
1749
|
+
const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
|
|
1750
|
+
requestId: claimed.claim.providerRequestId,
|
|
1751
|
+
strategy: capturePolicy.strategy
|
|
1752
|
+
});
|
|
1753
|
+
candidate = await registerCandidate(archive);
|
|
1754
|
+
const { wrote } = await persistWarmSnapshot(db, {
|
|
1755
|
+
accountId: ids.accountId,
|
|
1756
|
+
workspaceId: ids.workspaceId,
|
|
1757
|
+
sessionId: ids.sessionId,
|
|
1758
|
+
turnId: ids.turnId,
|
|
1759
|
+
attemptId: ids.attemptId,
|
|
1760
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1761
|
+
expectedEpoch: leaseEpoch,
|
|
1762
|
+
expectedInstanceId: instanceId,
|
|
1763
|
+
expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
1764
|
+
captureId,
|
|
1765
|
+
workspaceArchive: archive.base64,
|
|
1766
|
+
workspaceArchiveMeta: archive.descriptor,
|
|
1767
|
+
checkpointArtifactId: candidate?.id ?? null,
|
|
1768
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
1769
|
+
capturedAtMs
|
|
1770
|
+
});
|
|
1771
|
+
if (!wrote && candidate) {
|
|
1772
|
+
await abandonCandidate(candidate.id, "snapshot_publication_fenced");
|
|
1773
|
+
}
|
|
1774
|
+
return wrote;
|
|
1775
|
+
} catch (error) {
|
|
1776
|
+
if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
|
|
1777
|
+
throw error;
|
|
1778
|
+
} finally {
|
|
1779
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
1780
|
+
accountId: ids.accountId,
|
|
1781
|
+
workspaceId: ids.workspaceId,
|
|
1782
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1783
|
+
captureId,
|
|
1784
|
+
expectedEpoch: leaseEpoch,
|
|
1785
|
+
expectedInstanceId: instanceId
|
|
1786
|
+
}).catch((error) => {
|
|
1787
|
+
console.error(
|
|
1788
|
+
"mid-session workspace capture gate release failed",
|
|
1789
|
+
safeSnapshotError(error)
|
|
1790
|
+
);
|
|
1791
|
+
});
|
|
1792
|
+
}
|
|
1793
|
+
})();
|
|
1794
|
+
const settled = captureAndPublish.then(
|
|
1795
|
+
(persisted) => ({ kind: "settled", persisted }),
|
|
1796
|
+
(error) => {
|
|
1797
|
+
console.error(
|
|
1798
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
1799
|
+
safeSnapshotError(error)
|
|
1800
|
+
);
|
|
1801
|
+
return { kind: "settled", persisted: false };
|
|
1802
|
+
}
|
|
1803
|
+
);
|
|
1804
|
+
let cancelListener;
|
|
1805
|
+
const outcome = await Promise.race([
|
|
1806
|
+
settled,
|
|
1807
|
+
...signal ? [
|
|
1808
|
+
new Promise((resolve) => {
|
|
1809
|
+
cancelListener = () => resolve({ kind: "cancelled" });
|
|
1810
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
1811
|
+
if (signal.aborted) cancelListener();
|
|
1812
|
+
})
|
|
1813
|
+
] : [],
|
|
1814
|
+
new Promise((resolve) => {
|
|
1815
|
+
timeout = setTimeout(
|
|
1816
|
+
() => resolve({ kind: "timed_out" }),
|
|
1817
|
+
settings.sandboxSnapshotTimeoutMs
|
|
1818
|
+
);
|
|
1819
|
+
})
|
|
1820
|
+
]).finally(() => {
|
|
1821
|
+
if (timeout) clearTimeout(timeout);
|
|
1822
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
1823
|
+
});
|
|
1824
|
+
return outcome.kind === "settled" ? outcome.persisted : false;
|
|
1825
|
+
} catch (error) {
|
|
1826
|
+
console.error(
|
|
1827
|
+
"mid-session workspace snapshot failed (turn unaffected)",
|
|
1828
|
+
safeSnapshotError(error)
|
|
1829
|
+
);
|
|
1830
|
+
return false;
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
1834
|
+
const { db, settings } = services;
|
|
1835
|
+
const os = ids.os ?? "linux";
|
|
1836
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
1837
|
+
const cancellationSignal = services.cancellationSignal;
|
|
1838
|
+
let releaseStarted = false;
|
|
1839
|
+
let holderRelease = null;
|
|
1840
|
+
let quiescedRelease = null;
|
|
1841
|
+
let holderLivenessTimer;
|
|
1842
|
+
let cancellationListener;
|
|
1843
|
+
const release = (options) => {
|
|
1844
|
+
const writersQuiesced = options?.workspaceWritersQuiesced === true;
|
|
1845
|
+
if (!releaseStarted) {
|
|
1846
|
+
releaseStarted = true;
|
|
1847
|
+
if (cancellationListener) {
|
|
1848
|
+
cancellationSignal?.removeEventListener("abort", cancellationListener);
|
|
1849
|
+
cancellationListener = void 0;
|
|
1850
|
+
}
|
|
1851
|
+
if (holderLivenessTimer) {
|
|
1852
|
+
clearInterval(holderLivenessTimer);
|
|
1853
|
+
holderLivenessTimer = void 0;
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
const persistRelease = async (workspaceWritersQuiesced) => {
|
|
1857
|
+
await releaseLeaseHolder(db, {
|
|
1858
|
+
accountId: ids.accountId,
|
|
1859
|
+
workspaceId: ids.workspaceId,
|
|
1860
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1861
|
+
kind,
|
|
1862
|
+
holderId,
|
|
1863
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
1864
|
+
...workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}
|
|
1865
|
+
});
|
|
1866
|
+
};
|
|
1867
|
+
if (writersQuiesced) {
|
|
1868
|
+
if (!quiescedRelease) {
|
|
1869
|
+
const predecessor = holderRelease;
|
|
1870
|
+
const attempt = (predecessor ? predecessor.catch(() => void 0) : Promise.resolve()).then(
|
|
1871
|
+
async () => await persistRelease(true)
|
|
1872
|
+
);
|
|
1873
|
+
const tracked = attempt.catch((error) => {
|
|
1874
|
+
if (quiescedRelease === tracked) quiescedRelease = null;
|
|
1875
|
+
throw error;
|
|
1876
|
+
});
|
|
1877
|
+
quiescedRelease = tracked;
|
|
1878
|
+
holderRelease = tracked;
|
|
1879
|
+
}
|
|
1880
|
+
return quiescedRelease;
|
|
1881
|
+
}
|
|
1882
|
+
if (!holderRelease) {
|
|
1883
|
+
const attempt = persistRelease(false);
|
|
1884
|
+
const tracked = attempt.catch((error) => {
|
|
1885
|
+
if (holderRelease === tracked) holderRelease = null;
|
|
1886
|
+
throw error;
|
|
1887
|
+
});
|
|
1888
|
+
holderRelease = tracked;
|
|
1889
|
+
}
|
|
1890
|
+
return holderRelease;
|
|
1891
|
+
};
|
|
1892
|
+
const cancellationError = () => cancellationSignal?.reason instanceof Error ? cancellationSignal.reason : new Error("Sandbox resume was cancelled with its owning turn attempt");
|
|
1893
|
+
const throwIfReleasedOrCancelled = () => {
|
|
1894
|
+
if (releaseStarted || cancellationSignal?.aborted) {
|
|
1895
|
+
throw cancellationError();
|
|
1896
|
+
}
|
|
1897
|
+
};
|
|
1898
|
+
const acquired = await acquireLease(db, {
|
|
1899
|
+
accountId: ids.accountId,
|
|
1900
|
+
workspaceId: ids.workspaceId,
|
|
1901
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1902
|
+
kind,
|
|
1903
|
+
holderId,
|
|
1904
|
+
subjectId: ids.sessionId,
|
|
1905
|
+
backend: ids.backend,
|
|
1906
|
+
os,
|
|
1907
|
+
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
1908
|
+
// conflicts on a live box already running a different image. A
|
|
1909
|
+
// SandboxImageConflictError propagates while another holder is active; a
|
|
1910
|
+
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
1911
|
+
// after the cold successor can safely stamp the new image.
|
|
1912
|
+
...ids.image ? { image: ids.image } : {},
|
|
1913
|
+
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
1914
|
+
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
1915
|
+
// propagates while another holder is active; a solo change uses the same
|
|
1916
|
+
// durable capture-and-drain rotation as an image change.
|
|
1917
|
+
...ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {},
|
|
1918
|
+
leaseTtlMs,
|
|
1919
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
1920
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
1921
|
+
...cancellationSignal ? { waitSignal: cancellationSignal } : {}
|
|
1922
|
+
});
|
|
1923
|
+
if (cancellationSignal) {
|
|
1924
|
+
cancellationListener = () => {
|
|
1925
|
+
void release().catch(() => void 0);
|
|
1926
|
+
};
|
|
1927
|
+
cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
|
|
1928
|
+
}
|
|
1929
|
+
if (cancellationSignal?.aborted) {
|
|
1930
|
+
await release();
|
|
1931
|
+
throw cancellationError();
|
|
1932
|
+
}
|
|
1933
|
+
let holderLeaseHeartbeat = acquired.role === "spawner" ? {
|
|
1934
|
+
expectedEpoch: acquired.lease.leaseEpoch,
|
|
1935
|
+
leaseTtlMs: settings.sandboxLeaseWarmingTtlMs
|
|
1936
|
+
} : (acquired.role === "attached" || acquired.role === "rearmed") && acquired.lease.liveness === "warm" ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs } : null;
|
|
1937
|
+
holderLivenessTimer = setInterval(() => {
|
|
1938
|
+
const heartbeat = holderLeaseHeartbeat;
|
|
1939
|
+
const refresh = heartbeat ? heartbeatLeaseHolder(db, {
|
|
1940
|
+
accountId: ids.accountId,
|
|
1941
|
+
workspaceId: ids.workspaceId,
|
|
1942
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1943
|
+
kind,
|
|
1944
|
+
holderId,
|
|
1945
|
+
expectedEpoch: heartbeat.expectedEpoch,
|
|
1946
|
+
leaseTtlMs: heartbeat.leaseTtlMs
|
|
1947
|
+
}) : touchLeaseHolder(db, {
|
|
1948
|
+
accountId: ids.accountId,
|
|
1949
|
+
workspaceId: ids.workspaceId,
|
|
1950
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1951
|
+
kind,
|
|
1952
|
+
holderId
|
|
1953
|
+
});
|
|
1954
|
+
void refresh.then((touched) => {
|
|
1955
|
+
if (!touched && heartbeat === holderLeaseHeartbeat) {
|
|
1956
|
+
void release().catch(() => void 0);
|
|
1957
|
+
}
|
|
1958
|
+
}).catch(() => void 0);
|
|
1959
|
+
}, 1e4);
|
|
1960
|
+
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
1961
|
+
holderLivenessTimer.unref();
|
|
1962
|
+
}
|
|
1963
|
+
if (acquired.role === "blocked") {
|
|
1964
|
+
await release();
|
|
1965
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1966
|
+
ids.sandboxGroupId,
|
|
1967
|
+
acquired.lease.leaseEpoch,
|
|
1968
|
+
acquired.code,
|
|
1969
|
+
acquired.lease.recovery
|
|
1970
|
+
);
|
|
1971
|
+
}
|
|
1972
|
+
if (acquired.role === "fenced") {
|
|
1973
|
+
await release();
|
|
1974
|
+
if (acquired.reason !== "superseded") {
|
|
1975
|
+
throw new SandboxLeaseTransitionError(
|
|
1976
|
+
ids.sandboxGroupId,
|
|
1977
|
+
acquired.lease.leaseEpoch,
|
|
1978
|
+
acquired.reason,
|
|
1979
|
+
acquired.lease.backend,
|
|
1980
|
+
acquired.lease.instanceId,
|
|
1981
|
+
acquired.lease.liveness
|
|
1982
|
+
);
|
|
1983
|
+
}
|
|
1984
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
|
|
1985
|
+
backend: acquired.lease.backend,
|
|
1986
|
+
instanceId: acquired.lease.instanceId,
|
|
1987
|
+
liveness: acquired.lease.liveness
|
|
1988
|
+
});
|
|
1989
|
+
}
|
|
1990
|
+
if (acquired.role === "spawner") {
|
|
1991
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1992
|
+
let createdEstablished = null;
|
|
1993
|
+
let rematerialization = null;
|
|
1994
|
+
try {
|
|
1995
|
+
const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
1996
|
+
const fallbackArchiveEnvelope = acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
|
|
1997
|
+
let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
1998
|
+
const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
|
|
1999
|
+
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
2000
|
+
if (acquired.lease.recovery.archive.status === "available" && acquired.lease.archiveComplete || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null) {
|
|
2001
|
+
const rematerializationId = crypto.randomUUID();
|
|
2002
|
+
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
2003
|
+
const begun = await beginSandboxRematerialization(db, {
|
|
2004
|
+
accountId: ids.accountId,
|
|
2005
|
+
workspaceId: ids.workspaceId,
|
|
2006
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2007
|
+
expectedEpoch,
|
|
2008
|
+
rematerializationId,
|
|
2009
|
+
archiveSource,
|
|
2010
|
+
legacyNativeArchive
|
|
2011
|
+
});
|
|
2012
|
+
if (begun.status !== "started") {
|
|
2013
|
+
if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
|
|
2014
|
+
throw new SandboxLeaseSupersededError(
|
|
2015
|
+
ids.sandboxGroupId,
|
|
2016
|
+
begun.lease?.leaseEpoch ?? expectedEpoch
|
|
2017
|
+
);
|
|
2018
|
+
}
|
|
2019
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
2020
|
+
ids.sandboxGroupId,
|
|
2021
|
+
begun.lease?.leaseEpoch ?? expectedEpoch,
|
|
2022
|
+
begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
|
|
2023
|
+
begun.lease?.recovery ?? acquired.lease.recovery
|
|
2024
|
+
);
|
|
2025
|
+
}
|
|
2026
|
+
spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
|
|
2027
|
+
const selectedRevision = begun.lease.recovery.restore.selectedRevision;
|
|
2028
|
+
if (!selectedRevision) {
|
|
2029
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2030
|
+
"archive_metadata_invalid",
|
|
2031
|
+
"sandbox rematerialization selected no durable archive revision"
|
|
2032
|
+
);
|
|
2033
|
+
}
|
|
2034
|
+
rematerialization = {
|
|
2035
|
+
id: rematerializationId,
|
|
2036
|
+
selectedRevision,
|
|
2037
|
+
workspaceGeneration: begun.lease.workspaceGeneration,
|
|
2038
|
+
providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
|
|
2039
|
+
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
2040
|
+
legacyProviderBinding: null
|
|
2041
|
+
};
|
|
2042
|
+
} else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
2043
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
2044
|
+
ids.sandboxGroupId,
|
|
2045
|
+
expectedEpoch,
|
|
2046
|
+
"restore_degraded",
|
|
2047
|
+
acquired.lease.recovery
|
|
2048
|
+
);
|
|
2049
|
+
}
|
|
2050
|
+
const providerCreateStartedAt = /* @__PURE__ */ new Date();
|
|
2051
|
+
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
2052
|
+
sessionId: ids.sessionId,
|
|
2053
|
+
recovery: "create-or-restore",
|
|
2054
|
+
backendOverride: ids.backend,
|
|
2055
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
2056
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
2057
|
+
...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
|
|
2058
|
+
onSandboxCreated: async (created) => {
|
|
2059
|
+
createdEstablished = created;
|
|
2060
|
+
throwIfReleasedOrCancelled();
|
|
2061
|
+
if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
|
|
2062
|
+
if (created.backendId !== "modal") {
|
|
2063
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2064
|
+
"native_snapshot_reference_invalid",
|
|
2065
|
+
"Modal checkpoint restore resolved a non-Modal sandbox backend"
|
|
2066
|
+
);
|
|
2067
|
+
}
|
|
2068
|
+
if (rematerialization.providerBindingKey) {
|
|
2069
|
+
if (!await modalSessionMatchesCheckpointProviderBinding(
|
|
2070
|
+
settings,
|
|
2071
|
+
created.session,
|
|
2072
|
+
rematerialization.providerBindingKey
|
|
2073
|
+
)) {
|
|
2074
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2075
|
+
"native_snapshot_reference_invalid",
|
|
2076
|
+
"Modal checkpoint restore refused because the authenticated provider workspace changed"
|
|
2077
|
+
);
|
|
2078
|
+
}
|
|
2079
|
+
} else {
|
|
2080
|
+
const resolved = await resolveModalCheckpointProviderBindingForSession(
|
|
2081
|
+
settings,
|
|
2082
|
+
created.session
|
|
2083
|
+
);
|
|
2084
|
+
if (rematerialization.legacyProviderBinding && rematerialization.legacyProviderBinding.key !== resolved.key) {
|
|
2085
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2086
|
+
"native_snapshot_reference_invalid",
|
|
2087
|
+
"Legacy Modal checkpoint restore crossed authenticated provider workspaces"
|
|
2088
|
+
);
|
|
2089
|
+
}
|
|
2090
|
+
rematerialization.legacyProviderBinding = resolved;
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
|
|
2094
|
+
await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
|
|
2095
|
+
created.backendId
|
|
2096
|
+
);
|
|
2097
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
2098
|
+
accountId: ids.accountId,
|
|
2099
|
+
workspaceId: ids.workspaceId,
|
|
2100
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2101
|
+
expectedEpoch,
|
|
2102
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
2103
|
+
...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
|
|
2104
|
+
instanceId: created.instanceId,
|
|
2105
|
+
resumeBackendId: created.backendId,
|
|
2106
|
+
resumeState: resumeEnvelope2,
|
|
2107
|
+
...created.backendId === "modal" ? {
|
|
2108
|
+
providerCreatedAt: providerCreateStartedAt,
|
|
2109
|
+
providerDeadlineAt: new Date(
|
|
2110
|
+
providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1e3
|
|
2111
|
+
)
|
|
2112
|
+
} : {},
|
|
2113
|
+
leaseTtlMs,
|
|
2114
|
+
// Keep the warming budget after create(): manifest setup and
|
|
2115
|
+
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
2116
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs
|
|
2117
|
+
});
|
|
2118
|
+
if (!recorded.recorded) {
|
|
2119
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
2120
|
+
}
|
|
2121
|
+
if (created.backendId === "modal") {
|
|
2122
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
2123
|
+
leaseId: acquired.lease.id,
|
|
2124
|
+
workspaceId: ids.workspaceId,
|
|
2125
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
2126
|
+
}).catch(() => void 0);
|
|
2127
|
+
}
|
|
2128
|
+
throwIfReleasedOrCancelled();
|
|
2129
|
+
},
|
|
2130
|
+
onWorkspaceRestoreVerifying: async (descriptor) => {
|
|
2131
|
+
if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
|
|
2132
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2133
|
+
"archive_metadata_invalid",
|
|
2134
|
+
`hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
|
|
2135
|
+
);
|
|
2136
|
+
}
|
|
2137
|
+
if (rematerialization.legacyCheckpoint) {
|
|
2138
|
+
const binding = rematerialization.legacyProviderBinding;
|
|
2139
|
+
if (!binding) {
|
|
2140
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2141
|
+
"native_snapshot_reference_invalid",
|
|
2142
|
+
"Legacy Modal checkpoint restore produced no authenticated provider identity"
|
|
2143
|
+
);
|
|
2144
|
+
}
|
|
2145
|
+
const adopted = await adoptLegacyModalCheckpointArtifact(db, {
|
|
2146
|
+
accountId: ids.accountId,
|
|
2147
|
+
workspaceId: ids.workspaceId,
|
|
2148
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2149
|
+
leaseId: acquired.lease.id,
|
|
2150
|
+
leaseEpoch: expectedEpoch,
|
|
2151
|
+
workspaceGeneration: rematerialization.workspaceGeneration,
|
|
2152
|
+
slot: "current",
|
|
2153
|
+
archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
|
|
2154
|
+
descriptor: rematerialization.legacyCheckpoint.descriptor,
|
|
2155
|
+
providerBindingKey: binding.key,
|
|
2156
|
+
providerBinding: binding.binding,
|
|
2157
|
+
rematerializationId: rematerialization.id
|
|
2158
|
+
});
|
|
2159
|
+
if (!adopted) {
|
|
2160
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
2161
|
+
}
|
|
2162
|
+
rematerialization.providerBindingKey = binding.key;
|
|
2163
|
+
}
|
|
2164
|
+
const verifying = await markSandboxRestoreVerifying(db, {
|
|
2165
|
+
accountId: ids.accountId,
|
|
2166
|
+
workspaceId: ids.workspaceId,
|
|
2167
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2168
|
+
expectedEpoch,
|
|
2169
|
+
rematerializationId: rematerialization.id
|
|
2170
|
+
});
|
|
2171
|
+
if (!verifying.wrote) {
|
|
2172
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
2173
|
+
}
|
|
2174
|
+
}
|
|
2175
|
+
});
|
|
2176
|
+
createdEstablished = established;
|
|
2177
|
+
throwIfReleasedOrCancelled();
|
|
2178
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
2179
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
2180
|
+
});
|
|
2181
|
+
throwIfReleasedOrCancelled();
|
|
2182
|
+
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
2183
|
+
await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
|
|
2184
|
+
established.backendId
|
|
2185
|
+
);
|
|
2186
|
+
throwIfReleasedOrCancelled();
|
|
2187
|
+
if (rematerialization && !established.providerContinuity && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
|
|
2188
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
2189
|
+
"workspace_fingerprint_mismatch",
|
|
2190
|
+
"sandbox restore completed without the exact selected durable archive revision"
|
|
2191
|
+
);
|
|
2192
|
+
}
|
|
2193
|
+
const committed = await commitWarmingToWarm(db, {
|
|
2194
|
+
accountId: ids.accountId,
|
|
2195
|
+
workspaceId: ids.workspaceId,
|
|
2196
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2197
|
+
expectedEpoch,
|
|
2198
|
+
instanceId: established.instanceId,
|
|
2199
|
+
// Viewer attach resolves and records the tunnel URL lazily. A headless
|
|
2200
|
+
// turn must not touch the optional desktop data plane.
|
|
2201
|
+
dataPlaneUrl: null,
|
|
2202
|
+
resumeBackendId: established.backendId,
|
|
2203
|
+
resumeState: resumeEnvelope,
|
|
2204
|
+
...established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
|
|
2205
|
+
rematerialization: {
|
|
2206
|
+
id: rematerialization.id,
|
|
2207
|
+
verifiedRevision: rematerialization.selectedRevision
|
|
2208
|
+
}
|
|
2209
|
+
} : {},
|
|
2210
|
+
leaseTtlMs
|
|
2211
|
+
});
|
|
2212
|
+
if (!committed.committed || !committed.lease) {
|
|
2213
|
+
const terminated = await terminateEstablishedSandbox(established);
|
|
2214
|
+
if (terminated && rematerialization && !established.providerContinuity) {
|
|
2215
|
+
await failSandboxRematerialization(db, {
|
|
2216
|
+
accountId: ids.accountId,
|
|
2217
|
+
workspaceId: ids.workspaceId,
|
|
2218
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2219
|
+
expectedEpoch,
|
|
2220
|
+
rematerializationId: rematerialization.id,
|
|
2221
|
+
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
2222
|
+
retryable: false
|
|
2223
|
+
});
|
|
2224
|
+
} else if (terminated) {
|
|
2225
|
+
await failWarmingToCold(db, {
|
|
2226
|
+
accountId: ids.accountId,
|
|
2227
|
+
workspaceId: ids.workspaceId,
|
|
2228
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2229
|
+
expectedEpoch
|
|
2230
|
+
});
|
|
2231
|
+
}
|
|
2232
|
+
await release();
|
|
2233
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
2234
|
+
}
|
|
2235
|
+
holderLeaseHeartbeat = {
|
|
2236
|
+
expectedEpoch: committed.lease.leaseEpoch,
|
|
2237
|
+
leaseTtlMs
|
|
2238
|
+
};
|
|
2239
|
+
throwIfReleasedOrCancelled();
|
|
2240
|
+
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
2241
|
+
} catch (error) {
|
|
2242
|
+
if (error instanceof SandboxLeaseSupersededError) {
|
|
2243
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
2244
|
+
await release();
|
|
2245
|
+
throw error;
|
|
2246
|
+
}
|
|
2247
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
2248
|
+
if (terminated) {
|
|
2249
|
+
const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
|
|
2250
|
+
if (rematerialization && !createdEstablished?.providerContinuity) {
|
|
2251
|
+
await failSandboxRematerialization(db, {
|
|
2252
|
+
accountId: ids.accountId,
|
|
2253
|
+
workspaceId: ids.workspaceId,
|
|
2254
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2255
|
+
expectedEpoch,
|
|
2256
|
+
rematerializationId: rematerialization.id,
|
|
2257
|
+
failureCode: continuityUnavailable ? "provider_continuity_unavailable" : error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
|
|
2258
|
+
retryable: continuityUnavailable ? error.retryable : error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
|
|
2259
|
+
...continuityUnavailable ? { discardContinuity: true } : {}
|
|
2260
|
+
});
|
|
2261
|
+
} else {
|
|
2262
|
+
await failWarmingToCold(db, {
|
|
2263
|
+
accountId: ids.accountId,
|
|
2264
|
+
workspaceId: ids.workspaceId,
|
|
2265
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2266
|
+
expectedEpoch,
|
|
2267
|
+
...continuityUnavailable ? { discardContinuity: true } : {}
|
|
2268
|
+
});
|
|
2269
|
+
}
|
|
2270
|
+
}
|
|
2271
|
+
await release();
|
|
2272
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
2273
|
+
throw error;
|
|
2274
|
+
}
|
|
2275
|
+
}
|
|
2276
|
+
try {
|
|
2277
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
2278
|
+
if (acquired.lease.liveness === "warming") {
|
|
2279
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
2280
|
+
holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
|
|
2281
|
+
}
|
|
2282
|
+
throwIfReleasedOrCancelled();
|
|
2283
|
+
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
2284
|
+
if (!live || live.liveness !== "warm" || live.leaseEpoch !== leaseEpoch || live.instanceId === null) {
|
|
2285
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
|
|
2286
|
+
}
|
|
2287
|
+
let established;
|
|
2288
|
+
try {
|
|
2289
|
+
const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
|
|
2290
|
+
established = await establish(settings, live.resumeState, {
|
|
2291
|
+
sessionId: ids.sessionId,
|
|
2292
|
+
recovery: "resume-only",
|
|
2293
|
+
backendOverride: ids.backend,
|
|
2294
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
2295
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
2296
|
+
});
|
|
2297
|
+
throwIfReleasedOrCancelled();
|
|
2298
|
+
if (services.verifyAttachedSandboxReadiness) {
|
|
2299
|
+
await services.verifyAttachedSandboxReadiness(established);
|
|
2300
|
+
} else {
|
|
2301
|
+
await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
|
|
2302
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
2303
|
+
});
|
|
2304
|
+
}
|
|
2305
|
+
throwIfReleasedOrCancelled();
|
|
2306
|
+
} catch (error) {
|
|
2307
|
+
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
2308
|
+
throw error;
|
|
2309
|
+
}
|
|
2310
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
2311
|
+
accountId: ids.accountId,
|
|
2312
|
+
workspaceId: ids.workspaceId,
|
|
2313
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2314
|
+
expectedEpoch: leaseEpoch,
|
|
2315
|
+
expectedInstanceId: live.instanceId
|
|
2316
|
+
});
|
|
2317
|
+
if (marked.status === "marked") {
|
|
2318
|
+
await services.onSandboxLost?.({
|
|
2319
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2320
|
+
instanceId: live.instanceId,
|
|
2321
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
2322
|
+
});
|
|
2323
|
+
throw new SandboxLeaseInstanceLostError(
|
|
2324
|
+
ids.sandboxGroupId,
|
|
2325
|
+
marked.lease.leaseEpoch,
|
|
2326
|
+
live.instanceId
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
2329
|
+
throw new SandboxLeaseSupersededError(
|
|
2330
|
+
ids.sandboxGroupId,
|
|
2331
|
+
marked.lease?.leaseEpoch ?? leaseEpoch
|
|
2332
|
+
);
|
|
2333
|
+
}
|
|
2334
|
+
throwIfReleasedOrCancelled();
|
|
2335
|
+
return { established, leaseEpoch, release };
|
|
2336
|
+
} catch (error) {
|
|
2337
|
+
await release();
|
|
2338
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
2339
|
+
throw error;
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
async function waitForWarm(services, ids) {
|
|
2343
|
+
const { db, settings } = services;
|
|
2344
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
2345
|
+
let instanceId = null;
|
|
2346
|
+
while (Date.now() < deadline) {
|
|
2347
|
+
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
2348
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
2349
|
+
if (!lease) {
|
|
2350
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
2351
|
+
}
|
|
2352
|
+
instanceId = lease.instanceId;
|
|
2353
|
+
if (lease.liveness === "warm") {
|
|
2354
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
2355
|
+
}
|
|
2356
|
+
if (lease.liveness === "draining") {
|
|
2357
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
2358
|
+
}
|
|
2359
|
+
if (lease.liveness === "cold") {
|
|
2360
|
+
if (isRetryableDegradedRestore(lease.recovery.restore)) {
|
|
2361
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
2362
|
+
}
|
|
2363
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
|
|
2364
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
2365
|
+
ids.sandboxGroupId,
|
|
2366
|
+
lease.leaseEpoch,
|
|
2367
|
+
lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
|
|
2368
|
+
lease.recovery
|
|
2369
|
+
);
|
|
2370
|
+
}
|
|
2371
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
|
|
2375
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2376
|
+
instanceId
|
|
2377
|
+
});
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
// src/sandbox-routing.ts
|
|
2381
|
+
import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
|
|
2382
|
+
import {
|
|
2383
|
+
advanceWorkspaceGenerationForRetainedProcess,
|
|
2384
|
+
advanceWorkspaceGeneration,
|
|
2385
|
+
getRetainedProcess,
|
|
2386
|
+
retainWorkspaceMutationProcess,
|
|
2387
|
+
retainedProcessSettlementIdentity,
|
|
2388
|
+
settleRetainedProcess,
|
|
2389
|
+
verifyRetainedProcessMutationSettlement,
|
|
2390
|
+
verifyWorkspaceMutationSettlement,
|
|
2391
|
+
getSandbox,
|
|
2392
|
+
getEnrollment,
|
|
2393
|
+
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
2394
|
+
readLease as readLease2,
|
|
2395
|
+
readActiveSandbox,
|
|
2396
|
+
SandboxRetainedProcessPromotionFencedError
|
|
2397
|
+
} from "@opengeni/db";
|
|
2398
|
+
import {
|
|
2399
|
+
buildSelfhostedBackendSession,
|
|
2400
|
+
establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
|
|
2401
|
+
isProviderSandboxGoneDuringRoutedOperation,
|
|
2402
|
+
isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
|
|
2403
|
+
makeActiveBackendResolver,
|
|
2404
|
+
NatsControlRpc,
|
|
2405
|
+
NatsOpStreamTransport,
|
|
2406
|
+
RoutingBackendRecoveryRequiredError,
|
|
2407
|
+
RoutingSandboxSession,
|
|
2408
|
+
resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
|
|
2409
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
2410
|
+
sandboxBackendForSdkBackendId,
|
|
2411
|
+
selectBackend,
|
|
2412
|
+
verifySandboxExecReadiness as verifySandboxExecReadiness2
|
|
2413
|
+
} from "@opengeni/runtime";
|
|
2414
|
+
function relayConfigFromSettings(settings) {
|
|
2415
|
+
const raw = settings.selfhostedRelayUrl?.trim();
|
|
2416
|
+
if (!raw) {
|
|
2417
|
+
return { host: "relay.opengeni.local", port: 443, tls: true };
|
|
2418
|
+
}
|
|
2419
|
+
try {
|
|
2420
|
+
const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
|
|
2421
|
+
const tls = url.protocol === "wss:" || url.protocol === "https:";
|
|
2422
|
+
const port = url.port ? Number(url.port) : tls ? 443 : 80;
|
|
2423
|
+
return { host: url.hostname, port, tls };
|
|
2424
|
+
} catch {
|
|
2425
|
+
return { host: raw, port: 443, tls: true };
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
function selfhostedTimeoutsFromSettings(settings) {
|
|
2429
|
+
return {
|
|
2430
|
+
timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
|
|
2431
|
+
execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs
|
|
2432
|
+
};
|
|
2433
|
+
}
|
|
2434
|
+
function selfhostedResolverTimeouts(settings) {
|
|
2435
|
+
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
2436
|
+
return {
|
|
2437
|
+
selfhostedTimeoutMs: timeoutMs,
|
|
2438
|
+
selfhostedExecTimeoutMs: execTimeoutMs
|
|
2439
|
+
};
|
|
2440
|
+
}
|
|
2441
|
+
function homeRouteRecoveryDisposition(lease) {
|
|
2442
|
+
if (!lease) return "unrecoverable";
|
|
2443
|
+
if (lease.liveness === "warming") return "pending";
|
|
2444
|
+
if (lease.liveness === "draining") return "superseded";
|
|
2445
|
+
if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
|
|
2446
|
+
return "pending";
|
|
2447
|
+
}
|
|
2448
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
|
|
2449
|
+
return "degraded";
|
|
2450
|
+
}
|
|
2451
|
+
return "unrecoverable";
|
|
2452
|
+
}
|
|
2453
|
+
function homeRouteRecoveryError(lease, fallbackEpoch) {
|
|
2454
|
+
return new RoutingBackendRecoveryRequiredError(
|
|
2455
|
+
"resolve_home_backend",
|
|
2456
|
+
lease?.leaseEpoch ?? fallbackEpoch,
|
|
2457
|
+
homeRouteRecoveryDisposition(lease)
|
|
2458
|
+
);
|
|
2459
|
+
}
|
|
2460
|
+
function providerIdentityFromResumeState(resumeState) {
|
|
2461
|
+
return sandboxProviderInstanceIdFromEnvelope(resumeState);
|
|
2462
|
+
}
|
|
2463
|
+
async function resolveCurrentHomeBackend(services, ids, established) {
|
|
2464
|
+
const lease = await readLease2(services.db, ids.workspaceId, ids.sandboxGroupId);
|
|
2465
|
+
const fallbackEpoch = lease?.leaseEpoch ?? 0;
|
|
2466
|
+
if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
|
|
2467
|
+
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
2468
|
+
}
|
|
2469
|
+
if (lease.instanceId === established.instanceId) {
|
|
2470
|
+
services.onHomeSandboxRebound?.({
|
|
2471
|
+
established,
|
|
2472
|
+
leaseEpoch: lease.leaseEpoch
|
|
2473
|
+
});
|
|
2474
|
+
return {
|
|
2475
|
+
session: established.session,
|
|
2476
|
+
sandboxId: null,
|
|
2477
|
+
kind: established.backendId,
|
|
2478
|
+
leaseEpoch: lease.leaseEpoch,
|
|
2479
|
+
providerInstanceId: lease.instanceId
|
|
2480
|
+
};
|
|
2481
|
+
}
|
|
2482
|
+
const resumeState = lease.resumeState;
|
|
2483
|
+
const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
2484
|
+
const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
|
|
2485
|
+
if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
|
|
2486
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
2487
|
+
}
|
|
2488
|
+
let rebound;
|
|
2489
|
+
try {
|
|
2490
|
+
rebound = await establishSandboxSessionFromEnvelope2(services.settings, resumeState, {
|
|
2491
|
+
sessionId: ids.sessionId,
|
|
2492
|
+
recovery: "resume-only",
|
|
2493
|
+
backendOverride: resumeBackend
|
|
2494
|
+
});
|
|
2495
|
+
await verifySandboxExecReadiness2(rebound);
|
|
2496
|
+
} catch (error) {
|
|
2497
|
+
if (!isProviderSandboxNotFoundError2(resumeBackend, error)) {
|
|
2498
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
2499
|
+
"resolve_home_backend",
|
|
2500
|
+
lease.leaseEpoch,
|
|
2501
|
+
"pending"
|
|
2502
|
+
);
|
|
2503
|
+
}
|
|
2504
|
+
const marked = await markWarmLeaseInstanceLost2(services.db, {
|
|
2505
|
+
accountId: ids.accountId,
|
|
2506
|
+
workspaceId: ids.workspaceId,
|
|
2507
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2508
|
+
expectedEpoch: lease.leaseEpoch,
|
|
2509
|
+
expectedInstanceId: lease.instanceId,
|
|
2510
|
+
diagnostic: "provider_not_found_during_home_route_rebind"
|
|
2511
|
+
});
|
|
2512
|
+
if (marked.status === "marked") {
|
|
2513
|
+
await services.onHomeSandboxLost?.({
|
|
2514
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2515
|
+
instanceId: lease.instanceId,
|
|
2516
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
2517
|
+
});
|
|
2518
|
+
}
|
|
2519
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
2520
|
+
"resolve_home_backend",
|
|
2521
|
+
marked.lease?.leaseEpoch ?? lease.leaseEpoch,
|
|
2522
|
+
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
|
|
2523
|
+
);
|
|
2524
|
+
}
|
|
2525
|
+
const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
|
|
2526
|
+
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
2527
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
2528
|
+
}
|
|
2529
|
+
services.onHomeSandboxRebound?.({
|
|
2530
|
+
established: rebound,
|
|
2531
|
+
leaseEpoch: lease.leaseEpoch
|
|
2532
|
+
});
|
|
2533
|
+
Object.assign(established, rebound);
|
|
2534
|
+
return {
|
|
2535
|
+
session: rebound.session,
|
|
2536
|
+
sandboxId: null,
|
|
2537
|
+
kind: rebound.backendId,
|
|
2538
|
+
leaseEpoch: lease.leaseEpoch,
|
|
2539
|
+
providerInstanceId: lease.instanceId
|
|
2540
|
+
};
|
|
2541
|
+
}
|
|
2542
|
+
function beforePersistableHomeMutation(services, ids, home) {
|
|
2543
|
+
const fence = ids.workspaceMutationFence;
|
|
2544
|
+
if (!home || !fence) return void 0;
|
|
2545
|
+
return async ({ op, backend }) => {
|
|
2546
|
+
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
2547
|
+
return null;
|
|
2548
|
+
}
|
|
2549
|
+
const providerBinding = home.backend === "modal" ? await resolveModalCheckpointProviderBindingForSession2(services.settings, backend.session) : null;
|
|
2550
|
+
const admission = await advanceWorkspaceGeneration(services.db, {
|
|
2551
|
+
accountId: fence.accountId,
|
|
2552
|
+
workspaceId: ids.workspaceId,
|
|
2553
|
+
sessionId: ids.sessionId,
|
|
2554
|
+
turnId: fence.turnId,
|
|
2555
|
+
executionGeneration: fence.executionGeneration,
|
|
2556
|
+
attemptId: fence.attemptId,
|
|
2557
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
2558
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2559
|
+
expectedEpoch: backend.leaseEpoch,
|
|
2560
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
2561
|
+
operation: op,
|
|
2562
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
|
|
2563
|
+
...services.waitSignal ? { waitSignal: services.waitSignal } : {}
|
|
2564
|
+
});
|
|
2565
|
+
return { admission, providerBinding };
|
|
2566
|
+
};
|
|
2567
|
+
}
|
|
2568
|
+
function afterPersistableHomeMutation(services, ids, home) {
|
|
2569
|
+
const fence = ids.workspaceMutationFence;
|
|
2570
|
+
if (!home || !fence) return void 0;
|
|
2571
|
+
return async ({ op, backend, admission, outcome, retainedProcess }) => {
|
|
2572
|
+
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object") {
|
|
2573
|
+
return;
|
|
2574
|
+
}
|
|
2575
|
+
const boundAdmission = admission;
|
|
2576
|
+
const exactAdmission = boundAdmission.admission;
|
|
2577
|
+
if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
|
|
2578
|
+
throw new Error("Persistable home mutation settlement lacked its bound admission");
|
|
2579
|
+
}
|
|
2580
|
+
if (outcome === "resolved" && retainedProcess) {
|
|
2581
|
+
try {
|
|
2582
|
+
await retainWorkspaceMutationProcess(services.db, {
|
|
2583
|
+
accountId: fence.accountId,
|
|
2584
|
+
workspaceId: ids.workspaceId,
|
|
2585
|
+
sessionId: ids.sessionId,
|
|
2586
|
+
processId: retainedProcess.id,
|
|
2587
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
2588
|
+
admissionId: exactAdmission.id,
|
|
2589
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
2590
|
+
operation: op,
|
|
2591
|
+
providerBinding: boundAdmission.providerBinding ?? null,
|
|
2592
|
+
owner: {
|
|
2593
|
+
kind: "turn",
|
|
2594
|
+
turnId: fence.turnId,
|
|
2595
|
+
executionGeneration: fence.executionGeneration,
|
|
2596
|
+
attemptId: fence.attemptId,
|
|
2597
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
2598
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2599
|
+
expectedEpoch: backend.leaseEpoch,
|
|
2600
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
2601
|
+
routeKind: exactAdmission.routeKind,
|
|
2602
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
2603
|
+
routeEpoch: exactAdmission.routeEpoch
|
|
2604
|
+
}
|
|
2605
|
+
});
|
|
2606
|
+
} catch (error) {
|
|
2607
|
+
if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
|
|
2608
|
+
const durable = error.process;
|
|
2609
|
+
if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
|
|
2610
|
+
throw new Error("Durable retained-process promotion returned a mismatched identity", {
|
|
2611
|
+
cause: error
|
|
2612
|
+
});
|
|
2613
|
+
}
|
|
2614
|
+
return {
|
|
2615
|
+
status: "retained_process_durable_output_rejected",
|
|
2616
|
+
retainedProcess: {
|
|
2617
|
+
id: durable.id,
|
|
2618
|
+
providerSessionId: durable.providerSessionId
|
|
2619
|
+
}
|
|
2620
|
+
};
|
|
2621
|
+
}
|
|
2622
|
+
return;
|
|
2623
|
+
}
|
|
2624
|
+
await verifyWorkspaceMutationSettlement(services.db, {
|
|
2625
|
+
accountId: fence.accountId,
|
|
2626
|
+
workspaceId: ids.workspaceId,
|
|
2627
|
+
sessionId: ids.sessionId,
|
|
2628
|
+
turnId: fence.turnId,
|
|
2629
|
+
executionGeneration: fence.executionGeneration,
|
|
2630
|
+
attemptId: fence.attemptId,
|
|
2631
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
2632
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2633
|
+
expectedEpoch: backend.leaseEpoch,
|
|
2634
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
2635
|
+
admission: exactAdmission,
|
|
2636
|
+
operation: op,
|
|
2637
|
+
outcome
|
|
2638
|
+
});
|
|
2639
|
+
};
|
|
2640
|
+
}
|
|
2641
|
+
function beforeRetainedProcessMutation(services, ids) {
|
|
2642
|
+
const fence = ids.workspaceMutationFence;
|
|
2643
|
+
if (!fence) return void 0;
|
|
2644
|
+
return async ({ op, process }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
|
|
2645
|
+
accountId: fence.accountId,
|
|
2646
|
+
workspaceId: ids.workspaceId,
|
|
2647
|
+
sessionId: ids.sessionId,
|
|
2648
|
+
processId: process.id,
|
|
2649
|
+
operation: op,
|
|
2650
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
|
|
2651
|
+
...services.waitSignal ? { waitSignal: services.waitSignal } : {}
|
|
2652
|
+
});
|
|
2653
|
+
}
|
|
2654
|
+
function afterRetainedProcessMutation(services, ids) {
|
|
2655
|
+
const fence = ids.workspaceMutationFence;
|
|
2656
|
+
if (!fence) return void 0;
|
|
2657
|
+
return async ({ op, process, admission, outcome }) => {
|
|
2658
|
+
if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
|
|
2659
|
+
throw new Error("Retained-process mutation settlement lacked its exact admission");
|
|
2660
|
+
}
|
|
2661
|
+
await verifyRetainedProcessMutationSettlement(services.db, {
|
|
2662
|
+
accountId: fence.accountId,
|
|
2663
|
+
workspaceId: ids.workspaceId,
|
|
2664
|
+
sessionId: ids.sessionId,
|
|
2665
|
+
processId: process.id,
|
|
2666
|
+
admission,
|
|
2667
|
+
operation: op,
|
|
2668
|
+
outcome
|
|
2669
|
+
});
|
|
2670
|
+
};
|
|
2671
|
+
}
|
|
2672
|
+
function settleRetainedProcessForTurn(services, ids) {
|
|
2673
|
+
const fence = ids.workspaceMutationFence;
|
|
2674
|
+
if (!fence) return void 0;
|
|
2675
|
+
return async ({ backend, process, proof }) => {
|
|
2676
|
+
const durable = await getRetainedProcess(services.db, {
|
|
2677
|
+
workspaceId: ids.workspaceId,
|
|
2678
|
+
sessionId: ids.sessionId,
|
|
2679
|
+
processId: process.id
|
|
2680
|
+
});
|
|
2681
|
+
if (!durable || durable.providerSessionId !== process.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
2682
|
+
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
2683
|
+
}
|
|
2684
|
+
await settleRetainedProcess(services.db, {
|
|
2685
|
+
accountId: fence.accountId,
|
|
2686
|
+
workspaceId: ids.workspaceId,
|
|
2687
|
+
sessionId: ids.sessionId,
|
|
2688
|
+
processId: process.id,
|
|
2689
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
2690
|
+
outcome: proof.outcome,
|
|
2691
|
+
exitCode: proof.exitCode,
|
|
2692
|
+
reason: proof.reason,
|
|
2693
|
+
idleGraceMs: services.settings.sandboxIdleGraceMs
|
|
2694
|
+
});
|
|
2695
|
+
};
|
|
2696
|
+
}
|
|
2697
|
+
function controlRpcFactory(bus) {
|
|
2698
|
+
return () => new NatsControlRpc(async () => {
|
|
2699
|
+
if (!bus) {
|
|
2700
|
+
return null;
|
|
2701
|
+
}
|
|
2702
|
+
return bus.getRequestConnection();
|
|
2703
|
+
});
|
|
2704
|
+
}
|
|
2705
|
+
function wrapTurnBoxWithRouting(services, ids, established) {
|
|
2706
|
+
const { db, settings, bus, onOp } = services;
|
|
2707
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
|
|
2708
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
|
|
2709
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
2710
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
2711
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
2712
|
+
const resolver = makeActiveBackendResolver({
|
|
2713
|
+
workspaceId: ids.workspaceId,
|
|
2714
|
+
defaultBackend: established.session,
|
|
2715
|
+
defaultKind: established.backendId,
|
|
2716
|
+
getSandbox: async (sandboxId) => {
|
|
2717
|
+
const sandbox = await getSandbox(db, ids.workspaceId, sandboxId);
|
|
2718
|
+
return sandbox ? {
|
|
2719
|
+
id: sandbox.id,
|
|
2720
|
+
kind: sandbox.kind,
|
|
2721
|
+
name: sandbox.name,
|
|
2722
|
+
enrollmentId: sandbox.enrollmentId
|
|
2723
|
+
} : null;
|
|
2724
|
+
},
|
|
2725
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
2726
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
2727
|
+
if (!sandbox.enrollmentId) return void 0;
|
|
2728
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
2729
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
2730
|
+
},
|
|
2731
|
+
relay: relayConfigFromSettings(settings),
|
|
2732
|
+
// A selfhosted swap target runs real commands too, so give it the same split
|
|
2733
|
+
// deadlines the machine-primary establish path uses (short control, long exec).
|
|
2734
|
+
...selfhostedResolverTimeouts(settings),
|
|
2735
|
+
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
2736
|
+
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
2737
|
+
// SDK's per-turn manifest-env delta is empty (no "cannot change manifest
|
|
2738
|
+
// environment variables" throw when the turn pins to a vm). Mirrors the group
|
|
2739
|
+
// box, which is created WITH this same environment (resumeBoxForTurn).
|
|
2740
|
+
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
2741
|
+
// Stage D machine-primary: pin THIS established SelfhostedSession for the machine
|
|
2742
|
+
// pointer so the resolver returns the SAME instance (no two-instance manifest
|
|
2743
|
+
// divergence). `established.session` is the SelfhostedSession the establish branch
|
|
2744
|
+
// bound to (sandboxId, epoch).
|
|
2745
|
+
...ids.pinnedSelfhosted ? {
|
|
2746
|
+
pinnedSelfhosted: {
|
|
2747
|
+
sandboxId: ids.pinnedSelfhosted.sandboxId,
|
|
2748
|
+
epoch: ids.pinnedSelfhosted.epoch,
|
|
2749
|
+
session: established.session
|
|
2750
|
+
}
|
|
2751
|
+
} : {},
|
|
2752
|
+
// A modal swap target in the turn path would need its own lease resume-by-id;
|
|
2753
|
+
// that is a future cross-group-box concern. Until then a modal swap target is
|
|
2754
|
+
// unresolvable (the swap tool validates liveness, so this only triggers if a
|
|
2755
|
+
// session points at a sibling modal box the turn cannot resume here) and the
|
|
2756
|
+
// op surfaces unresolvable — never a silent wrong-box landing.
|
|
2757
|
+
//
|
|
2758
|
+
// For a machine-primary turn of a Modal-HOME session (pinned to a machine, no
|
|
2759
|
+
// group box established this turn), a mid-turn clear-to-null must NOT fall back to
|
|
2760
|
+
// the pinned machine — passing defaultIsHome:false makes the null branch throw typed
|
|
2761
|
+
// `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
|
|
2762
|
+
...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
|
|
2763
|
+
...ids.homeLease ? {
|
|
2764
|
+
resolveDefaultBackend: () => resolveCurrentHomeBackend(
|
|
2765
|
+
services,
|
|
2766
|
+
{
|
|
2767
|
+
workspaceId: ids.workspaceId,
|
|
2768
|
+
sessionId: ids.sessionId,
|
|
2769
|
+
accountId: ids.homeLease.accountId,
|
|
2770
|
+
sandboxGroupId: ids.homeLease.sandboxGroupId,
|
|
2771
|
+
backend: ids.homeLease.backend
|
|
2772
|
+
},
|
|
2773
|
+
established
|
|
2774
|
+
)
|
|
2775
|
+
} : {}
|
|
2776
|
+
});
|
|
2777
|
+
const proxy = new RoutingSandboxSession({
|
|
2778
|
+
// Seed the DEFAULT backend (the established group box) at construction so
|
|
2779
|
+
// `session.state` is the real backend's state object BEFORE the first op. The
|
|
2780
|
+
// SDK reads `session.state.manifest` at turn START (and writes it back); an
|
|
2781
|
+
// empty `{}` there crashes serializeManifestEnvironment /
|
|
2782
|
+
// validateProvidedSessionManifestUpdate. This is byte-identical to what the
|
|
2783
|
+
// resolver returns for the default pointer (`activeSandboxId === null`).
|
|
2784
|
+
defaultResolved: {
|
|
2785
|
+
session: established.session,
|
|
2786
|
+
sandboxId: null,
|
|
2787
|
+
kind: established.backendId,
|
|
2788
|
+
...ids.homeLease ? {
|
|
2789
|
+
leaseEpoch: ids.homeLease.leaseEpoch,
|
|
2790
|
+
providerInstanceId: ids.homeLease.instanceId
|
|
2791
|
+
} : {}
|
|
2792
|
+
},
|
|
2793
|
+
readPointer: async () => {
|
|
2794
|
+
if (!routingEnabled(settings)) {
|
|
2795
|
+
return { activeSandboxId: null, activeEpoch: 0 };
|
|
2796
|
+
}
|
|
2797
|
+
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
2798
|
+
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
2799
|
+
},
|
|
2800
|
+
resolveActiveBackend: resolver,
|
|
2801
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
2802
|
+
...beforeMutation ? { beforeMutation } : {},
|
|
2803
|
+
...afterMutation ? { afterMutation } : {},
|
|
2804
|
+
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
2805
|
+
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
2806
|
+
...settleProcess ? { settleProcess } : {},
|
|
2807
|
+
...ids.homeLease ? {
|
|
2808
|
+
onDefaultBackendError: async ({
|
|
2809
|
+
error,
|
|
2810
|
+
backend
|
|
2811
|
+
}) => {
|
|
2812
|
+
const home = ids.homeLease;
|
|
2813
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
2814
|
+
const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
|
|
2815
|
+
const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
|
|
2816
|
+
const marked = await markWarmLeaseInstanceLost2(db, {
|
|
2817
|
+
accountId: home.accountId,
|
|
2818
|
+
workspaceId: ids.workspaceId,
|
|
2819
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2820
|
+
expectedEpoch,
|
|
2821
|
+
expectedInstanceId,
|
|
2822
|
+
diagnostic: "provider_not_found_during_routed_operation"
|
|
2823
|
+
});
|
|
2824
|
+
if (marked.status === "marked") {
|
|
2825
|
+
await services.onHomeSandboxLost?.({
|
|
2826
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2827
|
+
instanceId: expectedInstanceId,
|
|
2828
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
2829
|
+
});
|
|
2830
|
+
}
|
|
2831
|
+
const lease = marked.lease;
|
|
2832
|
+
const restore = lease?.recovery.restore.status;
|
|
2833
|
+
return {
|
|
2834
|
+
leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
|
|
2835
|
+
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
2836
|
+
};
|
|
2837
|
+
}
|
|
2838
|
+
} : {}
|
|
2839
|
+
});
|
|
2840
|
+
return { ...established, session: proxy };
|
|
2841
|
+
}
|
|
2842
|
+
function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
2843
|
+
const { db, settings, bus, onOp } = services;
|
|
2844
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
2845
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
2846
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
2847
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
2848
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
2849
|
+
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
2850
|
+
const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
|
|
2851
|
+
const syntheticSession = {
|
|
2852
|
+
state: { manifest: args.agentDefaultManifest },
|
|
2853
|
+
// The SDK binds shell tools synchronously before lazy provisioning. Preserve
|
|
2854
|
+
// the authoritative descriptor's PTY shape so a Modal home exposes
|
|
2855
|
+
// write_stdin on its first turn operation; the async cancellation transport
|
|
2856
|
+
// below still resolves the exact live route before executing a command.
|
|
2857
|
+
supportsPty: () => defaultSupportsPty
|
|
2858
|
+
};
|
|
2859
|
+
const routedResolver = makeActiveBackendResolver({
|
|
2860
|
+
workspaceId: ids.workspaceId,
|
|
2861
|
+
defaultBackend: syntheticSession,
|
|
2862
|
+
defaultKind: "unprovisioned",
|
|
2863
|
+
getSandbox: async (sandboxId) => {
|
|
2864
|
+
const sandbox = await getSandbox(db, ids.workspaceId, sandboxId);
|
|
2865
|
+
return sandbox ? {
|
|
2866
|
+
id: sandbox.id,
|
|
2867
|
+
kind: sandbox.kind,
|
|
2868
|
+
name: sandbox.name,
|
|
2869
|
+
enrollmentId: sandbox.enrollmentId
|
|
2870
|
+
} : null;
|
|
2871
|
+
},
|
|
2872
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
2873
|
+
resolveSelfhostedOpStream: async (sandbox) => {
|
|
2874
|
+
if (!sandbox.enrollmentId) return void 0;
|
|
2875
|
+
const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
|
|
2876
|
+
return opStreamDepsFor(services, enrollment?.opStream === true);
|
|
2877
|
+
},
|
|
2878
|
+
relay: relayConfigFromSettings(settings),
|
|
2879
|
+
...selfhostedResolverTimeouts(settings),
|
|
2880
|
+
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
2881
|
+
...ids.environment !== void 0 ? { environment: ids.environment } : {}
|
|
2882
|
+
});
|
|
2883
|
+
const proxy = new RoutingSandboxSession({
|
|
2884
|
+
// Before the first op the SDK reads `state.manifest`; the synthetic backend
|
|
2885
|
+
// points at agent.defaultManifest BY REFERENCE so the provided-session delta is
|
|
2886
|
+
// empty. The first default-pointer op resolves the real box through the
|
|
2887
|
+
// provisioner and `state` switches to that real backend by reference.
|
|
2888
|
+
defaultResolved: {
|
|
2889
|
+
session: syntheticSession,
|
|
2890
|
+
sandboxId: null,
|
|
2891
|
+
kind: "unprovisioned"
|
|
2892
|
+
},
|
|
2893
|
+
readPointer: async () => {
|
|
2894
|
+
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
2895
|
+
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
2896
|
+
},
|
|
2897
|
+
resolveActiveBackend: async (pointer) => {
|
|
2898
|
+
if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
|
|
2899
|
+
const provisioned = await args.provisioner.get();
|
|
2900
|
+
if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
|
|
2901
|
+
return resolveCurrentHomeBackend(
|
|
2902
|
+
services,
|
|
2903
|
+
{
|
|
2904
|
+
workspaceId: ids.workspaceId,
|
|
2905
|
+
sessionId: ids.sessionId,
|
|
2906
|
+
accountId: args.homeLeaseIdentity.accountId,
|
|
2907
|
+
sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
|
|
2908
|
+
backend: args.homeLeaseIdentity.backend
|
|
2909
|
+
},
|
|
2910
|
+
provisioned.established
|
|
2911
|
+
);
|
|
2912
|
+
}
|
|
2913
|
+
return {
|
|
2914
|
+
session: provisioned.established.session,
|
|
2915
|
+
sandboxId: null,
|
|
2916
|
+
kind: provisioned.established.backendId
|
|
2917
|
+
};
|
|
2918
|
+
}
|
|
2919
|
+
return routedResolver(pointer);
|
|
2920
|
+
},
|
|
2921
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
2922
|
+
...beforeMutation ? { beforeMutation } : {},
|
|
2923
|
+
...afterMutation ? { afterMutation } : {},
|
|
2924
|
+
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
2925
|
+
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
2926
|
+
...settleProcess ? { settleProcess } : {},
|
|
2927
|
+
...args.homeLeaseIdentity ? {
|
|
2928
|
+
onDefaultBackendError: async ({
|
|
2929
|
+
error,
|
|
2930
|
+
backend
|
|
2931
|
+
}) => {
|
|
2932
|
+
const home = args.homeLeaseIdentity;
|
|
2933
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
2934
|
+
if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
2935
|
+
return null;
|
|
2936
|
+
}
|
|
2937
|
+
const marked = await markWarmLeaseInstanceLost2(db, {
|
|
2938
|
+
accountId: home.accountId,
|
|
2939
|
+
workspaceId: ids.workspaceId,
|
|
2940
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2941
|
+
expectedEpoch: backend.leaseEpoch,
|
|
2942
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
2943
|
+
diagnostic: "provider_not_found_during_routed_operation"
|
|
2944
|
+
});
|
|
2945
|
+
if (marked.status === "marked") {
|
|
2946
|
+
await services.onHomeSandboxLost?.({
|
|
2947
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2948
|
+
instanceId: backend.providerInstanceId,
|
|
2949
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
2950
|
+
});
|
|
2951
|
+
}
|
|
2952
|
+
const lease = marked.lease;
|
|
2953
|
+
const restore = lease?.recovery.restore.status;
|
|
2954
|
+
return {
|
|
2955
|
+
leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
|
|
2956
|
+
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
2957
|
+
};
|
|
2958
|
+
}
|
|
2959
|
+
} : {}
|
|
2960
|
+
});
|
|
2961
|
+
return {
|
|
2962
|
+
client: args.client,
|
|
2963
|
+
session: proxy,
|
|
2964
|
+
sessionState: void 0,
|
|
2965
|
+
instanceId: "unprovisioned",
|
|
2966
|
+
backendId: args.backendId
|
|
2967
|
+
};
|
|
2968
|
+
}
|
|
2969
|
+
function opStreamDepsFor(services, machineAdvertisesOpStream) {
|
|
2970
|
+
const { settings, bus, opJournal } = services;
|
|
2971
|
+
if (!machineAdvertisesOpStream || settings.agentOpStreamEnabled !== true || !bus) {
|
|
2972
|
+
return void 0;
|
|
2973
|
+
}
|
|
2974
|
+
return {
|
|
2975
|
+
transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
|
|
2976
|
+
...opJournal !== void 0 ? { journal: opJournal } : {}
|
|
2977
|
+
};
|
|
2978
|
+
}
|
|
2979
|
+
async function establishSelfhostedTurnSession(services, args) {
|
|
2980
|
+
const { settings, bus, onOp } = services;
|
|
2981
|
+
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
2982
|
+
const opStream = opStreamDepsFor(services, args.opStream);
|
|
2983
|
+
const { client, session } = await buildSelfhostedBackendSession({
|
|
2984
|
+
workspaceId: args.workspaceId,
|
|
2985
|
+
agentId: args.agentId,
|
|
2986
|
+
relay: relayConfigFromSettings(settings),
|
|
2987
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
2988
|
+
epoch: args.epoch,
|
|
2989
|
+
environment: args.environment,
|
|
2990
|
+
workingDir: args.workingDir,
|
|
2991
|
+
// Give this turn's exec ops the long deadline (control ops stay short) so a real
|
|
2992
|
+
// command is not killed at the control wall.
|
|
2993
|
+
timeoutMs,
|
|
2994
|
+
execTimeoutMs,
|
|
2995
|
+
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
2996
|
+
...onOp !== void 0 ? { onOp } : {},
|
|
2997
|
+
// The streaming exec transport — present iff the machine advertised the
|
|
2998
|
+
// capability AND the server flag is on. It is required when execTimeoutMs=0;
|
|
2999
|
+
// legacy request/reply remains available only for explicitly bounded exec.
|
|
3000
|
+
...opStream !== void 0 ? { opStream } : {}
|
|
3001
|
+
});
|
|
3002
|
+
return {
|
|
3003
|
+
client,
|
|
3004
|
+
session,
|
|
3005
|
+
sessionState: { agentId: args.agentId },
|
|
3006
|
+
instanceId: args.agentId,
|
|
3007
|
+
backendId: "selfhosted"
|
|
3008
|
+
};
|
|
3009
|
+
}
|
|
3010
|
+
function routingEnabled(settings) {
|
|
3011
|
+
return settings.sandboxSelfhostedEnabled === true;
|
|
3012
|
+
}
|
|
3013
|
+
function lazyProvisionEnabled(settings) {
|
|
3014
|
+
return settings.sandboxLazyProvisionEnabled === true && settings.sandboxOwnershipEnabled === true;
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
// src/activities/common.ts
|
|
3018
|
+
import {
|
|
3019
|
+
mergeResourceRefs,
|
|
3020
|
+
mergeToolRefs as mergeToolRefs2,
|
|
3021
|
+
reasoningEffortForMetadata as reasoningEffortForMetadata2
|
|
3022
|
+
} from "@opengeni/contracts";
|
|
3023
|
+
function assertReusableSessionRevivable(status) {
|
|
3024
|
+
if (status === "cancelled") {
|
|
3025
|
+
throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
|
|
3026
|
+
}
|
|
3027
|
+
}
|
|
3028
|
+
function scheduledUserMessagePayload(prompt, resources, tools, taskId, runId) {
|
|
3029
|
+
return {
|
|
3030
|
+
type: "scheduled_occurrence",
|
|
3031
|
+
text: prompt,
|
|
3032
|
+
scheduledTaskId: taskId,
|
|
3033
|
+
scheduledTaskRunId: runId,
|
|
3034
|
+
...resources.length ? { resources } : {},
|
|
3035
|
+
...tools.length ? { tools } : {}
|
|
3036
|
+
};
|
|
3037
|
+
}
|
|
3038
|
+
function workflowIdForSession(sessionId) {
|
|
3039
|
+
return `session-${sessionId}`;
|
|
3040
|
+
}
|
|
3041
|
+
|
|
3042
|
+
export {
|
|
3043
|
+
createSharedActivityServices,
|
|
3044
|
+
classifyCodexPin,
|
|
3045
|
+
CODEX_USAGE_EXHAUSTED_PCT,
|
|
3046
|
+
isCodexCredentialEligible,
|
|
3047
|
+
shardCredentialForSession,
|
|
3048
|
+
earliestCodexReset,
|
|
3049
|
+
authoritativeCodexCapacityResetAt,
|
|
3050
|
+
computeIdleDelayMs,
|
|
3051
|
+
computeReactiveRotationResume,
|
|
3052
|
+
chooseRotationActive,
|
|
3053
|
+
selectCodexCredentialLeaseForTurn,
|
|
3054
|
+
refreshCodexUsageAndRepairCapacityWaiters,
|
|
3055
|
+
signalCodexCapacityWakeTargets,
|
|
3056
|
+
signalPendingCodexCapacityWakeTargets,
|
|
3057
|
+
createCodexCapacityActivities,
|
|
3058
|
+
createGoalActivities,
|
|
3059
|
+
withFirstPartyTools,
|
|
3060
|
+
settingsWithPackSandboxImage,
|
|
3061
|
+
settingsWithRigImage,
|
|
3062
|
+
rigProviderImageSourceImage,
|
|
3063
|
+
settingsWithRigProviderImage,
|
|
3064
|
+
resolveWorkspacePackRuntime,
|
|
3065
|
+
resolveWorkspaceSkillLibraryRuntime,
|
|
3066
|
+
mergeRigDefaultVariableSetEnvironment,
|
|
3067
|
+
createRuntimeBatcher,
|
|
3068
|
+
currentActivityContext,
|
|
3069
|
+
startActivityHeartbeat,
|
|
3070
|
+
nextStreamEvent,
|
|
3071
|
+
notifyParentOfChildIdle,
|
|
3072
|
+
deliverFailedChildTurnToParent,
|
|
3073
|
+
reconcilePendingParentSystemUpdates,
|
|
3074
|
+
reconcilePendingSessionWorkflowWakes,
|
|
3075
|
+
sandboxLeaseHolderIdForAttempt,
|
|
3076
|
+
SandboxWarmingTimeoutError,
|
|
3077
|
+
waitForWarmSnapshot,
|
|
3078
|
+
maybePersistWarmWorkspaceSnapshot,
|
|
3079
|
+
resumeBoxForTurn,
|
|
3080
|
+
providerIdentityFromResumeState,
|
|
3081
|
+
wrapTurnBoxWithRouting,
|
|
3082
|
+
wrapLazyTurnBoxWithRouting,
|
|
3083
|
+
establishSelfhostedTurnSession,
|
|
3084
|
+
routingEnabled,
|
|
3085
|
+
lazyProvisionEnabled,
|
|
3086
|
+
assertReusableSessionRevivable,
|
|
3087
|
+
scheduledUserMessagePayload,
|
|
3088
|
+
workflowIdForSession,
|
|
3089
|
+
mergeResourceRefs
|
|
3090
|
+
};
|
|
3091
|
+
//# sourceMappingURL=chunk-KKCRGAGZ.js.map
|