@opengeni/runtime 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-BHA2SU3H.js → chunk-RYDCVTMW.js} +1959 -208
- package/dist/chunk-RYDCVTMW.js.map +1 -0
- package/dist/{index-otZkoLYq.d.ts → index-DIiiwQY2.d.ts} +693 -55
- package/dist/index.d.ts +293 -121
- package/dist/index.js +1819 -786
- package/dist/index.js.map +1 -1
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +79 -3
- package/package.json +21 -20
- package/src/codex-tool-search.ts +185 -31
- package/src/context-compaction.ts +78 -243
- package/src/history-sanitizer.ts +172 -20
- package/src/index.ts +2573 -897
- package/src/metrics.ts +28 -2
- package/src/sandbox/capabilities.ts +12 -3
- package/src/sandbox/channel-a.ts +420 -72
- package/src/sandbox/display-stack.ts +18 -4
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +398 -112
- package/src/sandbox/op-correlation.ts +70 -0
- package/src/sandbox/providers/index.ts +9 -3
- package/src/sandbox/providers/modal.ts +99 -24
- package/src/sandbox/providers/selfhosted.ts +1 -4
- package/src/sandbox/recording.ts +146 -90
- package/src/sandbox/routing/backend-resolver.ts +144 -10
- package/src/sandbox/routing/routing-session.ts +129 -25
- package/src/sandbox/select.ts +60 -8
- package/src/sandbox/selfhosted/capabilities.ts +59 -25
- package/src/sandbox/selfhosted/control-rpc.ts +160 -28
- package/src/sandbox/selfhosted/fault-rendering.ts +226 -0
- package/src/sandbox/selfhosted/op-observer.ts +54 -0
- package/src/sandbox/selfhosted/op-stream.ts +801 -0
- package/src/sandbox/selfhosted/op-testing.ts +468 -0
- package/src/sandbox/selfhosted/op-transport.ts +151 -0
- package/src/sandbox/selfhosted/retry-policy.ts +177 -0
- package/src/sandbox/selfhosted/session.ts +498 -55
- package/src/sandbox/selfhosted/testing.ts +60 -19
- package/src/sandbox/stream-port.ts +17 -5
- package/src/sandbox/stream-token.ts +8 -2
- package/src/sandbox/terminal-server.ts +18 -4
- package/src/sandbox-computer.ts +332 -70
- package/src/usage-telemetry.ts +73 -0
- package/dist/chunk-BHA2SU3H.js.map +0 -1
- package/src/image-history.ts +0 -149
package/src/sandbox/recording.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// @opengeni/runtime/sandbox — the recording loop (P4.3).
|
|
2
2
|
//
|
|
3
|
-
// ffmpeg x11grab on :0 → mp4/webm artifact on the box →
|
|
3
|
+
// ffmpeg x11grab on :0 → mp4/webm artifact on the box → direct scoped PUT to
|
|
4
4
|
// object storage → recording.available. The "agent films itself proving the fix"
|
|
5
5
|
// loop: ffmpeg reads exactly the :0 framebuffer the agent's computer-use draws to
|
|
6
6
|
// and the human watches over Channel B (zero projection).
|
|
@@ -9,22 +9,16 @@
|
|
|
9
9
|
// Temporal, NO worker RPC, NO actor. They live in the agent-loop-free leaf so the
|
|
10
10
|
// SAME process that already holds the resumed-by-id box (the agent turn's own
|
|
11
11
|
// activity for an on-turn recording, or the API in-process for an off-turn/manual
|
|
12
|
-
// finalize)
|
|
13
|
-
//
|
|
14
|
-
// activity result — the 256 MB-vs-payload-limit concern dissolves (F10).
|
|
12
|
+
// finalize) authorizes a direct box-to-storage upload. Recording bytes never
|
|
13
|
+
// enter worker memory or a Temporal payload (F10).
|
|
15
14
|
//
|
|
16
15
|
// ── Adversarial-review fixes folded in (module 05 §Adversarial) ──────────────
|
|
17
16
|
// F1 exec is OPTIONAL on Modal (only execCommand) — every command dual-paths.
|
|
18
|
-
// F3 exec/execCommand YIELDS — the
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
// FR the byte transfer is a DIRECT exec (`base64 <abs-path>`), NOT readFile:
|
|
24
|
-
// the recording lives at an absolute /tmp path (never the user's workspace
|
|
25
|
-
// /git tree), and readFile rejects paths outside the manifest workspace
|
|
26
|
-
// root ("escapes the workspace root"). The base64 exec passes
|
|
27
|
-
// maxOutputTokens:null so a large recording is never truncated.
|
|
17
|
+
// F3 exec/execCommand YIELDS — the stop and direct-upload commands are bounded.
|
|
18
|
+
// F8 size is gated on the box first (stat); over-limit artifacts fail instead
|
|
19
|
+
// of being uploaded partially.
|
|
20
|
+
// FR curl reads the absolute /tmp artifact in the box and sends it directly to
|
|
21
|
+
// the short-lived scoped URL. No workspace-root read API is involved.
|
|
28
22
|
// F9 the box file is deleted ONLY after the storage PUT confirms — never
|
|
29
23
|
// before (so a failed upload leaves the bytes recoverable for a retry).
|
|
30
24
|
// F12 ffmpeg/x11vnc backgrounding does not block the yield (nohup … & echo $!).
|
|
@@ -54,24 +48,44 @@ export function extForCodec(codec: RecordingCodec): string {
|
|
|
54
48
|
|
|
55
49
|
/** No exec/execCommand on the session — the box cannot run ffmpeg. */
|
|
56
50
|
export class RecordingUnavailableError extends Error {
|
|
57
|
-
constructor(message: string) {
|
|
51
|
+
constructor(message: string) {
|
|
52
|
+
super(message);
|
|
53
|
+
this.name = "RecordingUnavailableError";
|
|
54
|
+
}
|
|
58
55
|
}
|
|
59
|
-
/** ffmpeg failed, the file is missing, or the
|
|
56
|
+
/** ffmpeg failed, the file is missing, or the artifact is over its size limit. */
|
|
60
57
|
export class RecordingError extends Error {
|
|
61
|
-
constructor(
|
|
58
|
+
constructor(
|
|
59
|
+
message: string,
|
|
60
|
+
readonly reason: "ffmpeg-error" | "box-death" | "max-bytes-exceeded" | "display-unavailable",
|
|
61
|
+
) {
|
|
62
62
|
super(message);
|
|
63
63
|
this.name = "RecordingError";
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
// The structural slice of a provider session the recording loop drives. exec and
|
|
68
|
-
// execCommand are optional (Modal has only execCommand — F1)
|
|
69
|
-
|
|
70
|
-
|
|
68
|
+
// execCommand are optional (Modal has only execCommand — F1).
|
|
69
|
+
type ExecResultLike = {
|
|
70
|
+
output?: string;
|
|
71
|
+
stdout?: string;
|
|
72
|
+
stderr?: string;
|
|
73
|
+
exitCode?: number | null;
|
|
74
|
+
sessionId?: number;
|
|
75
|
+
};
|
|
71
76
|
type RecordingSession = {
|
|
72
|
-
exec?: (args: {
|
|
73
|
-
|
|
74
|
-
|
|
77
|
+
exec?: (args: {
|
|
78
|
+
cmd: string;
|
|
79
|
+
runAs?: string;
|
|
80
|
+
yieldTimeMs?: number;
|
|
81
|
+
maxOutputTokens?: number;
|
|
82
|
+
}) => Promise<ExecResultLike>;
|
|
83
|
+
execCommand?: (args: {
|
|
84
|
+
cmd: string;
|
|
85
|
+
runAs?: string;
|
|
86
|
+
yieldTimeMs?: number;
|
|
87
|
+
maxOutputTokens?: number;
|
|
88
|
+
}) => Promise<string>;
|
|
75
89
|
};
|
|
76
90
|
|
|
77
91
|
function shq(s: string): string {
|
|
@@ -80,11 +94,12 @@ function shq(s: string): string {
|
|
|
80
94
|
|
|
81
95
|
function resultOutput(result: ExecResultLike | string): string {
|
|
82
96
|
if (typeof result === "string") return result;
|
|
83
|
-
return [result.output, result.stderr, result.stdout]
|
|
97
|
+
return [result.output, result.stderr, result.stdout]
|
|
98
|
+
.filter((v): v is string => typeof v === "string" && v.length > 0)
|
|
99
|
+
.join("\n");
|
|
84
100
|
}
|
|
85
101
|
|
|
86
|
-
// Default per-command output cap (tokens).
|
|
87
|
-
// `null` (no truncation) so a base64-encoded recording is never clipped.
|
|
102
|
+
// Default per-command output cap (tokens). Recording transfer does not use stdout.
|
|
88
103
|
const DEFAULT_MAX_OUTPUT_TOKENS = 4_000;
|
|
89
104
|
|
|
90
105
|
async function run(
|
|
@@ -92,20 +107,23 @@ async function run(
|
|
|
92
107
|
cmd: string,
|
|
93
108
|
runAs?: string,
|
|
94
109
|
yieldTimeMs = EXEC_YIELD_MS,
|
|
95
|
-
maxOutputTokens
|
|
110
|
+
maxOutputTokens = DEFAULT_MAX_OUTPUT_TOKENS,
|
|
96
111
|
): Promise<string> {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
112
|
+
const args: {
|
|
113
|
+
cmd: string;
|
|
114
|
+
runAs?: string;
|
|
115
|
+
yieldTimeMs?: number;
|
|
116
|
+
maxOutputTokens?: number;
|
|
117
|
+
} = { cmd, ...(runAs ? { runAs } : {}), yieldTimeMs, maxOutputTokens };
|
|
102
118
|
if (typeof session.exec === "function") {
|
|
103
119
|
return resultOutput(await session.exec(args));
|
|
104
120
|
}
|
|
105
121
|
if (typeof session.execCommand === "function") {
|
|
106
122
|
return resultOutput(await session.execCommand(args));
|
|
107
123
|
}
|
|
108
|
-
throw new RecordingUnavailableError(
|
|
124
|
+
throw new RecordingUnavailableError(
|
|
125
|
+
"session cannot run commands (no exec/execCommand) — recording unavailable",
|
|
126
|
+
);
|
|
109
127
|
}
|
|
110
128
|
|
|
111
129
|
// Extract the command body from a provider exec banner. Modal's execCommand
|
|
@@ -149,7 +167,10 @@ export type RecordingProcess = {
|
|
|
149
167
|
* block on the recording). A hard `-t <maxSeconds>` ceiling bounds a runaway file
|
|
150
168
|
* across a multi-day turn. Returns the handle the caller carries to stop+finalize.
|
|
151
169
|
*/
|
|
152
|
-
export async function startRecording(
|
|
170
|
+
export async function startRecording(
|
|
171
|
+
session: unknown,
|
|
172
|
+
input: StartRecordingInput,
|
|
173
|
+
): Promise<RecordingProcess> {
|
|
153
174
|
const s = session as RecordingSession;
|
|
154
175
|
const codec = input.codec ?? "h264-mp4";
|
|
155
176
|
const dimensions = input.dimensions ?? DEFAULT_DIMENSIONS;
|
|
@@ -162,9 +183,10 @@ export async function startRecording(session: unknown, input: StartRecordingInpu
|
|
|
162
183
|
const pidFile = `${tmp}/og-rec-${input.recordingId}.pid`;
|
|
163
184
|
const logFile = `${tmp}/og-rec-${input.recordingId}.log`;
|
|
164
185
|
const [w, h] = dimensions;
|
|
165
|
-
const enc =
|
|
166
|
-
|
|
167
|
-
|
|
186
|
+
const enc =
|
|
187
|
+
codec === "vp9-webm"
|
|
188
|
+
? `-c:v libvpx-vp9 -b:v 0 -crf 32 -row-mt 1`
|
|
189
|
+
: `-c:v libx264 -preset veryfast -pix_fmt yuv420p -movflags +faststart`;
|
|
168
190
|
const ffmpeg =
|
|
169
191
|
`nohup ffmpeg -hide_banner -loglevel error -f x11grab -draw_mouse 1 -framerate ${framerate} ` +
|
|
170
192
|
`-video_size ${w}x${h} -i ${display}.0 -t ${maxSeconds} ${enc} ${boxPath} ` +
|
|
@@ -194,45 +216,41 @@ export async function stopRecording(session: unknown, proc: RecordingProcess): P
|
|
|
194
216
|
await run(s, `bash -lc ${shq(wait)}`, proc.runAs, STOP_YIELD_MS).catch(() => undefined);
|
|
195
217
|
}
|
|
196
218
|
|
|
197
|
-
export type
|
|
198
|
-
bytes: Uint8Array;
|
|
219
|
+
export type RecordingArtifactMetadata = {
|
|
199
220
|
contentType: RecordingContentType;
|
|
200
221
|
sizeBytes: number;
|
|
201
222
|
durationSeconds: number;
|
|
202
223
|
};
|
|
203
224
|
|
|
204
225
|
/**
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
* TRANSPORT: the bytes are read via a DIRECT exec (`base64 <path>` over stdout),
|
|
208
|
-
* NOT via session.readFile(). The recording artifact lives at an absolute /tmp
|
|
209
|
-
* path on purpose — recordings must never be written inside the user's workspace
|
|
210
|
-
* /git tree — but session.readFile() resolves every path against the manifest
|
|
211
|
-
* workspace root and rejects anything outside it ("Sandbox path … escapes the
|
|
212
|
-
* workspace root"), which fataled finalize. Raw exec runs unrestricted shell, so
|
|
213
|
-
* `base64` reads the /tmp file directly; we decode the base64 back to bytes here.
|
|
214
|
-
* The byte-read exec passes `maxOutputTokens: null` so the provider never
|
|
215
|
-
* truncates a large recording's base64.
|
|
216
|
-
*
|
|
217
|
-
* F8: we DO NOT assume any over-limit behavior. First `stat` the file size on the
|
|
218
|
-
* box; if it exceeds maxBytes, fail `max-bytes-exceeded` (never upload a truncated
|
|
219
|
-
* video). Otherwise read the raw bytes.
|
|
220
|
-
*
|
|
221
|
-
* F9: this does NOT delete the box file. The caller deletes it (deleteRecordingArtifacts)
|
|
222
|
-
* ONLY after the storage PUT + `available` commit — so a failed upload leaves the
|
|
223
|
-
* bytes recoverable on the box for a retry.
|
|
224
|
-
*
|
|
225
|
-
* F14: duration is wall-clock (now − startedAt), a close approximation of the
|
|
226
|
-
* SIGINT-flushed video length.
|
|
226
|
+
* Inspect and size-gate the finalized recording while it is still on the box.
|
|
227
|
+
* No recording bytes cross into the worker process.
|
|
227
228
|
*/
|
|
228
|
-
export async function
|
|
229
|
+
export async function inspectRecordingArtifact(
|
|
230
|
+
session: unknown,
|
|
231
|
+
proc: RecordingProcess,
|
|
232
|
+
maxBytes = DEFAULT_MAX_BYTES,
|
|
233
|
+
): Promise<RecordingArtifactMetadata> {
|
|
229
234
|
const s = session as RecordingSession;
|
|
230
235
|
if (typeof s.exec !== "function" && typeof s.execCommand !== "function") {
|
|
231
|
-
throw new RecordingUnavailableError(
|
|
236
|
+
throw new RecordingUnavailableError(
|
|
237
|
+
"session cannot run commands (no exec/execCommand) — recording finalize unavailable",
|
|
238
|
+
);
|
|
232
239
|
}
|
|
233
|
-
// F8: size-gate on the box before
|
|
234
|
-
const sizeOut = (
|
|
235
|
-
|
|
240
|
+
// F8: size-gate on the box before authorizing the upload.
|
|
241
|
+
const sizeOut = (
|
|
242
|
+
await run(
|
|
243
|
+
s,
|
|
244
|
+
`bash -lc ${shq(`stat -c %s ${proc.boxPath} 2>/dev/null || echo MISSING`)}`,
|
|
245
|
+
proc.runAs,
|
|
246
|
+
)
|
|
247
|
+
).trim();
|
|
248
|
+
const sizeLine =
|
|
249
|
+
sizeOut
|
|
250
|
+
.split("\n")
|
|
251
|
+
.map((l) => l.trim())
|
|
252
|
+
.filter(Boolean)
|
|
253
|
+
.pop() ?? "MISSING";
|
|
236
254
|
if (sizeLine === "MISSING" || sizeLine === "") {
|
|
237
255
|
throw new RecordingError(`recording file missing on box: ${proc.boxPath}`, "box-death");
|
|
238
256
|
}
|
|
@@ -243,44 +261,82 @@ export async function readRecordingBytes(session: unknown, proc: RecordingProces
|
|
|
243
261
|
if (size > maxBytes) {
|
|
244
262
|
throw new RecordingError(`recording ${size}B exceeds max ${maxBytes}B`, "max-bytes-exceeded");
|
|
245
263
|
}
|
|
246
|
-
// Read the bytes via a DIRECT exec (base64, no output truncation), so the
|
|
247
|
-
// absolute /tmp path is NOT run through the workspace-root-scoped readFile guard.
|
|
248
|
-
const STOP_YIELD = 60_000; // a large recording's base64 read may take longer than the default exec yield.
|
|
249
|
-
const encoded = stripExecBanner(
|
|
250
|
-
await run(s, `bash -lc ${shq(`base64 ${proc.boxPath}`)}`, proc.runAs, STOP_YIELD, null),
|
|
251
|
-
);
|
|
252
|
-
const base64 = encoded.replace(/\s+/g, "");
|
|
253
|
-
if (base64.length === 0) {
|
|
254
|
-
throw new RecordingError(`recording read returned 0 bytes: ${proc.boxPath}`, "ffmpeg-error");
|
|
255
|
-
}
|
|
256
|
-
let bytes: Uint8Array;
|
|
257
|
-
try {
|
|
258
|
-
bytes = Uint8Array.from(Buffer.from(base64, "base64"));
|
|
259
|
-
} catch (error) {
|
|
260
|
-
throw new RecordingError(`recording base64 decode failed: ${error instanceof Error ? error.message : String(error)}`, "ffmpeg-error");
|
|
261
|
-
}
|
|
262
|
-
if (bytes.length === 0) {
|
|
263
|
-
throw new RecordingError(`recording read returned 0 bytes: ${proc.boxPath}`, "ffmpeg-error");
|
|
264
|
-
}
|
|
265
264
|
return {
|
|
266
|
-
bytes,
|
|
267
265
|
contentType: contentTypeForCodec(proc.codec),
|
|
268
|
-
sizeBytes:
|
|
266
|
+
sizeBytes: size,
|
|
269
267
|
durationSeconds: Math.max(0, Math.round((Date.now() - proc.startedAt) / 1000)),
|
|
270
268
|
};
|
|
271
269
|
}
|
|
272
270
|
|
|
271
|
+
const RECORDING_UPLOAD_OK = "OPENGENI_RECORDING_UPLOAD_OK";
|
|
272
|
+
const RECORDING_UPLOAD_MAX_SECONDS = 300;
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Upload the finalized artifact directly from the sandbox to a scoped object-
|
|
276
|
+
* storage URL. The worker carries only the short URL/headers and metadata; it
|
|
277
|
+
* never materializes a base64 string or Uint8Array for a recording that may be
|
|
278
|
+
* hundreds of megabytes. curl's success marker makes an exec transport that
|
|
279
|
+
* returns non-zero command output instead of throwing still fail closed.
|
|
280
|
+
*/
|
|
281
|
+
export async function uploadRecordingArtifact(
|
|
282
|
+
session: unknown,
|
|
283
|
+
proc: RecordingProcess,
|
|
284
|
+
input: {
|
|
285
|
+
url: string;
|
|
286
|
+
requiredHeaders: Record<string, string>;
|
|
287
|
+
maxBytes?: number;
|
|
288
|
+
maxSeconds?: number;
|
|
289
|
+
},
|
|
290
|
+
): Promise<RecordingArtifactMetadata> {
|
|
291
|
+
const maxSeconds = input.maxSeconds ?? RECORDING_UPLOAD_MAX_SECONDS;
|
|
292
|
+
if (!Number.isSafeInteger(maxSeconds) || maxSeconds <= 0 || maxSeconds > 300) {
|
|
293
|
+
throw new Error("recording upload maxSeconds must be an integer between 1 and 300");
|
|
294
|
+
}
|
|
295
|
+
const metadata = await inspectRecordingArtifact(session, proc, input.maxBytes);
|
|
296
|
+
const headers = Object.entries(input.requiredHeaders)
|
|
297
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
298
|
+
.map(([name, value]) => `--header ${shq(`${name}: ${value}`)}`)
|
|
299
|
+
.join(" ");
|
|
300
|
+
const upload = [
|
|
301
|
+
"set -euo pipefail",
|
|
302
|
+
`curl --fail --silent --show-error --connect-timeout 20 --max-time ${maxSeconds} --request PUT ${headers} --upload-file ${shq(proc.boxPath)} ${shq(input.url)}`,
|
|
303
|
+
`printf '${RECORDING_UPLOAD_OK}\\n'`,
|
|
304
|
+
].join("; ");
|
|
305
|
+
const output = stripExecBanner(
|
|
306
|
+
await run(
|
|
307
|
+
session as RecordingSession,
|
|
308
|
+
`bash -lc ${shq(upload)}`,
|
|
309
|
+
proc.runAs,
|
|
310
|
+
(maxSeconds + 10) * 1_000,
|
|
311
|
+
),
|
|
312
|
+
);
|
|
313
|
+
if (!output.includes(RECORDING_UPLOAD_OK)) {
|
|
314
|
+
throw new Error("recording upload did not confirm success");
|
|
315
|
+
}
|
|
316
|
+
return metadata;
|
|
317
|
+
}
|
|
318
|
+
|
|
273
319
|
/**
|
|
274
320
|
* Delete the box artifacts. F9: call this ONLY after the storage PUT confirmed
|
|
275
321
|
* and the `available` row committed — never before. Best-effort; never throws.
|
|
276
322
|
*/
|
|
277
|
-
export async function deleteRecordingArtifacts(
|
|
323
|
+
export async function deleteRecordingArtifacts(
|
|
324
|
+
session: unknown,
|
|
325
|
+
proc: RecordingProcess,
|
|
326
|
+
): Promise<void> {
|
|
278
327
|
const s = session as RecordingSession;
|
|
279
328
|
const logFile = proc.boxPath.replace(/\.(mp4|webm)$/, ".log");
|
|
280
|
-
await run(s, `rm -f ${proc.boxPath} ${proc.pidFile} ${logFile}`, proc.runAs).catch(
|
|
329
|
+
await run(s, `rm -f ${proc.boxPath} ${proc.pidFile} ${logFile}`, proc.runAs).catch(
|
|
330
|
+
() => undefined,
|
|
331
|
+
);
|
|
281
332
|
}
|
|
282
333
|
|
|
283
334
|
/** The storage object key for a recording artifact (parallels the file-asset layout). */
|
|
284
|
-
export function recordingStorageKey(
|
|
335
|
+
export function recordingStorageKey(
|
|
336
|
+
workspaceId: string,
|
|
337
|
+
sessionId: string,
|
|
338
|
+
recordingId: string,
|
|
339
|
+
codec: RecordingCodec,
|
|
340
|
+
): string {
|
|
285
341
|
return `recordings/${workspaceId}/${sessionId}/${recordingId}.${extForCodec(codec)}`;
|
|
286
342
|
}
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
// API/worker pass a resume-by-id closure for the sibling box's lease).
|
|
20
20
|
|
|
21
21
|
import { buildSelfhostedBackendSession, type SelfhostedRelayConfig } from "../selfhosted/session";
|
|
22
|
+
import type { SelfhostedOpObserver } from "../selfhosted/op-observer";
|
|
22
23
|
import type { ControlRpc } from "../selfhosted/control-rpc";
|
|
23
24
|
import type {
|
|
24
25
|
ActivePointer,
|
|
@@ -60,8 +61,16 @@ export interface ActiveBackendResolverDeps {
|
|
|
60
61
|
* absent, a modal swap target surfaces as unsupported (the caller validated
|
|
61
62
|
* liveness, so this is the "modal swap not wired in this context" guard). */
|
|
62
63
|
establishModalTarget?: (sandbox: RoutableSandbox) => Promise<RoutableBackendSession>;
|
|
63
|
-
/**
|
|
64
|
+
/** The selfhosted CONTROL-op timeout (ping/fs/desktop/pty) for a swap/pin target.
|
|
65
|
+
* Absent ⇒ the session's 30s default. */
|
|
64
66
|
selfhostedTimeoutMs?: number;
|
|
67
|
+
/** The selfhosted EXEC process deadline for a swap/pin target, distinct from the
|
|
68
|
+
* control timeout (a swapped-to machine runs real commands too). Absent ⇒ falls
|
|
69
|
+
* back to the control timeout, as in the session leaf. */
|
|
70
|
+
selfhostedExecTimeoutMs?: number;
|
|
71
|
+
/** The per-op observer threaded into a selfhosted swap/pin target (out-of-band
|
|
72
|
+
* telemetry — metrics + machine.* events). Absent ⇒ no-op. */
|
|
73
|
+
selfhostedOnOp?: SelfhostedOpObserver;
|
|
65
74
|
/**
|
|
66
75
|
* The run's declared sandbox environment — the SAME `Record<string,string>` the
|
|
67
76
|
* worker turn threads into the agent's TARGET manifest (and into the group box at
|
|
@@ -86,15 +95,104 @@ export interface ActiveBackendResolverDeps {
|
|
|
86
95
|
* path (which always builds fresh — it has no pre-established turn session).
|
|
87
96
|
*/
|
|
88
97
|
pinnedSelfhosted?: { sandboxId: string; epoch: number; session: RoutableBackendSession };
|
|
98
|
+
/**
|
|
99
|
+
* Whether `defaultBackend` IS the session's home box — i.e. whether the null pointer
|
|
100
|
+
* (== home) may resolve to it. Defaults to TRUE (omitted): a genuine machine-HOME turn
|
|
101
|
+
* (home IS the machine) and a Modal-home turn established on its group box both pass/omit
|
|
102
|
+
* true, so null resolves to `defaultBackend` as before. Set explicitly FALSE only for a
|
|
103
|
+
* machine-primary turn of a Modal-HOME session (pinned to a machine, group box never
|
|
104
|
+
* established this turn): the null pointer then has nothing to resolve to THIS turn, so
|
|
105
|
+
* rather than silently keep serving the pinned machine as if the detach never happened,
|
|
106
|
+
* the null branch throws a typed `home_unavailable_this_turn` error — the detach's pointer
|
|
107
|
+
* commit stands and takes effect on the NEXT turn, which starts null and establishes the
|
|
108
|
+
* home box normally. Lazily establishing the home box mid-turn on such a clear is a
|
|
109
|
+
* deferred follow-up (issue #341); until then this makes the gap fail typed-and-specific,
|
|
110
|
+
* never silent.
|
|
111
|
+
*/
|
|
112
|
+
defaultIsHome?: boolean;
|
|
89
113
|
}
|
|
90
114
|
|
|
115
|
+
/** Why a persisted pointer / swap target cannot be turned into a live backend. A
|
|
116
|
+
* typed discriminant so callers (turn-start reconcile, the swap tool, future
|
|
117
|
+
* reports) can distinguish a stale pointer from an unaddressable enrollment from a
|
|
118
|
+
* backend the turn context simply cannot establish — instead of string-matching
|
|
119
|
+
* one opaque message (issue #341: "typed diagnostics that distinguish stale
|
|
120
|
+
* pointer, offline enrollment, unsupported backend context, transient failure").
|
|
121
|
+
* - `stale_pointer` — the pointed-at sandbox row is gone (deleted/orphaned).
|
|
122
|
+
* - `offline_enrollment` — a selfhosted target carries no enrollment/agent id to address.
|
|
123
|
+
* - `unsupported_backend_context` — a target no turn routing context can establish
|
|
124
|
+
* (a non-group Modal sibling; an unknown kind).
|
|
125
|
+
* - `transient_establishment` — a momentary establish failure worth a retry (reserved;
|
|
126
|
+
* control-plane timeouts surface as their own typed error).
|
|
127
|
+
* - `home_unavailable_this_turn` — the pointer was cleared to the session default (home)
|
|
128
|
+
* mid-turn, but this turn started pinned to a machine and
|
|
129
|
+
* never established the home box, so null has nothing to
|
|
130
|
+
* resolve to THIS turn (the detach takes effect next turn). */
|
|
131
|
+
export type BackendUnresolvableCode =
|
|
132
|
+
| "stale_pointer"
|
|
133
|
+
| "offline_enrollment"
|
|
134
|
+
| "unsupported_backend_context"
|
|
135
|
+
| "transient_establishment"
|
|
136
|
+
| "home_unavailable_this_turn";
|
|
137
|
+
|
|
91
138
|
/** Thrown when a swap target cannot be resolved (unknown sandbox, or a modal
|
|
92
|
-
* target with no establisher in this context). The caller maps it to a 409.
|
|
139
|
+
* target with no establisher in this context). The caller maps it to a 409.
|
|
140
|
+
* Carries a typed {@link BackendUnresolvableCode} in addition to the message. */
|
|
93
141
|
export class ActiveBackendUnresolvableError extends Error {
|
|
94
142
|
readonly name = "ActiveBackendUnresolvableError";
|
|
95
|
-
|
|
143
|
+
readonly code: BackendUnresolvableCode;
|
|
144
|
+
constructor(code: BackendUnresolvableCode, message: string) {
|
|
96
145
|
super(message);
|
|
146
|
+
this.code = code;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The outcome of {@link swapTargetEstablishability}. */
|
|
151
|
+
export type SwapTargetEstablishability =
|
|
152
|
+
| { ok: true }
|
|
153
|
+
| { ok: false; code: BackendUnresolvableCode; reason: string };
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* THE SINGLE SOURCE OF TRUTH for "can a turn's routing context ESTABLISH this swap
|
|
157
|
+
* target?" — shared by swap/seed ADMISSION (`fleet.resolveTarget`, before the CAS
|
|
158
|
+
* pointer commit) and turn-time ESTABLISHMENT (this resolver + `wrapTurnBoxWithRouting`).
|
|
159
|
+
* Admission MUST reject any target this predicate calls unestablishable, so the
|
|
160
|
+
* pointer is never committed to a backend no turn can resume (issue #341 Shape 1:
|
|
161
|
+
* a Modal-sibling swap was admitted + epoch-bumped, then every following op
|
|
162
|
+
* stranded because no context wires an establisher for it).
|
|
163
|
+
*
|
|
164
|
+
* - the session's OWN group box (`isSessionGroup`) is the null/home target and is
|
|
165
|
+
* always establishable (it IS the default backend);
|
|
166
|
+
* - a `selfhosted` machine target is establishable (`buildSelfhostedBackendSession`);
|
|
167
|
+
* admission's separate liveness probe — not this predicate — gates online-ness;
|
|
168
|
+
* - a NON-group `modal` target is NOT establishable: no turn context wires
|
|
169
|
+
* `establishModalTarget` (cross-group Modal resume-by-id is a future, billing-
|
|
170
|
+
* touching feature — sandbox-routing.ts). Wiring that establisher is the ONE
|
|
171
|
+
* toggle that flips this branch; keep admission and the resolver in lockstep.
|
|
172
|
+
*/
|
|
173
|
+
export function swapTargetEstablishability(target: {
|
|
174
|
+
kind: string;
|
|
175
|
+
isSessionGroup: boolean;
|
|
176
|
+
}): SwapTargetEstablishability {
|
|
177
|
+
if (target.isSessionGroup) {
|
|
178
|
+
return { ok: true };
|
|
179
|
+
}
|
|
180
|
+
if (target.kind === "selfhosted") {
|
|
181
|
+
return { ok: true };
|
|
182
|
+
}
|
|
183
|
+
if (target.kind === "modal") {
|
|
184
|
+
return {
|
|
185
|
+
ok: false,
|
|
186
|
+
code: "unsupported_backend_context",
|
|
187
|
+
reason:
|
|
188
|
+
"a Modal sandbox other than this session's own box cannot be established by a turn yet; swap back to the session default or attach a Connected Machine",
|
|
189
|
+
};
|
|
97
190
|
}
|
|
191
|
+
return {
|
|
192
|
+
ok: false,
|
|
193
|
+
code: "unsupported_backend_context",
|
|
194
|
+
reason: `a sandbox of kind "${target.kind}" cannot be established by a turn`,
|
|
195
|
+
};
|
|
98
196
|
}
|
|
99
197
|
|
|
100
198
|
/**
|
|
@@ -115,9 +213,29 @@ export function makeActiveBackendResolver(
|
|
|
115
213
|
deps: ActiveBackendResolverDeps,
|
|
116
214
|
): (pointer: ActivePointer) => Promise<ResolvedActiveBackend> {
|
|
117
215
|
return async (pointer: ActivePointer): Promise<ResolvedActiveBackend> => {
|
|
118
|
-
//
|
|
119
|
-
//
|
|
216
|
+
// NULL == the session's HOME backend (issue #341 invariant 1). `defaultBackend`
|
|
217
|
+
// is the already-established home the wiring handed us — the Modal group box for a
|
|
218
|
+
// Modal-home turn, or the machine itself for a machine-home turn. It is NEVER a
|
|
219
|
+
// swap target's SelfhostedSession: a clear-to-null must re-land on the home, never
|
|
220
|
+
// keep serving the machine a prior op swapped to. (The proxy caches by the FULL
|
|
221
|
+
// pointer tuple, so a clear-to-null — epoch-bumped OR a same-epoch FK null —
|
|
222
|
+
// invalidates the cached machine and re-enters this branch.) The lease owns the
|
|
223
|
+
// home box lifecycle; the proxy does not re-establish it.
|
|
120
224
|
if (pointer.activeSandboxId === null) {
|
|
225
|
+
// A machine-pinned turn (Modal-home session pinned to a machine) never
|
|
226
|
+
// established its home box, so a mid-turn clear-to-null has no home to resolve
|
|
227
|
+
// THIS turn. Fail typed-and-specific — the detach was accepted and its pointer
|
|
228
|
+
// commit STANDS, taking effect on the next turn (which starts null and
|
|
229
|
+
// establishes the home box) — instead of silently serving the pinned machine as
|
|
230
|
+
// if the clear never happened. Only an EXPLICIT `false` throws; omitted (the
|
|
231
|
+
// common path) resolves to the home default as before. (Lazily establishing the
|
|
232
|
+
// home box mid-turn on such a clear is a deferred follow-up; issue #341.)
|
|
233
|
+
if (deps.defaultIsHome === false) {
|
|
234
|
+
throw new ActiveBackendUnresolvableError(
|
|
235
|
+
"home_unavailable_this_turn",
|
|
236
|
+
"the active sandbox was detached to the session default (home), but this turn started pinned to a machine and established no home box; the detach is accepted and takes effect on the next turn — this turn has no active home box",
|
|
237
|
+
);
|
|
238
|
+
}
|
|
121
239
|
return { session: deps.defaultBackend, sandboxId: null, kind: deps.defaultKind };
|
|
122
240
|
}
|
|
123
241
|
|
|
@@ -129,16 +247,21 @@ export function makeActiveBackendResolver(
|
|
|
129
247
|
// to a fresh build fenced under the CURRENT epoch (the stale pinned instance is
|
|
130
248
|
// fenced at the old epoch and must not be reused).
|
|
131
249
|
if (
|
|
132
|
-
deps.pinnedSelfhosted
|
|
133
|
-
|
|
134
|
-
|
|
250
|
+
deps.pinnedSelfhosted &&
|
|
251
|
+
pointer.activeSandboxId === deps.pinnedSelfhosted.sandboxId &&
|
|
252
|
+
pointer.activeEpoch === deps.pinnedSelfhosted.epoch
|
|
135
253
|
) {
|
|
136
|
-
return {
|
|
254
|
+
return {
|
|
255
|
+
session: deps.pinnedSelfhosted.session,
|
|
256
|
+
sandboxId: pointer.activeSandboxId,
|
|
257
|
+
kind: "selfhosted",
|
|
258
|
+
};
|
|
137
259
|
}
|
|
138
260
|
|
|
139
261
|
const sandbox = await deps.getSandbox(pointer.activeSandboxId);
|
|
140
262
|
if (!sandbox) {
|
|
141
263
|
throw new ActiveBackendUnresolvableError(
|
|
264
|
+
"stale_pointer",
|
|
142
265
|
`active sandbox ${pointer.activeSandboxId} not found in workspace ${deps.workspaceId}`,
|
|
143
266
|
);
|
|
144
267
|
}
|
|
@@ -146,6 +269,7 @@ export function makeActiveBackendResolver(
|
|
|
146
269
|
if (sandbox.kind === "selfhosted") {
|
|
147
270
|
if (!sandbox.enrollmentId) {
|
|
148
271
|
throw new ActiveBackendUnresolvableError(
|
|
272
|
+
"offline_enrollment",
|
|
149
273
|
`selfhosted sandbox ${sandbox.id} has no enrollment (agent id) to address`,
|
|
150
274
|
);
|
|
151
275
|
}
|
|
@@ -161,6 +285,10 @@ export function makeActiveBackendResolver(
|
|
|
161
285
|
agentId: sandbox.enrollmentId,
|
|
162
286
|
epoch: pointer.activeEpoch,
|
|
163
287
|
...(deps.selfhostedTimeoutMs !== undefined ? { timeoutMs: deps.selfhostedTimeoutMs } : {}),
|
|
288
|
+
...(deps.selfhostedExecTimeoutMs !== undefined
|
|
289
|
+
? { execTimeoutMs: deps.selfhostedExecTimeoutMs }
|
|
290
|
+
: {}),
|
|
291
|
+
...(deps.selfhostedOnOp !== undefined ? { onOp: deps.selfhostedOnOp } : {}),
|
|
164
292
|
// The turn's declared environment → the session's manifest.environment, so
|
|
165
293
|
// the SDK's per-turn manifest-env delta is empty (no "cannot change manifest
|
|
166
294
|
// environment variables" throw on a pin-to-vm turn).
|
|
@@ -169,12 +297,17 @@ export function makeActiveBackendResolver(
|
|
|
169
297
|
// for this selfhosted backend. Absent/empty ⇒ the default workspace_root.
|
|
170
298
|
...(pointer.workingDir ? { workingDir: pointer.workingDir } : {}),
|
|
171
299
|
});
|
|
172
|
-
return {
|
|
300
|
+
return {
|
|
301
|
+
session: session as RoutableBackendSession,
|
|
302
|
+
sandboxId: sandbox.id,
|
|
303
|
+
kind: "selfhosted",
|
|
304
|
+
};
|
|
173
305
|
}
|
|
174
306
|
|
|
175
307
|
if (sandbox.kind === "modal") {
|
|
176
308
|
if (!deps.establishModalTarget) {
|
|
177
309
|
throw new ActiveBackendUnresolvableError(
|
|
310
|
+
"unsupported_backend_context",
|
|
178
311
|
`modal swap target ${sandbox.id} cannot be established in this context (no establisher wired)`,
|
|
179
312
|
);
|
|
180
313
|
}
|
|
@@ -183,6 +316,7 @@ export function makeActiveBackendResolver(
|
|
|
183
316
|
}
|
|
184
317
|
|
|
185
318
|
throw new ActiveBackendUnresolvableError(
|
|
319
|
+
"unsupported_backend_context",
|
|
186
320
|
`unsupported swap target kind "${sandbox.kind}" for sandbox ${sandbox.id}`,
|
|
187
321
|
);
|
|
188
322
|
};
|