@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/agent-service-sandbox.d.ts +7 -1
- package/dist/agent-service-sandbox.js +417 -39
- package/dist/agent-service-ws-protocol.d.ts +3 -3
- package/dist/agent-service-ws-protocol.js +5 -2
- package/dist/cli.js +19 -1
- package/dist/file-candidates.js +46 -39
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/run-dispatcher.d.ts +1 -1
- package/dist/run-dispatcher.js +10 -7
- package/dist/runtime-env.js +4 -4
- package/dist/runtime-quiescence.d.ts +19 -0
- package/dist/runtime-quiescence.js +130 -0
- package/dist/runtimes/deepseek-tui.js +11 -0
- package/dist/tool-observation.d.ts +6 -5
- package/dist/tool-observation.js +8 -5
- package/dist/trace-projection.d.ts +1 -1
- package/dist/trace-projection.js +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/workspace-filesystem.d.ts +20 -0
- package/dist/workspace-filesystem.js +120 -0
- package/dist/workspace-quota.d.ts +4 -0
- package/dist/workspace-quota.js +42 -0
- package/dist/workspace-restore.d.ts +9 -1
- package/dist/workspace-restore.js +30 -16
- package/dist/workspace-snapshot-control.d.ts +29 -0
- package/dist/workspace-snapshot-control.js +169 -0
- package/dist/workspace-snapshot-policy.d.ts +24 -0
- package/dist/workspace-snapshot-policy.js +45 -0
- package/dist/workspace-snapshot-staging.d.ts +2 -9
- package/dist/workspace-snapshot-staging.js +59 -25
- package/dist/workspace.d.ts +4 -4
- package/dist/workspace.js +26 -11
- package/package.json +1 -1
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { readFileSync, statfsSync, statSync } from "node:fs";
|
|
3
|
+
import { promisify } from "node:util";
|
|
4
|
+
import { assertRuntimeWorkspaceQuota } from "./workspace-quota.js";
|
|
5
|
+
export const WORKSPACE_IMAGE_FORMAT = "botlearn-workspace-image/1";
|
|
6
|
+
export const WORKSPACE_IMAGE_LOGICAL_BYTES = 3 * 1024 * 1024 * 1024;
|
|
7
|
+
const RUNTIME_LAUNCHER = "/opt/botlearn/bin/botlearn-runtime-launcher";
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
const defaultCapacityProbes = {
|
|
10
|
+
statfsBytes(workspaceRoot) {
|
|
11
|
+
const filesystem = statfsSync(workspaceRoot, { bigint: true });
|
|
12
|
+
return filesystem.blocks * filesystem.bsize;
|
|
13
|
+
},
|
|
14
|
+
backingFileBytes(imagePath) {
|
|
15
|
+
return BigInt(statSync(imagePath).size);
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
function decodeMountInfoPath(value) {
|
|
19
|
+
return value.replace(/\\([0-7]{3})/gu, (_match, octal) => String.fromCharCode(Number.parseInt(octal, 8)));
|
|
20
|
+
}
|
|
21
|
+
export function parseWorkspaceMountInfo(text) {
|
|
22
|
+
return text.split("\n").filter(Boolean).map((line) => {
|
|
23
|
+
const [left, right] = line.split(" - ", 2);
|
|
24
|
+
const fields = left?.split(" ") ?? [];
|
|
25
|
+
const filesystem = right?.split(" ") ?? [];
|
|
26
|
+
if (fields.length < 6 || filesystem.length < 3) {
|
|
27
|
+
throw new Error("workspace_mount_proof_invalid");
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
target: decodeMountInfoPath(fields[4]),
|
|
31
|
+
filesystemType: filesystem[0],
|
|
32
|
+
source: decodeMountInfoPath(filesystem[1]),
|
|
33
|
+
mountOptions: new Set(fields[5].split(",")),
|
|
34
|
+
superOptions: new Set(filesystem[2].split(",")),
|
|
35
|
+
};
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
export function nasWorkspaceEnabled(env = process.env) {
|
|
39
|
+
return env.BOTLEARN_WORKSPACE_IMAGE_FORMAT !== undefined;
|
|
40
|
+
}
|
|
41
|
+
export function workspaceFilesystemFlushCommand(env = process.env) {
|
|
42
|
+
if (!nasWorkspaceEnabled(env))
|
|
43
|
+
return null;
|
|
44
|
+
if (env.BOTLEARN_RUNTIME_LAUNCH_MODE === "direct-uid") {
|
|
45
|
+
return [RUNTIME_LAUNCHER, ["--workspace-filesystem-flush"]];
|
|
46
|
+
}
|
|
47
|
+
if (env.BOTLEARN_RUNTIME_LAUNCH_MODE === "sudo") {
|
|
48
|
+
return [
|
|
49
|
+
"/usr/bin/sudo",
|
|
50
|
+
["-n", "--", RUNTIME_LAUNCHER, "--workspace-filesystem-flush"],
|
|
51
|
+
];
|
|
52
|
+
}
|
|
53
|
+
if (env.BOTLEARN_DAEMON_ENABLE_FAKE_RUNTIME === "1" || env.NODE_ENV === "test") {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
throw new Error("workspace_nas_flush_failed");
|
|
57
|
+
}
|
|
58
|
+
export async function flushWorkspaceFilesystem(env = process.env) {
|
|
59
|
+
const command = workspaceFilesystemFlushCommand(env);
|
|
60
|
+
if (command === null)
|
|
61
|
+
return;
|
|
62
|
+
try {
|
|
63
|
+
await execFileAsync(command[0], command[1], { timeout: 30_000 });
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
throw new Error("workspace_nas_flush_failed");
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
export function isWorkspaceQuotaExceededError(error) {
|
|
70
|
+
if (!(error instanceof Error))
|
|
71
|
+
return false;
|
|
72
|
+
const candidate = error;
|
|
73
|
+
const code = String(candidate.code ?? candidate.failure?.error_code ?? "");
|
|
74
|
+
const detail = [
|
|
75
|
+
candidate.message,
|
|
76
|
+
candidate.failure?.error_message,
|
|
77
|
+
candidate.failure?.stderr_tail,
|
|
78
|
+
].map((value) => String(value ?? "")).join(" ");
|
|
79
|
+
return code === "ENOSPC"
|
|
80
|
+
|| code === "EDQUOT"
|
|
81
|
+
|| /\b(?:ENOSPC|EDQUOT|no space left on device|disk quota exceeded)\b/iu.test(detail)
|
|
82
|
+
|| (candidate.cause !== undefined && isWorkspaceQuotaExceededError(candidate.cause));
|
|
83
|
+
}
|
|
84
|
+
export function assertWorkspaceFilesystem(workspaceRoot, env = process.env, mountInfo, capacityProbes) {
|
|
85
|
+
if (!nasWorkspaceEnabled(env)) {
|
|
86
|
+
assertRuntimeWorkspaceQuota(workspaceRoot);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
if (env.BOTLEARN_WORKSPACE_IMAGE_FORMAT !== WORKSPACE_IMAGE_FORMAT
|
|
90
|
+
|| env.BOTLEARN_WORKSPACE_LOGICAL_BYTES !== String(WORKSPACE_IMAGE_LOGICAL_BYTES)
|
|
91
|
+
|| !/^[0-9a-f]{64}$/u.test(env.BOTLEARN_WORKSPACE_SCOPE_DIGEST ?? "")) {
|
|
92
|
+
throw new Error("workspace_quota_mismatch");
|
|
93
|
+
}
|
|
94
|
+
if (env.NODE_ENV === "test" && mountInfo === undefined)
|
|
95
|
+
return;
|
|
96
|
+
const mount = parseWorkspaceMountInfo(mountInfo ?? readFileSync("/proc/self/mountinfo", "utf8"))
|
|
97
|
+
.find((candidate) => candidate.target === workspaceRoot);
|
|
98
|
+
const validFilesystem = mount?.filesystemType === "ext4" || (mount?.filesystemType === "fuse.ext4"
|
|
99
|
+
&& mount.source === "/mnt/botlearn-workspace-store/workspace.ext4"
|
|
100
|
+
&& mount.superOptions.has("allow_other"));
|
|
101
|
+
if (!mount
|
|
102
|
+
|| !validFilesystem
|
|
103
|
+
|| !mount.mountOptions.has("rw")
|
|
104
|
+
|| !mount.mountOptions.has("nodev")
|
|
105
|
+
|| !mount.mountOptions.has("nosuid")) {
|
|
106
|
+
throw new Error("workspace_loop_mount_failed");
|
|
107
|
+
}
|
|
108
|
+
if (env.NODE_ENV === "test" && capacityProbes === undefined)
|
|
109
|
+
return;
|
|
110
|
+
const probes = capacityProbes ?? defaultCapacityProbes;
|
|
111
|
+
// fuse2fs reports a zero-sized statfs view in veFaaS even though the exact backing
|
|
112
|
+
// image is mounted and enforces the filesystem boundary. The mount proof above pins
|
|
113
|
+
// that image, so its validated file size is the authoritative capacity for FUSE.
|
|
114
|
+
const boundedBytes = mount.filesystemType === "fuse.ext4"
|
|
115
|
+
? probes.backingFileBytes(mount.source)
|
|
116
|
+
: probes.statfsBytes(workspaceRoot);
|
|
117
|
+
if (boundedBytes !== BigInt(WORKSPACE_IMAGE_LOGICAL_BYTES)) {
|
|
118
|
+
throw new Error("workspace_quota_mismatch");
|
|
119
|
+
}
|
|
120
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const RUNTIME_WORKSPACE_QUOTA_BYTES: number;
|
|
2
|
+
export declare const RUNTIME_WORKSPACE_INODE_QUOTA = 2000;
|
|
3
|
+
export declare function assertRuntimeWorkspaceQuota(workspaceRoot: string): void;
|
|
4
|
+
export declare function applyRuntimeWorkspaceQuota(runtimeSessionId: string, sandboxGeneration: number): Promise<void>;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { readFileSync, statfsSync, statSync } from "node:fs";
|
|
3
|
+
import { promisify } from "node:util";
|
|
4
|
+
export const RUNTIME_WORKSPACE_QUOTA_BYTES = 512 * 1024 * 1024;
|
|
5
|
+
export const RUNTIME_WORKSPACE_INODE_QUOTA = 2_000;
|
|
6
|
+
const WORKSPACE_BACKING_BYTES = 3 * 1024 * 1024 * 1024;
|
|
7
|
+
const QUOTA_MARKER = "/var/lib/botlearn-runtime/workspace-quota.ready";
|
|
8
|
+
const RUNTIME_LAUNCHER = "/opt/botlearn/bin/botlearn-runtime-launcher";
|
|
9
|
+
const execFileAsync = promisify(execFile);
|
|
10
|
+
export function assertRuntimeWorkspaceQuota(workspaceRoot) {
|
|
11
|
+
if (process.env.BOTLEARN_DAEMON_ENABLE_FAKE_RUNTIME === "1" || process.env.NODE_ENV === "test") {
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
if (process.env.BOTLEARN_RUNTIME_WORKSPACE_QUOTA_BYTES !== String(RUNTIME_WORKSPACE_QUOTA_BYTES) ||
|
|
15
|
+
process.env.BOTLEARN_RUNTIME_WORKSPACE_INODE_QUOTA !== String(RUNTIME_WORKSPACE_INODE_QUOTA)) {
|
|
16
|
+
throw new Error("workspace_runtime_quota_unverified");
|
|
17
|
+
}
|
|
18
|
+
const filesystem = statfsSync(workspaceRoot, { bigint: true });
|
|
19
|
+
const marker = statSync(QUOTA_MARKER);
|
|
20
|
+
if (filesystem.blocks * filesystem.bsize < BigInt(WORKSPACE_BACKING_BYTES) ||
|
|
21
|
+
marker.uid !== 0 || (marker.mode & 0o777) !== 0o600 ||
|
|
22
|
+
readFileSync(QUOTA_MARKER, "utf8") !== "workspace-filesystem-quota/1\n") {
|
|
23
|
+
throw new Error("workspace_runtime_quota_probe_failed");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export async function applyRuntimeWorkspaceQuota(runtimeSessionId, sandboxGeneration) {
|
|
27
|
+
if (process.env.BOTLEARN_DAEMON_ENABLE_FAKE_RUNTIME === "1" || process.env.NODE_ENV === "test") {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const args = ["--workspace-quota-apply", runtimeSessionId, String(sandboxGeneration)];
|
|
31
|
+
if (process.env.BOTLEARN_RUNTIME_LAUNCH_MODE === "sudo") {
|
|
32
|
+
await execFileAsync("/usr/bin/sudo", ["-n", "--", RUNTIME_LAUNCHER, ...args], {
|
|
33
|
+
timeout: 10_000,
|
|
34
|
+
});
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
if (process.env.BOTLEARN_RUNTIME_LAUNCH_MODE === "direct-uid") {
|
|
38
|
+
await execFileAsync(RUNTIME_LAUNCHER, args, { timeout: 10_000 });
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
throw new Error("workspace_runtime_quota_helper_unavailable");
|
|
42
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { type WorkspaceEntry, type WorkspaceEntrySetLimits } from "./workspace-entry-set.js";
|
|
2
2
|
export declare class WorkspaceRestoreError extends Error {
|
|
3
3
|
readonly code: string;
|
|
4
|
-
|
|
4
|
+
readonly retryable: boolean;
|
|
5
|
+
constructor(code: string, retryable?: boolean);
|
|
5
6
|
}
|
|
6
7
|
export interface WorkspaceRestoreDescriptor {
|
|
7
8
|
sandboxId: string;
|
|
@@ -25,6 +26,12 @@ export interface WorkspaceRestoreOptions {
|
|
|
25
26
|
requiredFreeBytes: number;
|
|
26
27
|
downloadFile(entry: WorkspaceEntry, partPath: string): Promise<void>;
|
|
27
28
|
}
|
|
29
|
+
export interface WorkspaceRestoreCapacityOptions {
|
|
30
|
+
workspaceDirectory: string;
|
|
31
|
+
descriptor: WorkspaceRestoreDescriptor;
|
|
32
|
+
requiredFreeBytes: number;
|
|
33
|
+
entryCopies?: 1 | 2;
|
|
34
|
+
}
|
|
28
35
|
export declare function workspaceRestoreJournalPath(controlDirectory: string, runtimeSessionId: string): string;
|
|
29
36
|
export declare function recoverWorkspaceRestoreSwap(options: {
|
|
30
37
|
workspaceDirectory: string;
|
|
@@ -32,3 +39,4 @@ export declare function recoverWorkspaceRestoreSwap(options: {
|
|
|
32
39
|
descriptor: WorkspaceRestoreDescriptor;
|
|
33
40
|
}): boolean;
|
|
34
41
|
export declare function restoreWorkspaceSnapshot(options: WorkspaceRestoreOptions): Promise<void>;
|
|
42
|
+
export declare function assertWorkspaceRestoreCapacity(options: WorkspaceRestoreCapacityOptions): void;
|
|
@@ -9,14 +9,18 @@ const RESTORE_JOURNAL_SCHEMA = "agent-workspace-restore-journal/1";
|
|
|
9
9
|
const COPY_CHUNK_BYTES = 64 * 1024;
|
|
10
10
|
export class WorkspaceRestoreError extends Error {
|
|
11
11
|
code;
|
|
12
|
-
|
|
12
|
+
retryable;
|
|
13
|
+
constructor(code, retryable = false) {
|
|
13
14
|
super(code);
|
|
14
15
|
this.code = code;
|
|
16
|
+
this.retryable = retryable;
|
|
15
17
|
this.name = "WorkspaceRestoreError";
|
|
16
18
|
}
|
|
17
19
|
}
|
|
18
20
|
function fail(code) {
|
|
19
|
-
throw new WorkspaceRestoreError(code
|
|
21
|
+
throw new WorkspaceRestoreError(code, code === "workspace_restore_unavailable" ||
|
|
22
|
+
code === "workspace_restore_disk_pressure" ||
|
|
23
|
+
code === "workspace_restore_inode_pressure");
|
|
20
24
|
}
|
|
21
25
|
function assertUuid(value, label) {
|
|
22
26
|
if (!UUID.test(value))
|
|
@@ -229,9 +233,7 @@ export async function restoreWorkspaceSnapshot(options) {
|
|
|
229
233
|
const descriptor = options.descriptor;
|
|
230
234
|
assertDescriptor(descriptor);
|
|
231
235
|
assertControlPath(options.workspaceDirectory, options.controlDirectory);
|
|
232
|
-
|
|
233
|
-
options.requiredFreeBytes < descriptor.expectedTotalBytes)
|
|
234
|
-
fail("workspace_restore_reservation_invalid");
|
|
236
|
+
assertWorkspaceRestoreCapacity(options);
|
|
235
237
|
recoverWorkspaceRestoreSwap({
|
|
236
238
|
workspaceDirectory: options.workspaceDirectory,
|
|
237
239
|
controlDirectory: options.controlDirectory,
|
|
@@ -244,17 +246,6 @@ export async function restoreWorkspaceSnapshot(options) {
|
|
|
244
246
|
(descriptor.revision > 0 && canonical.contentSha256 !== descriptor.contentSha256))
|
|
245
247
|
fail("workspace_restore_integrity_failed");
|
|
246
248
|
const parent = path.dirname(options.workspaceDirectory);
|
|
247
|
-
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
248
|
-
if (existsSync(options.workspaceDirectory)) {
|
|
249
|
-
const current = lstatSync(options.workspaceDirectory, { bigint: true });
|
|
250
|
-
if (!current.isDirectory() || current.isSymbolicLink()) {
|
|
251
|
-
fail("workspace_restore_existing_workspace_invalid");
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
const filesystem = statfsSync(parent, { bigint: true });
|
|
255
|
-
if (filesystem.bavail * filesystem.bsize < BigInt(options.requiredFreeBytes)) {
|
|
256
|
-
fail("workspace_restore_disk_pressure");
|
|
257
|
-
}
|
|
258
249
|
const nonce = randomUUID();
|
|
259
250
|
const stagingName = `.botlearn-restore-${descriptor.runtimeSessionId}-${nonce}`;
|
|
260
251
|
const backupName = `.botlearn-backup-${descriptor.runtimeSessionId}-${nonce}`;
|
|
@@ -331,3 +322,26 @@ export async function restoreWorkspaceSnapshot(options) {
|
|
|
331
322
|
throw error;
|
|
332
323
|
}
|
|
333
324
|
}
|
|
325
|
+
export function assertWorkspaceRestoreCapacity(options) {
|
|
326
|
+
const { descriptor } = options;
|
|
327
|
+
assertDescriptor(descriptor);
|
|
328
|
+
if (!Number.isSafeInteger(options.requiredFreeBytes) ||
|
|
329
|
+
options.requiredFreeBytes < descriptor.expectedTotalBytes)
|
|
330
|
+
fail("workspace_restore_reservation_invalid");
|
|
331
|
+
const parent = path.dirname(options.workspaceDirectory);
|
|
332
|
+
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
333
|
+
if (existsSync(options.workspaceDirectory)) {
|
|
334
|
+
const current = lstatSync(options.workspaceDirectory, { bigint: true });
|
|
335
|
+
if (!current.isDirectory() || current.isSymbolicLink()) {
|
|
336
|
+
fail("workspace_restore_existing_workspace_invalid");
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
const filesystem = statfsSync(parent, { bigint: true });
|
|
340
|
+
if (filesystem.bavail * filesystem.bsize < BigInt(options.requiredFreeBytes)) {
|
|
341
|
+
fail("workspace_restore_disk_pressure");
|
|
342
|
+
}
|
|
343
|
+
const entryCopies = options.entryCopies ?? 1;
|
|
344
|
+
if (filesystem.ffree < BigInt(descriptor.expectedEntryCount * entryCopies + 16)) {
|
|
345
|
+
fail("workspace_restore_inode_pressure");
|
|
346
|
+
}
|
|
347
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { WorkspaceWriterFreezeProof } from "./runtime-quiescence.js";
|
|
2
|
+
interface WorkspaceCheckpointScope {
|
|
3
|
+
checkpointId: string;
|
|
4
|
+
baseRevision: number;
|
|
5
|
+
sandboxId: string;
|
|
6
|
+
sandboxGeneration: number;
|
|
7
|
+
connectionEpoch: number;
|
|
8
|
+
runtimeSessionId: string;
|
|
9
|
+
agentRunId?: string;
|
|
10
|
+
workerAttempt?: number;
|
|
11
|
+
activationId?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare class WorkspaceSnapshotControlError extends Error {
|
|
14
|
+
readonly code: string;
|
|
15
|
+
readonly retryable: boolean;
|
|
16
|
+
constructor(code: string, retryable: boolean);
|
|
17
|
+
}
|
|
18
|
+
export declare function checkpointWorkspace(options: {
|
|
19
|
+
wsUrl: string;
|
|
20
|
+
reconnectToken: string;
|
|
21
|
+
workspaceDirectory: string;
|
|
22
|
+
scope: WorkspaceCheckpointScope;
|
|
23
|
+
freezeProof: WorkspaceWriterFreezeProof;
|
|
24
|
+
}): Promise<{
|
|
25
|
+
revision: number;
|
|
26
|
+
snapshotId: string | null;
|
|
27
|
+
contentSha256: string;
|
|
28
|
+
}>;
|
|
29
|
+
export {};
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { createReadStream, rmSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { ensureDaemonHome } from "./auth-store.js";
|
|
4
|
+
import { writeWorkspaceMaterializationMarker } from "./workspace-materialization.js";
|
|
5
|
+
import { WORKSPACE_SNAPSHOT_POLICY_V1 } from "./workspace-snapshot-policy.js";
|
|
6
|
+
import { stageWorkspaceSnapshot } from "./workspace-snapshot-staging.js";
|
|
7
|
+
export class WorkspaceSnapshotControlError extends Error {
|
|
8
|
+
code;
|
|
9
|
+
retryable;
|
|
10
|
+
constructor(code, retryable) {
|
|
11
|
+
super(code);
|
|
12
|
+
this.code = code;
|
|
13
|
+
this.retryable = retryable;
|
|
14
|
+
this.name = "WorkspaceSnapshotControlError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function controlBase(wsUrl) {
|
|
18
|
+
const url = new URL(wsUrl);
|
|
19
|
+
url.protocol = url.protocol === "wss:" ? "https:" : "http:";
|
|
20
|
+
url.pathname = "/internal/v1/workspace-checkpoints";
|
|
21
|
+
url.search = "";
|
|
22
|
+
url.hash = "";
|
|
23
|
+
return url.toString().replace(/\/$/, "");
|
|
24
|
+
}
|
|
25
|
+
async function requestJson(url, token, body, signal) {
|
|
26
|
+
const response = await fetch(url, {
|
|
27
|
+
method: "POST",
|
|
28
|
+
headers: {
|
|
29
|
+
Authorization: `Bearer ${token}`,
|
|
30
|
+
"Content-Type": "application/json",
|
|
31
|
+
},
|
|
32
|
+
body: JSON.stringify(body),
|
|
33
|
+
signal,
|
|
34
|
+
});
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
let code = `workspace_snapshot_control_${response.status}`;
|
|
37
|
+
let explicitRetryable;
|
|
38
|
+
try {
|
|
39
|
+
const payload = await response.json();
|
|
40
|
+
const detail = payload.detail;
|
|
41
|
+
if (detail && typeof detail === "object" && !Array.isArray(detail)) {
|
|
42
|
+
const stable = detail.code;
|
|
43
|
+
if (typeof stable === "string" && /^workspace_[a-z0-9_]+$/u.test(stable))
|
|
44
|
+
code = stable;
|
|
45
|
+
const retryable = detail.retryable;
|
|
46
|
+
if (typeof retryable === "boolean")
|
|
47
|
+
explicitRetryable = retryable;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
// Content-free HTTP fallback remains stable and retry classification stays status-based.
|
|
52
|
+
}
|
|
53
|
+
throw new WorkspaceSnapshotControlError(code, explicitRetryable ?? (response.status >= 500 || response.status === 408 || response.status === 429 ||
|
|
54
|
+
response.status === 409));
|
|
55
|
+
}
|
|
56
|
+
return await response.json();
|
|
57
|
+
}
|
|
58
|
+
function fence(scope) {
|
|
59
|
+
return {
|
|
60
|
+
sandbox_id: scope.sandboxId,
|
|
61
|
+
sandbox_generation: scope.sandboxGeneration,
|
|
62
|
+
connection_epoch: scope.connectionEpoch,
|
|
63
|
+
runtime_session_id: scope.runtimeSessionId,
|
|
64
|
+
...(scope.agentRunId ? { agent_run_id: scope.agentRunId } : {}),
|
|
65
|
+
...(scope.workerAttempt ? { worker_attempt: scope.workerAttempt } : {}),
|
|
66
|
+
...(scope.activationId ? { activation_id: scope.activationId } : {}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export async function checkpointWorkspace(options) {
|
|
70
|
+
const { scope } = options;
|
|
71
|
+
const signal = AbortSignal.timeout(WORKSPACE_SNAPSHOT_POLICY_V1.checkpointAttemptDeadlineMs);
|
|
72
|
+
const checkpointUrl = `${controlBase(options.wsUrl)}/${scope.checkpointId}`;
|
|
73
|
+
const controlDirectory = path.join(ensureDaemonHome(), "agent-service-sandboxes", scope.sandboxId, "workspace-control");
|
|
74
|
+
const staged = stageWorkspaceSnapshot({
|
|
75
|
+
workspaceDirectory: options.workspaceDirectory,
|
|
76
|
+
controlDirectory,
|
|
77
|
+
checkpointId: scope.checkpointId,
|
|
78
|
+
limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
|
|
79
|
+
maxStabilityRetries: WORKSPACE_SNAPSHOT_POLICY_V1.maxStabilityRetries,
|
|
80
|
+
freezeProof: options.freezeProof,
|
|
81
|
+
});
|
|
82
|
+
try {
|
|
83
|
+
const entrySet = {
|
|
84
|
+
schemaVersion: "agent-workspace-entry-set/1",
|
|
85
|
+
entries: staged.entrySet.entries,
|
|
86
|
+
};
|
|
87
|
+
const proposal = await requestJson(`${checkpointUrl}/proposal`, options.reconnectToken, {
|
|
88
|
+
fence: fence(scope),
|
|
89
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
90
|
+
base_revision: scope.baseRevision,
|
|
91
|
+
entry_count: staged.entrySet.entries.length,
|
|
92
|
+
file_count: staged.entrySet.fileCount,
|
|
93
|
+
total_bytes: staged.entrySet.totalBytes,
|
|
94
|
+
content_sha256: staged.entrySet.contentSha256,
|
|
95
|
+
entry_set: entrySet,
|
|
96
|
+
}, signal);
|
|
97
|
+
if (proposal.unchanged) {
|
|
98
|
+
if (proposal.revision === 0 && proposal.snapshot_id === null) {
|
|
99
|
+
writeWorkspaceMaterializationMarker({
|
|
100
|
+
schemaVersion: "agent-workspace-materialization/1",
|
|
101
|
+
sandboxId: scope.sandboxId,
|
|
102
|
+
sandboxGeneration: scope.sandboxGeneration,
|
|
103
|
+
runtimeSessionId: scope.runtimeSessionId,
|
|
104
|
+
workspaceContinuityState: "healthy",
|
|
105
|
+
snapshotId: null,
|
|
106
|
+
revision: 0,
|
|
107
|
+
contentSha256: null,
|
|
108
|
+
writtenAt: new Date().toISOString(),
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
revision: proposal.revision,
|
|
113
|
+
snapshotId: proposal.snapshot_id,
|
|
114
|
+
contentSha256: proposal.content_sha256,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
for (let offset = 0; offset < proposal.missing_blob_ids.length; offset += 25) {
|
|
118
|
+
const requestedBlobIds = proposal.missing_blob_ids.slice(offset, offset + 25);
|
|
119
|
+
const grants = await requestJson(`${checkpointUrl}/upload-grants`, options.reconnectToken, { fence: fence(scope), blob_ids: requestedBlobIds }, signal);
|
|
120
|
+
if (grants.length !== requestedBlobIds.length) {
|
|
121
|
+
throw new Error("workspace_snapshot_upload_grant_count_mismatch");
|
|
122
|
+
}
|
|
123
|
+
for (const item of grants) {
|
|
124
|
+
const source = staged.files.find((file) => file.sha256 === item.sha256 && file.sizeBytes === item.size_bytes);
|
|
125
|
+
if (!source)
|
|
126
|
+
throw new Error("workspace_snapshot_upload_source_missing");
|
|
127
|
+
const response = await fetch(item.grant.url, {
|
|
128
|
+
method: item.grant.method,
|
|
129
|
+
headers: item.grant.headers,
|
|
130
|
+
body: createReadStream(source.stagingPath),
|
|
131
|
+
duplex: "half",
|
|
132
|
+
signal,
|
|
133
|
+
});
|
|
134
|
+
if (!response.ok)
|
|
135
|
+
throw new Error(`workspace_snapshot_upload_${response.status}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
let committed = await requestJson(`${checkpointUrl}/complete`, options.reconnectToken, { fence: fence(scope), policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId }, signal);
|
|
139
|
+
while (committed.status === "pending") {
|
|
140
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
141
|
+
committed = await requestJson(`${checkpointUrl}/status`, options.reconnectToken, { fence: fence(scope) }, signal);
|
|
142
|
+
}
|
|
143
|
+
if (committed.status !== "committed" ||
|
|
144
|
+
committed.snapshot_id === null ||
|
|
145
|
+
committed.revision === null ||
|
|
146
|
+
committed.content_sha256 === null) {
|
|
147
|
+
throw new WorkspaceSnapshotControlError(committed.error_code || "workspace_snapshot_verification_failed", false);
|
|
148
|
+
}
|
|
149
|
+
writeWorkspaceMaterializationMarker({
|
|
150
|
+
schemaVersion: "agent-workspace-materialization/1",
|
|
151
|
+
sandboxId: scope.sandboxId,
|
|
152
|
+
sandboxGeneration: scope.sandboxGeneration,
|
|
153
|
+
runtimeSessionId: scope.runtimeSessionId,
|
|
154
|
+
workspaceContinuityState: "healthy",
|
|
155
|
+
snapshotId: committed.snapshot_id,
|
|
156
|
+
revision: committed.revision,
|
|
157
|
+
contentSha256: committed.content_sha256,
|
|
158
|
+
writtenAt: new Date().toISOString(),
|
|
159
|
+
});
|
|
160
|
+
return {
|
|
161
|
+
revision: committed.revision,
|
|
162
|
+
snapshotId: committed.snapshot_id,
|
|
163
|
+
contentSha256: committed.content_sha256,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
finally {
|
|
167
|
+
rmSync(staged.stagingDirectory, { recursive: true, force: true });
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Protocol limits frozen with Course Service's WorkspaceSnapshotPolicyV1.
|
|
3
|
+
* Deployment may enable the capability, but must never override these values.
|
|
4
|
+
*/
|
|
5
|
+
export declare const WORKSPACE_SNAPSHOT_POLICY_V1: {
|
|
6
|
+
readonly policyId: "WorkspaceSnapshotPolicyV1";
|
|
7
|
+
readonly limits: {
|
|
8
|
+
maxPathSegmentBytes: number;
|
|
9
|
+
maxPathBytes: number;
|
|
10
|
+
maxDepth: number;
|
|
11
|
+
maxFileBytes: number;
|
|
12
|
+
maxFileCount: number;
|
|
13
|
+
maxTotalBytes: number;
|
|
14
|
+
maxEntrySetBytes: number;
|
|
15
|
+
};
|
|
16
|
+
readonly maxStabilityRetries: 2;
|
|
17
|
+
readonly checkpointAttemptDeadlineMs: 30000;
|
|
18
|
+
readonly requiredRestoreFreeBytes: number;
|
|
19
|
+
readonly sessionRetainedHardLimitBytes: number;
|
|
20
|
+
readonly checkpointReservationBytes: number;
|
|
21
|
+
readonly restorePageEntries: 25;
|
|
22
|
+
};
|
|
23
|
+
/** Wire representation validated exactly by Agent Service during the v0.3 handshake. */
|
|
24
|
+
export declare function workspaceSnapshotPolicyCapability(): Record<string, unknown>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { RUNTIME_WORKSPACE_INODE_QUOTA, RUNTIME_WORKSPACE_QUOTA_BYTES, } from "./workspace-quota.js";
|
|
2
|
+
/**
|
|
3
|
+
* Protocol limits frozen with Course Service's WorkspaceSnapshotPolicyV1.
|
|
4
|
+
* Deployment may enable the capability, but must never override these values.
|
|
5
|
+
*/
|
|
6
|
+
export const WORKSPACE_SNAPSHOT_POLICY_V1 = {
|
|
7
|
+
policyId: "WorkspaceSnapshotPolicyV1",
|
|
8
|
+
limits: {
|
|
9
|
+
maxPathSegmentBytes: 255,
|
|
10
|
+
maxPathBytes: 1024,
|
|
11
|
+
maxDepth: 32,
|
|
12
|
+
maxFileBytes: 50 * 1024 * 1024,
|
|
13
|
+
maxFileCount: 2_000,
|
|
14
|
+
maxTotalBytes: 512 * 1024 * 1024,
|
|
15
|
+
maxEntrySetBytes: 1024 * 1024,
|
|
16
|
+
},
|
|
17
|
+
maxStabilityRetries: 2,
|
|
18
|
+
checkpointAttemptDeadlineMs: 30_000,
|
|
19
|
+
requiredRestoreFreeBytes: 1024 * 1024,
|
|
20
|
+
sessionRetainedHardLimitBytes: 2 * 1024 * 1024 * 1024,
|
|
21
|
+
checkpointReservationBytes: 512 * 1024 * 1024,
|
|
22
|
+
restorePageEntries: 25,
|
|
23
|
+
};
|
|
24
|
+
/** Wire representation validated exactly by Agent Service during the v0.3 handshake. */
|
|
25
|
+
export function workspaceSnapshotPolicyCapability() {
|
|
26
|
+
return {
|
|
27
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
28
|
+
max_path_segment_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxPathSegmentBytes,
|
|
29
|
+
max_path_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxPathBytes,
|
|
30
|
+
max_depth: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxDepth,
|
|
31
|
+
max_file_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxFileBytes,
|
|
32
|
+
max_file_count: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxFileCount,
|
|
33
|
+
max_total_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxTotalBytes,
|
|
34
|
+
max_entry_set_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxEntrySetBytes,
|
|
35
|
+
required_restore_free_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes,
|
|
36
|
+
session_retained_hard_limit_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.sessionRetainedHardLimitBytes,
|
|
37
|
+
checkpoint_reservation_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.checkpointReservationBytes,
|
|
38
|
+
restore_page_entries: WORKSPACE_SNAPSHOT_POLICY_V1.restorePageEntries,
|
|
39
|
+
max_stability_retries: WORKSPACE_SNAPSHOT_POLICY_V1.maxStabilityRetries,
|
|
40
|
+
checkpoint_attempt_deadline_ms: WORKSPACE_SNAPSHOT_POLICY_V1.checkpointAttemptDeadlineMs,
|
|
41
|
+
prepare_deadline_seconds: 15 * 60,
|
|
42
|
+
runtime_filesystem_quota_bytes: RUNTIME_WORKSPACE_QUOTA_BYTES,
|
|
43
|
+
runtime_inode_quota: RUNTIME_WORKSPACE_INODE_QUOTA,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type WorkspaceWriterFreezeProof } from "./runtime-quiescence.js";
|
|
1
2
|
import { type CanonicalWorkspaceEntrySet, type WorkspaceEntrySetLimits } from "./workspace-entry-set.js";
|
|
2
3
|
export declare class WorkspaceSnapshotStagingError extends Error {
|
|
3
4
|
readonly code: string;
|
|
@@ -21,14 +22,6 @@ export interface WorkspaceSnapshotStagingOptions {
|
|
|
21
22
|
checkpointId: string;
|
|
22
23
|
limits: WorkspaceEntrySetLimits;
|
|
23
24
|
maxStabilityRetries: number;
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
export interface WorkspaceQuiescenceProof {
|
|
27
|
-
proofVersion: "workspace-quiescence-proof/1";
|
|
28
|
-
sandboxId: string;
|
|
29
|
-
sandboxGeneration: number;
|
|
30
|
-
runtimeSessionId: string;
|
|
31
|
-
checkpointId: string;
|
|
32
|
-
allWritersStopped: true;
|
|
25
|
+
freezeProof: WorkspaceWriterFreezeProof;
|
|
33
26
|
}
|
|
34
27
|
export declare function stageWorkspaceSnapshot(options: WorkspaceSnapshotStagingOptions): StagedWorkspaceSnapshot;
|