@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-service-sandbox.d.ts +7 -1
- package/dist/agent-service-sandbox.js +384 -16
- package/dist/agent-service-ws-protocol.d.ts +3 -3
- package/dist/agent-service-ws-protocol.js +4 -2
- package/dist/cli.js +19 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/run-dispatcher.d.ts +1 -1
- package/dist/run-dispatcher.js +1 -1
- package/dist/runtime-env.js +4 -4
- package/dist/runtime-quiescence.d.ts +16 -0
- package/dist/runtime-quiescence.js +42 -0
- package/dist/workspace-quota.d.ts +4 -0
- package/dist/workspace-quota.js +42 -0
- package/dist/workspace-restore.d.ts +9 -1
- package/dist/workspace-restore.js +30 -16
- package/dist/workspace-snapshot-control.d.ts +29 -0
- package/dist/workspace-snapshot-control.js +169 -0
- package/dist/workspace-snapshot-policy.d.ts +24 -0
- package/dist/workspace-snapshot-policy.js +45 -0
- package/dist/workspace-snapshot-staging.d.ts +1 -8
- package/dist/workspace-snapshot-staging.js +53 -19
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
export const AGENT_SERVICE_WS_SCHEMA = "botlearn-agent-sandbox-ws/0.
|
|
3
|
-
export const AGENT_SERVICE_WS_SUBPROTOCOL = "botlearn-agent-sandbox.
|
|
2
|
+
export const AGENT_SERVICE_WS_SCHEMA = "botlearn-agent-sandbox-ws/0.3";
|
|
3
|
+
export const AGENT_SERVICE_WS_SUBPROTOCOL = "botlearn-agent-sandbox.v3";
|
|
4
4
|
export const WORKSPACE_FILE_READ_BINARY_CAPABILITY = "workspace_file_read_binary_v1";
|
|
5
5
|
export const WORKSPACE_FILE_CHUNK_BYTES = 64 * 1024;
|
|
6
6
|
export const WORKSPACE_FILE_CHUNK_HEADER_BYTES = 29;
|
|
@@ -18,6 +18,7 @@ const FRAME_TYPES = new Set([
|
|
|
18
18
|
"event.ack",
|
|
19
19
|
"auth.rotate",
|
|
20
20
|
"workspace.file.read",
|
|
21
|
+
"workspace.checkpoint",
|
|
21
22
|
"ping",
|
|
22
23
|
"sandbox.ready",
|
|
23
24
|
"sandbox.heartbeat",
|
|
@@ -42,6 +43,7 @@ const SESSION_TYPES = new Set([
|
|
|
42
43
|
"session.open_failed",
|
|
43
44
|
"session.closed",
|
|
44
45
|
"workspace.file.read",
|
|
46
|
+
"workspace.checkpoint",
|
|
45
47
|
"workspace.file.result",
|
|
46
48
|
]);
|
|
47
49
|
/** TURN 帧:必须带 runtime_session_id + agent_run_id + worker_attempt + activation_id 四元组。 */
|
package/dist/cli.js
CHANGED
|
@@ -372,7 +372,25 @@ async function cmdAgentServiceSession(args) {
|
|
|
372
372
|
deepseekTuiVersion: process.env.BOTLEARN_MANAGED_DEEPSEEK_TUI_VERSION?.trim() || undefined,
|
|
373
373
|
});
|
|
374
374
|
clearAgentServiceControlEnv();
|
|
375
|
-
|
|
375
|
+
let stopping = false;
|
|
376
|
+
const stop = () => {
|
|
377
|
+
if (stopping)
|
|
378
|
+
return;
|
|
379
|
+
stopping = true;
|
|
380
|
+
void client.stopGracefully().catch((error) => {
|
|
381
|
+
console.error(error instanceof Error ? error.message : "sandbox shutdown failed");
|
|
382
|
+
client.stop();
|
|
383
|
+
});
|
|
384
|
+
};
|
|
385
|
+
process.once("SIGTERM", stop);
|
|
386
|
+
process.once("SIGINT", stop);
|
|
387
|
+
try {
|
|
388
|
+
await client.run();
|
|
389
|
+
}
|
|
390
|
+
finally {
|
|
391
|
+
process.removeListener("SIGTERM", stop);
|
|
392
|
+
process.removeListener("SIGINT", stop);
|
|
393
|
+
}
|
|
376
394
|
return 0;
|
|
377
395
|
}
|
|
378
396
|
// ---------------------------------------------------------------
|
package/dist/index.d.ts
CHANGED
|
@@ -14,6 +14,8 @@ export * from "./workspace.js";
|
|
|
14
14
|
export * from "./workspace-entry-set.js";
|
|
15
15
|
export * from "./workspace-materialization.js";
|
|
16
16
|
export * from "./workspace-snapshot-staging.js";
|
|
17
|
+
export * from "./workspace-snapshot-policy.js";
|
|
18
|
+
export * from "./workspace-snapshot-control.js";
|
|
17
19
|
export * from "./workspace-restore.js";
|
|
18
20
|
export * from "./transcript.js";
|
|
19
21
|
export * from "./file-candidates.js";
|
package/dist/index.js
CHANGED
|
@@ -14,6 +14,8 @@ export * from "./workspace.js";
|
|
|
14
14
|
export * from "./workspace-entry-set.js";
|
|
15
15
|
export * from "./workspace-materialization.js";
|
|
16
16
|
export * from "./workspace-snapshot-staging.js";
|
|
17
|
+
export * from "./workspace-snapshot-policy.js";
|
|
18
|
+
export * from "./workspace-snapshot-control.js";
|
|
17
19
|
export * from "./workspace-restore.js";
|
|
18
20
|
export * from "./transcript.js";
|
|
19
21
|
export * from "./file-candidates.js";
|
package/dist/run-dispatcher.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export interface PreparedPersistentTurn {
|
|
|
23
23
|
export interface PersistentSessionExecution {
|
|
24
24
|
prepareTurn(payload: RunStartPayload): PreparedPersistentTurn;
|
|
25
25
|
persistNativeSession(sessionId: string): void;
|
|
26
|
-
finishTurn(payload: RunStartPayload): void
|
|
26
|
+
finishTurn(payload: RunStartPayload): Promise<void>;
|
|
27
27
|
}
|
|
28
28
|
export interface RunReportingClient {
|
|
29
29
|
postEvent(agentRunId: string, event: RunEvent): Promise<void | RunEventReceipt>;
|
package/dist/run-dispatcher.js
CHANGED
package/dist/runtime-env.js
CHANGED
|
@@ -42,11 +42,15 @@ export function clearAgentServiceControlEnv(env = process.env) {
|
|
|
42
42
|
export function runtimeChildEnv(env = process.env) {
|
|
43
43
|
const childEnv = { ...env };
|
|
44
44
|
const runtimeHome = childEnv.BOTLEARN_RUNTIME_HOME;
|
|
45
|
+
const activationId = childEnv.BOTLEARN_AGENT_SERVICE_ACTIVATION_ID;
|
|
45
46
|
clearAgentServiceControlEnv(childEnv);
|
|
46
47
|
for (const key of AGENT_SERVICE_SUPERVISOR_ENV_KEYS)
|
|
47
48
|
delete childEnv[key];
|
|
48
49
|
if (runtimeHome)
|
|
49
50
|
childEnv.HOME = runtimeHome;
|
|
51
|
+
if (activationId && /^[A-Za-z0-9_-]{1,120}$/u.test(activationId)) {
|
|
52
|
+
childEnv.BOTLEARN_RUNTIME_ACTIVATION_SCOPE = activationId;
|
|
53
|
+
}
|
|
50
54
|
return childEnv;
|
|
51
55
|
}
|
|
52
56
|
/**
|
|
@@ -146,10 +150,6 @@ export function runtimeChildLaunch(binary, args, env = process.env, options = {}
|
|
|
146
150
|
"-n",
|
|
147
151
|
"-H",
|
|
148
152
|
"-E",
|
|
149
|
-
"-u",
|
|
150
|
-
runtimeUser,
|
|
151
|
-
"-g",
|
|
152
|
-
runtimeGroup,
|
|
153
153
|
"--",
|
|
154
154
|
MANAGED_RUNTIME_LAUNCHER,
|
|
155
155
|
binary,
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface WorkspaceQuiescenceProof {
|
|
2
|
+
readonly proofVersion: "workspace-quiescence-proof/1";
|
|
3
|
+
readonly sandboxId: string;
|
|
4
|
+
readonly sandboxGeneration: number;
|
|
5
|
+
readonly runtimeSessionId: string;
|
|
6
|
+
readonly checkpointId: string;
|
|
7
|
+
readonly allWritersStopped: true;
|
|
8
|
+
}
|
|
9
|
+
export declare function quiesceRuntimeWriters(activationId: string | null): Promise<void>;
|
|
10
|
+
export declare function issueWorkspaceQuiescenceProof(scope: {
|
|
11
|
+
sandboxId: string;
|
|
12
|
+
sandboxGeneration: number;
|
|
13
|
+
runtimeSessionId: string;
|
|
14
|
+
checkpointId: string;
|
|
15
|
+
}): WorkspaceQuiescenceProof;
|
|
16
|
+
export declare function isWorkspaceQuiescenceProof(value: object): value is WorkspaceQuiescenceProof;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { promisify } from "node:util";
|
|
3
|
+
const execFileAsync = promisify(execFile);
|
|
4
|
+
const RUNTIME_LAUNCHER = "/opt/botlearn/bin/botlearn-runtime-launcher";
|
|
5
|
+
const proofs = new WeakSet();
|
|
6
|
+
function managedQuiescenceCommand(activationId) {
|
|
7
|
+
const mode = process.env.BOTLEARN_RUNTIME_LAUNCH_MODE;
|
|
8
|
+
if (mode === "direct-uid") {
|
|
9
|
+
return [RUNTIME_LAUNCHER, [activationId ? "--quiesce" : "--quiesce-all", ...(activationId ? [activationId] : [])]];
|
|
10
|
+
}
|
|
11
|
+
if (mode === "sudo") {
|
|
12
|
+
return [
|
|
13
|
+
"/usr/bin/sudo",
|
|
14
|
+
["-n", "--", RUNTIME_LAUNCHER, activationId ? "--quiesce" : "--quiesce-all", ...(activationId ? [activationId] : [])],
|
|
15
|
+
];
|
|
16
|
+
}
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
export async function quiesceRuntimeWriters(activationId) {
|
|
20
|
+
const command = managedQuiescenceCommand(activationId);
|
|
21
|
+
if (command === null) {
|
|
22
|
+
// Local/BYOA runtimes execute as the daemon user and are not eligible for the managed
|
|
23
|
+
// durable capability. Unit/e2e fake runtimes have no descendant writer to reap.
|
|
24
|
+
if (process.env.BOTLEARN_DAEMON_ENABLE_FAKE_RUNTIME === "1" || process.env.NODE_ENV === "test") {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
throw new Error("workspace_runtime_cgroup_unavailable");
|
|
28
|
+
}
|
|
29
|
+
await execFileAsync(command[0], command[1], { timeout: 10_000 });
|
|
30
|
+
}
|
|
31
|
+
export function issueWorkspaceQuiescenceProof(scope) {
|
|
32
|
+
const proof = Object.freeze({
|
|
33
|
+
proofVersion: "workspace-quiescence-proof/1",
|
|
34
|
+
...scope,
|
|
35
|
+
allWritersStopped: true,
|
|
36
|
+
});
|
|
37
|
+
proofs.add(proof);
|
|
38
|
+
return proof;
|
|
39
|
+
}
|
|
40
|
+
export function isWorkspaceQuiescenceProof(value) {
|
|
41
|
+
return proofs.has(value);
|
|
42
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const RUNTIME_WORKSPACE_QUOTA_BYTES: number;
|
|
2
|
+
export declare const RUNTIME_WORKSPACE_INODE_QUOTA = 2000;
|
|
3
|
+
export declare function assertRuntimeWorkspaceQuota(workspaceRoot: string): void;
|
|
4
|
+
export declare function applyRuntimeWorkspaceQuota(runtimeSessionId: string, sandboxGeneration: number): Promise<void>;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { readFileSync, statfsSync, statSync } from "node:fs";
|
|
3
|
+
import { promisify } from "node:util";
|
|
4
|
+
export const RUNTIME_WORKSPACE_QUOTA_BYTES = 512 * 1024 * 1024;
|
|
5
|
+
export const RUNTIME_WORKSPACE_INODE_QUOTA = 2_000;
|
|
6
|
+
const WORKSPACE_BACKING_BYTES = 3 * 1024 * 1024 * 1024;
|
|
7
|
+
const QUOTA_MARKER = "/var/lib/botlearn-runtime/workspace-quota.ready";
|
|
8
|
+
const RUNTIME_LAUNCHER = "/opt/botlearn/bin/botlearn-runtime-launcher";
|
|
9
|
+
const execFileAsync = promisify(execFile);
|
|
10
|
+
export function assertRuntimeWorkspaceQuota(workspaceRoot) {
|
|
11
|
+
if (process.env.BOTLEARN_DAEMON_ENABLE_FAKE_RUNTIME === "1" || process.env.NODE_ENV === "test") {
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
if (process.env.BOTLEARN_RUNTIME_WORKSPACE_QUOTA_BYTES !== String(RUNTIME_WORKSPACE_QUOTA_BYTES) ||
|
|
15
|
+
process.env.BOTLEARN_RUNTIME_WORKSPACE_INODE_QUOTA !== String(RUNTIME_WORKSPACE_INODE_QUOTA)) {
|
|
16
|
+
throw new Error("workspace_runtime_quota_unverified");
|
|
17
|
+
}
|
|
18
|
+
const filesystem = statfsSync(workspaceRoot, { bigint: true });
|
|
19
|
+
const marker = statSync(QUOTA_MARKER);
|
|
20
|
+
if (filesystem.blocks * filesystem.bsize < BigInt(WORKSPACE_BACKING_BYTES) ||
|
|
21
|
+
marker.uid !== 0 || (marker.mode & 0o777) !== 0o600 ||
|
|
22
|
+
readFileSync(QUOTA_MARKER, "utf8") !== "workspace-filesystem-quota/1\n") {
|
|
23
|
+
throw new Error("workspace_runtime_quota_probe_failed");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export async function applyRuntimeWorkspaceQuota(runtimeSessionId, sandboxGeneration) {
|
|
27
|
+
if (process.env.BOTLEARN_DAEMON_ENABLE_FAKE_RUNTIME === "1" || process.env.NODE_ENV === "test") {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const args = ["--workspace-quota-apply", runtimeSessionId, String(sandboxGeneration)];
|
|
31
|
+
if (process.env.BOTLEARN_RUNTIME_LAUNCH_MODE === "sudo") {
|
|
32
|
+
await execFileAsync("/usr/bin/sudo", ["-n", "--", RUNTIME_LAUNCHER, ...args], {
|
|
33
|
+
timeout: 10_000,
|
|
34
|
+
});
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
if (process.env.BOTLEARN_RUNTIME_LAUNCH_MODE === "direct-uid") {
|
|
38
|
+
await execFileAsync(RUNTIME_LAUNCHER, args, { timeout: 10_000 });
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
throw new Error("workspace_runtime_quota_helper_unavailable");
|
|
42
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { type WorkspaceEntry, type WorkspaceEntrySetLimits } from "./workspace-entry-set.js";
|
|
2
2
|
export declare class WorkspaceRestoreError extends Error {
|
|
3
3
|
readonly code: string;
|
|
4
|
-
|
|
4
|
+
readonly retryable: boolean;
|
|
5
|
+
constructor(code: string, retryable?: boolean);
|
|
5
6
|
}
|
|
6
7
|
export interface WorkspaceRestoreDescriptor {
|
|
7
8
|
sandboxId: string;
|
|
@@ -25,6 +26,12 @@ export interface WorkspaceRestoreOptions {
|
|
|
25
26
|
requiredFreeBytes: number;
|
|
26
27
|
downloadFile(entry: WorkspaceEntry, partPath: string): Promise<void>;
|
|
27
28
|
}
|
|
29
|
+
export interface WorkspaceRestoreCapacityOptions {
|
|
30
|
+
workspaceDirectory: string;
|
|
31
|
+
descriptor: WorkspaceRestoreDescriptor;
|
|
32
|
+
requiredFreeBytes: number;
|
|
33
|
+
entryCopies?: 1 | 2;
|
|
34
|
+
}
|
|
28
35
|
export declare function workspaceRestoreJournalPath(controlDirectory: string, runtimeSessionId: string): string;
|
|
29
36
|
export declare function recoverWorkspaceRestoreSwap(options: {
|
|
30
37
|
workspaceDirectory: string;
|
|
@@ -32,3 +39,4 @@ export declare function recoverWorkspaceRestoreSwap(options: {
|
|
|
32
39
|
descriptor: WorkspaceRestoreDescriptor;
|
|
33
40
|
}): boolean;
|
|
34
41
|
export declare function restoreWorkspaceSnapshot(options: WorkspaceRestoreOptions): Promise<void>;
|
|
42
|
+
export declare function assertWorkspaceRestoreCapacity(options: WorkspaceRestoreCapacityOptions): void;
|
|
@@ -9,14 +9,18 @@ const RESTORE_JOURNAL_SCHEMA = "agent-workspace-restore-journal/1";
|
|
|
9
9
|
const COPY_CHUNK_BYTES = 64 * 1024;
|
|
10
10
|
export class WorkspaceRestoreError extends Error {
|
|
11
11
|
code;
|
|
12
|
-
|
|
12
|
+
retryable;
|
|
13
|
+
constructor(code, retryable = false) {
|
|
13
14
|
super(code);
|
|
14
15
|
this.code = code;
|
|
16
|
+
this.retryable = retryable;
|
|
15
17
|
this.name = "WorkspaceRestoreError";
|
|
16
18
|
}
|
|
17
19
|
}
|
|
18
20
|
function fail(code) {
|
|
19
|
-
throw new WorkspaceRestoreError(code
|
|
21
|
+
throw new WorkspaceRestoreError(code, code === "workspace_restore_unavailable" ||
|
|
22
|
+
code === "workspace_restore_disk_pressure" ||
|
|
23
|
+
code === "workspace_restore_inode_pressure");
|
|
20
24
|
}
|
|
21
25
|
function assertUuid(value, label) {
|
|
22
26
|
if (!UUID.test(value))
|
|
@@ -229,9 +233,7 @@ export async function restoreWorkspaceSnapshot(options) {
|
|
|
229
233
|
const descriptor = options.descriptor;
|
|
230
234
|
assertDescriptor(descriptor);
|
|
231
235
|
assertControlPath(options.workspaceDirectory, options.controlDirectory);
|
|
232
|
-
|
|
233
|
-
options.requiredFreeBytes < descriptor.expectedTotalBytes)
|
|
234
|
-
fail("workspace_restore_reservation_invalid");
|
|
236
|
+
assertWorkspaceRestoreCapacity(options);
|
|
235
237
|
recoverWorkspaceRestoreSwap({
|
|
236
238
|
workspaceDirectory: options.workspaceDirectory,
|
|
237
239
|
controlDirectory: options.controlDirectory,
|
|
@@ -244,17 +246,6 @@ export async function restoreWorkspaceSnapshot(options) {
|
|
|
244
246
|
(descriptor.revision > 0 && canonical.contentSha256 !== descriptor.contentSha256))
|
|
245
247
|
fail("workspace_restore_integrity_failed");
|
|
246
248
|
const parent = path.dirname(options.workspaceDirectory);
|
|
247
|
-
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
248
|
-
if (existsSync(options.workspaceDirectory)) {
|
|
249
|
-
const current = lstatSync(options.workspaceDirectory, { bigint: true });
|
|
250
|
-
if (!current.isDirectory() || current.isSymbolicLink()) {
|
|
251
|
-
fail("workspace_restore_existing_workspace_invalid");
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
const filesystem = statfsSync(parent, { bigint: true });
|
|
255
|
-
if (filesystem.bavail * filesystem.bsize < BigInt(options.requiredFreeBytes)) {
|
|
256
|
-
fail("workspace_restore_disk_pressure");
|
|
257
|
-
}
|
|
258
249
|
const nonce = randomUUID();
|
|
259
250
|
const stagingName = `.botlearn-restore-${descriptor.runtimeSessionId}-${nonce}`;
|
|
260
251
|
const backupName = `.botlearn-backup-${descriptor.runtimeSessionId}-${nonce}`;
|
|
@@ -331,3 +322,26 @@ export async function restoreWorkspaceSnapshot(options) {
|
|
|
331
322
|
throw error;
|
|
332
323
|
}
|
|
333
324
|
}
|
|
325
|
+
export function assertWorkspaceRestoreCapacity(options) {
|
|
326
|
+
const { descriptor } = options;
|
|
327
|
+
assertDescriptor(descriptor);
|
|
328
|
+
if (!Number.isSafeInteger(options.requiredFreeBytes) ||
|
|
329
|
+
options.requiredFreeBytes < descriptor.expectedTotalBytes)
|
|
330
|
+
fail("workspace_restore_reservation_invalid");
|
|
331
|
+
const parent = path.dirname(options.workspaceDirectory);
|
|
332
|
+
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
333
|
+
if (existsSync(options.workspaceDirectory)) {
|
|
334
|
+
const current = lstatSync(options.workspaceDirectory, { bigint: true });
|
|
335
|
+
if (!current.isDirectory() || current.isSymbolicLink()) {
|
|
336
|
+
fail("workspace_restore_existing_workspace_invalid");
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
const filesystem = statfsSync(parent, { bigint: true });
|
|
340
|
+
if (filesystem.bavail * filesystem.bsize < BigInt(options.requiredFreeBytes)) {
|
|
341
|
+
fail("workspace_restore_disk_pressure");
|
|
342
|
+
}
|
|
343
|
+
const entryCopies = options.entryCopies ?? 1;
|
|
344
|
+
if (filesystem.ffree < BigInt(descriptor.expectedEntryCount * entryCopies + 16)) {
|
|
345
|
+
fail("workspace_restore_inode_pressure");
|
|
346
|
+
}
|
|
347
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { WorkspaceQuiescenceProof } from "./runtime-quiescence.js";
|
|
2
|
+
interface WorkspaceCheckpointScope {
|
|
3
|
+
checkpointId: string;
|
|
4
|
+
baseRevision: number;
|
|
5
|
+
sandboxId: string;
|
|
6
|
+
sandboxGeneration: number;
|
|
7
|
+
connectionEpoch: number;
|
|
8
|
+
runtimeSessionId: string;
|
|
9
|
+
agentRunId?: string;
|
|
10
|
+
workerAttempt?: number;
|
|
11
|
+
activationId?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare class WorkspaceSnapshotControlError extends Error {
|
|
14
|
+
readonly code: string;
|
|
15
|
+
readonly retryable: boolean;
|
|
16
|
+
constructor(code: string, retryable: boolean);
|
|
17
|
+
}
|
|
18
|
+
export declare function checkpointWorkspace(options: {
|
|
19
|
+
wsUrl: string;
|
|
20
|
+
reconnectToken: string;
|
|
21
|
+
workspaceDirectory: string;
|
|
22
|
+
scope: WorkspaceCheckpointScope;
|
|
23
|
+
quiescenceProof: WorkspaceQuiescenceProof;
|
|
24
|
+
}): Promise<{
|
|
25
|
+
revision: number;
|
|
26
|
+
snapshotId: string | null;
|
|
27
|
+
contentSha256: string;
|
|
28
|
+
}>;
|
|
29
|
+
export {};
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { createReadStream, rmSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { ensureDaemonHome } from "./auth-store.js";
|
|
4
|
+
import { writeWorkspaceMaterializationMarker } from "./workspace-materialization.js";
|
|
5
|
+
import { WORKSPACE_SNAPSHOT_POLICY_V1 } from "./workspace-snapshot-policy.js";
|
|
6
|
+
import { stageWorkspaceSnapshot } from "./workspace-snapshot-staging.js";
|
|
7
|
+
export class WorkspaceSnapshotControlError extends Error {
|
|
8
|
+
code;
|
|
9
|
+
retryable;
|
|
10
|
+
constructor(code, retryable) {
|
|
11
|
+
super(code);
|
|
12
|
+
this.code = code;
|
|
13
|
+
this.retryable = retryable;
|
|
14
|
+
this.name = "WorkspaceSnapshotControlError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function controlBase(wsUrl) {
|
|
18
|
+
const url = new URL(wsUrl);
|
|
19
|
+
url.protocol = url.protocol === "wss:" ? "https:" : "http:";
|
|
20
|
+
url.pathname = "/internal/v1/workspace-checkpoints";
|
|
21
|
+
url.search = "";
|
|
22
|
+
url.hash = "";
|
|
23
|
+
return url.toString().replace(/\/$/, "");
|
|
24
|
+
}
|
|
25
|
+
async function requestJson(url, token, body, signal) {
|
|
26
|
+
const response = await fetch(url, {
|
|
27
|
+
method: "POST",
|
|
28
|
+
headers: {
|
|
29
|
+
Authorization: `Bearer ${token}`,
|
|
30
|
+
"Content-Type": "application/json",
|
|
31
|
+
},
|
|
32
|
+
body: JSON.stringify(body),
|
|
33
|
+
signal,
|
|
34
|
+
});
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
let code = `workspace_snapshot_control_${response.status}`;
|
|
37
|
+
let explicitRetryable;
|
|
38
|
+
try {
|
|
39
|
+
const payload = await response.json();
|
|
40
|
+
const detail = payload.detail;
|
|
41
|
+
if (detail && typeof detail === "object" && !Array.isArray(detail)) {
|
|
42
|
+
const stable = detail.code;
|
|
43
|
+
if (typeof stable === "string" && /^workspace_[a-z0-9_]+$/u.test(stable))
|
|
44
|
+
code = stable;
|
|
45
|
+
const retryable = detail.retryable;
|
|
46
|
+
if (typeof retryable === "boolean")
|
|
47
|
+
explicitRetryable = retryable;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
// Content-free HTTP fallback remains stable and retry classification stays status-based.
|
|
52
|
+
}
|
|
53
|
+
throw new WorkspaceSnapshotControlError(code, explicitRetryable ?? (response.status >= 500 || response.status === 408 || response.status === 429 ||
|
|
54
|
+
response.status === 409));
|
|
55
|
+
}
|
|
56
|
+
return await response.json();
|
|
57
|
+
}
|
|
58
|
+
function fence(scope) {
|
|
59
|
+
return {
|
|
60
|
+
sandbox_id: scope.sandboxId,
|
|
61
|
+
sandbox_generation: scope.sandboxGeneration,
|
|
62
|
+
connection_epoch: scope.connectionEpoch,
|
|
63
|
+
runtime_session_id: scope.runtimeSessionId,
|
|
64
|
+
...(scope.agentRunId ? { agent_run_id: scope.agentRunId } : {}),
|
|
65
|
+
...(scope.workerAttempt ? { worker_attempt: scope.workerAttempt } : {}),
|
|
66
|
+
...(scope.activationId ? { activation_id: scope.activationId } : {}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export async function checkpointWorkspace(options) {
|
|
70
|
+
const { scope } = options;
|
|
71
|
+
const signal = AbortSignal.timeout(WORKSPACE_SNAPSHOT_POLICY_V1.checkpointAttemptDeadlineMs);
|
|
72
|
+
const checkpointUrl = `${controlBase(options.wsUrl)}/${scope.checkpointId}`;
|
|
73
|
+
const controlDirectory = path.join(ensureDaemonHome(), "agent-service-sandboxes", scope.sandboxId, "workspace-control");
|
|
74
|
+
const staged = stageWorkspaceSnapshot({
|
|
75
|
+
workspaceDirectory: options.workspaceDirectory,
|
|
76
|
+
controlDirectory,
|
|
77
|
+
checkpointId: scope.checkpointId,
|
|
78
|
+
limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
|
|
79
|
+
maxStabilityRetries: WORKSPACE_SNAPSHOT_POLICY_V1.maxStabilityRetries,
|
|
80
|
+
quiescenceProof: options.quiescenceProof,
|
|
81
|
+
});
|
|
82
|
+
try {
|
|
83
|
+
const entrySet = {
|
|
84
|
+
schemaVersion: "agent-workspace-entry-set/1",
|
|
85
|
+
entries: staged.entrySet.entries,
|
|
86
|
+
};
|
|
87
|
+
const proposal = await requestJson(`${checkpointUrl}/proposal`, options.reconnectToken, {
|
|
88
|
+
fence: fence(scope),
|
|
89
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
90
|
+
base_revision: scope.baseRevision,
|
|
91
|
+
entry_count: staged.entrySet.entries.length,
|
|
92
|
+
file_count: staged.entrySet.fileCount,
|
|
93
|
+
total_bytes: staged.entrySet.totalBytes,
|
|
94
|
+
content_sha256: staged.entrySet.contentSha256,
|
|
95
|
+
entry_set: entrySet,
|
|
96
|
+
}, signal);
|
|
97
|
+
if (proposal.unchanged) {
|
|
98
|
+
if (proposal.revision === 0 && proposal.snapshot_id === null) {
|
|
99
|
+
writeWorkspaceMaterializationMarker({
|
|
100
|
+
schemaVersion: "agent-workspace-materialization/1",
|
|
101
|
+
sandboxId: scope.sandboxId,
|
|
102
|
+
sandboxGeneration: scope.sandboxGeneration,
|
|
103
|
+
runtimeSessionId: scope.runtimeSessionId,
|
|
104
|
+
workspaceContinuityState: "healthy",
|
|
105
|
+
snapshotId: null,
|
|
106
|
+
revision: 0,
|
|
107
|
+
contentSha256: null,
|
|
108
|
+
writtenAt: new Date().toISOString(),
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
revision: proposal.revision,
|
|
113
|
+
snapshotId: proposal.snapshot_id,
|
|
114
|
+
contentSha256: proposal.content_sha256,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
for (let offset = 0; offset < proposal.missing_blob_ids.length; offset += 25) {
|
|
118
|
+
const requestedBlobIds = proposal.missing_blob_ids.slice(offset, offset + 25);
|
|
119
|
+
const grants = await requestJson(`${checkpointUrl}/upload-grants`, options.reconnectToken, { fence: fence(scope), blob_ids: requestedBlobIds }, signal);
|
|
120
|
+
if (grants.length !== requestedBlobIds.length) {
|
|
121
|
+
throw new Error("workspace_snapshot_upload_grant_count_mismatch");
|
|
122
|
+
}
|
|
123
|
+
for (const item of grants) {
|
|
124
|
+
const source = staged.files.find((file) => file.sha256 === item.sha256 && file.sizeBytes === item.size_bytes);
|
|
125
|
+
if (!source)
|
|
126
|
+
throw new Error("workspace_snapshot_upload_source_missing");
|
|
127
|
+
const response = await fetch(item.grant.url, {
|
|
128
|
+
method: item.grant.method,
|
|
129
|
+
headers: item.grant.headers,
|
|
130
|
+
body: createReadStream(source.stagingPath),
|
|
131
|
+
duplex: "half",
|
|
132
|
+
signal,
|
|
133
|
+
});
|
|
134
|
+
if (!response.ok)
|
|
135
|
+
throw new Error(`workspace_snapshot_upload_${response.status}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
let committed = await requestJson(`${checkpointUrl}/complete`, options.reconnectToken, { fence: fence(scope), policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId }, signal);
|
|
139
|
+
while (committed.status === "pending") {
|
|
140
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
141
|
+
committed = await requestJson(`${checkpointUrl}/status`, options.reconnectToken, { fence: fence(scope) }, signal);
|
|
142
|
+
}
|
|
143
|
+
if (committed.status !== "committed" ||
|
|
144
|
+
committed.snapshot_id === null ||
|
|
145
|
+
committed.revision === null ||
|
|
146
|
+
committed.content_sha256 === null) {
|
|
147
|
+
throw new WorkspaceSnapshotControlError(committed.error_code || "workspace_snapshot_verification_failed", false);
|
|
148
|
+
}
|
|
149
|
+
writeWorkspaceMaterializationMarker({
|
|
150
|
+
schemaVersion: "agent-workspace-materialization/1",
|
|
151
|
+
sandboxId: scope.sandboxId,
|
|
152
|
+
sandboxGeneration: scope.sandboxGeneration,
|
|
153
|
+
runtimeSessionId: scope.runtimeSessionId,
|
|
154
|
+
workspaceContinuityState: "healthy",
|
|
155
|
+
snapshotId: committed.snapshot_id,
|
|
156
|
+
revision: committed.revision,
|
|
157
|
+
contentSha256: committed.content_sha256,
|
|
158
|
+
writtenAt: new Date().toISOString(),
|
|
159
|
+
});
|
|
160
|
+
return {
|
|
161
|
+
revision: committed.revision,
|
|
162
|
+
snapshotId: committed.snapshot_id,
|
|
163
|
+
contentSha256: committed.content_sha256,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
finally {
|
|
167
|
+
rmSync(staged.stagingDirectory, { recursive: true, force: true });
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Protocol limits frozen with Course Service's WorkspaceSnapshotPolicyV1.
|
|
3
|
+
* Deployment may enable the capability, but must never override these values.
|
|
4
|
+
*/
|
|
5
|
+
export declare const WORKSPACE_SNAPSHOT_POLICY_V1: {
|
|
6
|
+
readonly policyId: "WorkspaceSnapshotPolicyV1";
|
|
7
|
+
readonly limits: {
|
|
8
|
+
maxPathSegmentBytes: number;
|
|
9
|
+
maxPathBytes: number;
|
|
10
|
+
maxDepth: number;
|
|
11
|
+
maxFileBytes: number;
|
|
12
|
+
maxFileCount: number;
|
|
13
|
+
maxTotalBytes: number;
|
|
14
|
+
maxEntrySetBytes: number;
|
|
15
|
+
};
|
|
16
|
+
readonly maxStabilityRetries: 2;
|
|
17
|
+
readonly checkpointAttemptDeadlineMs: 30000;
|
|
18
|
+
readonly requiredRestoreFreeBytes: number;
|
|
19
|
+
readonly sessionRetainedHardLimitBytes: number;
|
|
20
|
+
readonly checkpointReservationBytes: number;
|
|
21
|
+
readonly restorePageEntries: 25;
|
|
22
|
+
};
|
|
23
|
+
/** Wire representation validated exactly by Agent Service during the v0.3 handshake. */
|
|
24
|
+
export declare function workspaceSnapshotPolicyCapability(): Record<string, unknown>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { RUNTIME_WORKSPACE_INODE_QUOTA, RUNTIME_WORKSPACE_QUOTA_BYTES, } from "./workspace-quota.js";
|
|
2
|
+
/**
|
|
3
|
+
* Protocol limits frozen with Course Service's WorkspaceSnapshotPolicyV1.
|
|
4
|
+
* Deployment may enable the capability, but must never override these values.
|
|
5
|
+
*/
|
|
6
|
+
export const WORKSPACE_SNAPSHOT_POLICY_V1 = {
|
|
7
|
+
policyId: "WorkspaceSnapshotPolicyV1",
|
|
8
|
+
limits: {
|
|
9
|
+
maxPathSegmentBytes: 255,
|
|
10
|
+
maxPathBytes: 1024,
|
|
11
|
+
maxDepth: 32,
|
|
12
|
+
maxFileBytes: 50 * 1024 * 1024,
|
|
13
|
+
maxFileCount: 2_000,
|
|
14
|
+
maxTotalBytes: 512 * 1024 * 1024,
|
|
15
|
+
maxEntrySetBytes: 1024 * 1024,
|
|
16
|
+
},
|
|
17
|
+
maxStabilityRetries: 2,
|
|
18
|
+
checkpointAttemptDeadlineMs: 30_000,
|
|
19
|
+
requiredRestoreFreeBytes: 1024 * 1024,
|
|
20
|
+
sessionRetainedHardLimitBytes: 2 * 1024 * 1024 * 1024,
|
|
21
|
+
checkpointReservationBytes: 512 * 1024 * 1024,
|
|
22
|
+
restorePageEntries: 25,
|
|
23
|
+
};
|
|
24
|
+
/** Wire representation validated exactly by Agent Service during the v0.3 handshake. */
|
|
25
|
+
export function workspaceSnapshotPolicyCapability() {
|
|
26
|
+
return {
|
|
27
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
28
|
+
max_path_segment_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxPathSegmentBytes,
|
|
29
|
+
max_path_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxPathBytes,
|
|
30
|
+
max_depth: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxDepth,
|
|
31
|
+
max_file_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxFileBytes,
|
|
32
|
+
max_file_count: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxFileCount,
|
|
33
|
+
max_total_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxTotalBytes,
|
|
34
|
+
max_entry_set_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxEntrySetBytes,
|
|
35
|
+
required_restore_free_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes,
|
|
36
|
+
session_retained_hard_limit_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.sessionRetainedHardLimitBytes,
|
|
37
|
+
checkpoint_reservation_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.checkpointReservationBytes,
|
|
38
|
+
restore_page_entries: WORKSPACE_SNAPSHOT_POLICY_V1.restorePageEntries,
|
|
39
|
+
max_stability_retries: WORKSPACE_SNAPSHOT_POLICY_V1.maxStabilityRetries,
|
|
40
|
+
checkpoint_attempt_deadline_ms: WORKSPACE_SNAPSHOT_POLICY_V1.checkpointAttemptDeadlineMs,
|
|
41
|
+
prepare_deadline_seconds: 15 * 60,
|
|
42
|
+
runtime_filesystem_quota_bytes: RUNTIME_WORKSPACE_QUOTA_BYTES,
|
|
43
|
+
runtime_inode_quota: RUNTIME_WORKSPACE_INODE_QUOTA,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type WorkspaceQuiescenceProof } from "./runtime-quiescence.js";
|
|
1
2
|
import { type CanonicalWorkspaceEntrySet, type WorkspaceEntrySetLimits } from "./workspace-entry-set.js";
|
|
2
3
|
export declare class WorkspaceSnapshotStagingError extends Error {
|
|
3
4
|
readonly code: string;
|
|
@@ -23,12 +24,4 @@ export interface WorkspaceSnapshotStagingOptions {
|
|
|
23
24
|
maxStabilityRetries: number;
|
|
24
25
|
quiescenceProof: WorkspaceQuiescenceProof;
|
|
25
26
|
}
|
|
26
|
-
export interface WorkspaceQuiescenceProof {
|
|
27
|
-
proofVersion: "workspace-quiescence-proof/1";
|
|
28
|
-
sandboxId: string;
|
|
29
|
-
sandboxGeneration: number;
|
|
30
|
-
runtimeSessionId: string;
|
|
31
|
-
checkpointId: string;
|
|
32
|
-
allWritersStopped: true;
|
|
33
|
-
}
|
|
34
27
|
export declare function stageWorkspaceSnapshot(options: WorkspaceSnapshotStagingOptions): StagedWorkspaceSnapshot;
|