@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-service-sandbox.d.ts +7 -1
- package/dist/agent-service-sandbox.js +384 -16
- package/dist/agent-service-ws-protocol.d.ts +3 -3
- package/dist/agent-service-ws-protocol.js +4 -2
- package/dist/cli.js +19 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/run-dispatcher.d.ts +1 -1
- package/dist/run-dispatcher.js +1 -1
- package/dist/runtime-env.js +4 -4
- package/dist/runtime-quiescence.d.ts +16 -0
- package/dist/runtime-quiescence.js +42 -0
- package/dist/workspace-quota.d.ts +4 -0
- package/dist/workspace-quota.js +42 -0
- package/dist/workspace-restore.d.ts +9 -1
- package/dist/workspace-restore.js +30 -16
- package/dist/workspace-snapshot-control.d.ts +29 -0
- package/dist/workspace-snapshot-control.js +169 -0
- package/dist/workspace-snapshot-policy.d.ts +24 -0
- package/dist/workspace-snapshot-policy.js +45 -0
- package/dist/workspace-snapshot-staging.d.ts +1 -8
- package/dist/workspace-snapshot-staging.js +53 -19
- package/package.json +1 -1
|
@@ -33,12 +33,15 @@ export declare class AgentServiceSandboxClient implements RunReportingClient, Pe
|
|
|
33
33
|
private readonly dispatcher;
|
|
34
34
|
/** agent_run_id → TURN scope(session/attempt/activation),事件与文件帧路由用。 */
|
|
35
35
|
private readonly turnScopes;
|
|
36
|
+
private readonly turnWorkspaceCheckpoints;
|
|
37
|
+
private readonly completedWorkspaceCheckpoints;
|
|
36
38
|
/** 每 run 内嵌 profile(turn payload 自带时优先)。 */
|
|
37
39
|
private readonly runProfiles;
|
|
38
40
|
/** activate 时装配的 per-session profile,turn.start 复用。 */
|
|
39
41
|
private readonly sessionProfiles;
|
|
40
42
|
/** activation-scoped 模型短凭据;只保存在内存中,绝不进入 state.json。 */
|
|
41
43
|
private readonly activationContexts;
|
|
44
|
+
private readonly activationCleanupTasks;
|
|
42
45
|
private readonly pendingAcks;
|
|
43
46
|
private readonly pendingFileReports;
|
|
44
47
|
private readonly inflightCommands;
|
|
@@ -70,11 +73,13 @@ export declare class AgentServiceSandboxClient implements RunReportingClient, Pe
|
|
|
70
73
|
constructor(options: AgentServiceSandboxOptions);
|
|
71
74
|
prepareTurn(payload: RunStartPayload): PreparedPersistentTurn;
|
|
72
75
|
persistNativeSession(nativeSessionId: string): void;
|
|
73
|
-
finishTurn(payload: RunStartPayload): void
|
|
76
|
+
finishTurn(payload: RunStartPayload): Promise<void>;
|
|
74
77
|
private completePendingActivationCleanup;
|
|
78
|
+
private runPendingActivationCleanup;
|
|
75
79
|
private recoverPendingActivationCleanups;
|
|
76
80
|
run(): Promise<void>;
|
|
77
81
|
stop(): void;
|
|
82
|
+
stopGracefully(): Promise<void>;
|
|
78
83
|
postEvent(agentRunId: string, event: RunEvent): Promise<RunEventReceipt>;
|
|
79
84
|
postFile(agentRunId: string, file: RunFileCandidate): Promise<RunFileRecord>;
|
|
80
85
|
recordWorkspaceScanMeasurement(agentRunId: string, measurement: WorkspaceScanMeasurement): Promise<void>;
|
|
@@ -88,6 +93,7 @@ export declare class AgentServiceSandboxClient implements RunReportingClient, Pe
|
|
|
88
93
|
/** sandbox.sync 只做对账:重放 spool、关闭待关 session、应用 drain/shutdown。 */
|
|
89
94
|
private applySync;
|
|
90
95
|
private handleSessionOpen;
|
|
96
|
+
private handleWorkspaceCheckpoint;
|
|
91
97
|
private handleSessionActivate;
|
|
92
98
|
private handleTurnStart;
|
|
93
99
|
private handleTurnCancel;
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, closeSync, createWriteStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import { Readable } from "node:stream";
|
|
4
|
+
import { pipeline } from "node:stream/promises";
|
|
2
5
|
import path from "node:path";
|
|
3
6
|
import { ensureDaemonHome } from "./auth-store.js";
|
|
4
7
|
import { AGENT_SERVICE_WS_SCHEMA, AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, createWorkspaceFileChunk, parseSandboxFrame, UnsupportedSandboxProtocolError, WORKSPACE_FILE_READ_BINARY_CAPABILITY, } from "./agent-service-ws-protocol.js";
|
|
@@ -10,6 +13,14 @@ import { parseRuntimeSkillProviderGrantSet, prepareRuntimeSkillProvider, Runtime
|
|
|
10
13
|
import { RunDispatcher, } from "./run-dispatcher.js";
|
|
11
14
|
import { cleanupRuntimeSessionWorkspaceCopyStaging, copyRuntimeSessionWorkspace, ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, finalizeRuntimeSessionWorkspaceCopy, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, WORKSPACE_COPY_POLICY_V1, WorkspaceCopyError, workspaceCopyV1Supported, } from "./workspace.js";
|
|
12
15
|
import { readWorkspaceFile, WorkspaceFileReadError } from "./workspace-file-read.js";
|
|
16
|
+
import { checkpointWorkspace, WorkspaceSnapshotControlError, } from "./workspace-snapshot-control.js";
|
|
17
|
+
import { issueWorkspaceQuiescenceProof, quiesceRuntimeWriters, } from "./runtime-quiescence.js";
|
|
18
|
+
import { WorkspaceSnapshotStagingError } from "./workspace-snapshot-staging.js";
|
|
19
|
+
import { validateWorkspaceEntrySet, WorkspaceEntrySetError, } from "./workspace-entry-set.js";
|
|
20
|
+
import { WORKSPACE_SNAPSHOT_POLICY_V1, workspaceSnapshotPolicyCapability, } from "./workspace-snapshot-policy.js";
|
|
21
|
+
import { assertWorkspaceRestoreCapacity, restoreWorkspaceSnapshot, WorkspaceRestoreError, } from "./workspace-restore.js";
|
|
22
|
+
import { readWorkspaceMaterializationMarker } from "./workspace-materialization.js";
|
|
23
|
+
import { applyRuntimeWorkspaceQuota, assertRuntimeWorkspaceQuota } from "./workspace-quota.js";
|
|
13
24
|
import { WebSocketClient, } from "./websocket-client.js";
|
|
14
25
|
const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
|
|
15
26
|
const MAX_SPOOL_FRAMES = 1024;
|
|
@@ -19,6 +30,14 @@ const LEGACY_EVENT_ACK_WINDOW = 1;
|
|
|
19
30
|
const INPUT_ATTACHMENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
20
31
|
/** 出站 seq 基址:seq = connection_epoch * SEQ_EPOCH_BASE + n,跨重连单调(合同 §1.1)。 */
|
|
21
32
|
const SEQ_EPOCH_BASE = 1_000_000_000;
|
|
33
|
+
function agentServiceHttpBase(wsUrl) {
|
|
34
|
+
const url = new URL(wsUrl);
|
|
35
|
+
url.protocol = url.protocol === "wss:" ? "https:" : "http:";
|
|
36
|
+
url.pathname = "/internal/v1";
|
|
37
|
+
url.search = "";
|
|
38
|
+
url.hash = "";
|
|
39
|
+
return url.toString().replace(/\/$/, "");
|
|
40
|
+
}
|
|
22
41
|
const RUNTIME_LOG_WINDOW_MS = 60_000;
|
|
23
42
|
const WORKSPACE_FILE_TRANSFER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
24
43
|
const DISABLED_RUNTIME_LOG_POLICY = {
|
|
@@ -340,12 +359,15 @@ export class AgentServiceSandboxClient {
|
|
|
340
359
|
dispatcher;
|
|
341
360
|
/** agent_run_id → TURN scope(session/attempt/activation),事件与文件帧路由用。 */
|
|
342
361
|
turnScopes = new Map();
|
|
362
|
+
turnWorkspaceCheckpoints = new Map();
|
|
363
|
+
completedWorkspaceCheckpoints = new Set();
|
|
343
364
|
/** 每 run 内嵌 profile(turn payload 自带时优先)。 */
|
|
344
365
|
runProfiles = new Map();
|
|
345
366
|
/** activate 时装配的 per-session profile,turn.start 复用。 */
|
|
346
367
|
sessionProfiles = new Map();
|
|
347
368
|
/** activation-scoped 模型短凭据;只保存在内存中,绝不进入 state.json。 */
|
|
348
369
|
activationContexts = new Map();
|
|
370
|
+
activationCleanupTasks = new Map();
|
|
349
371
|
pendingAcks = new Map();
|
|
350
372
|
pendingFileReports = new Map();
|
|
351
373
|
inflightCommands = new Set();
|
|
@@ -438,7 +460,7 @@ export class AgentServiceSandboxClient {
|
|
|
438
460
|
session.nativeSessionId = nativeSessionId.trim() || null;
|
|
439
461
|
this.persist();
|
|
440
462
|
}
|
|
441
|
-
finishTurn(payload) {
|
|
463
|
+
async finishTurn(payload) {
|
|
442
464
|
const scope = this.turnScopes.get(payload.agent_run_id);
|
|
443
465
|
if (!scope)
|
|
444
466
|
return;
|
|
@@ -454,7 +476,7 @@ export class AgentServiceSandboxClient {
|
|
|
454
476
|
};
|
|
455
477
|
this.persist();
|
|
456
478
|
try {
|
|
457
|
-
this.completePendingActivationCleanup(scope.sessionId);
|
|
479
|
+
await this.completePendingActivationCleanup(scope.sessionId);
|
|
458
480
|
}
|
|
459
481
|
catch (error) {
|
|
460
482
|
this.log.error("failed to revoke a terminal runtime session workspace", {
|
|
@@ -472,11 +494,27 @@ export class AgentServiceSandboxClient {
|
|
|
472
494
|
this.currentTurnSessionId = null;
|
|
473
495
|
this.persist();
|
|
474
496
|
}
|
|
475
|
-
completePendingActivationCleanup(sessionId) {
|
|
497
|
+
async completePendingActivationCleanup(sessionId) {
|
|
498
|
+
const existing = this.activationCleanupTasks.get(sessionId);
|
|
499
|
+
if (existing)
|
|
500
|
+
return await existing;
|
|
501
|
+
const task = this.runPendingActivationCleanup(sessionId);
|
|
502
|
+
this.activationCleanupTasks.set(sessionId, task);
|
|
503
|
+
try {
|
|
504
|
+
await task;
|
|
505
|
+
}
|
|
506
|
+
finally {
|
|
507
|
+
if (this.activationCleanupTasks.get(sessionId) === task) {
|
|
508
|
+
this.activationCleanupTasks.delete(sessionId);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
async runPendingActivationCleanup(sessionId) {
|
|
476
513
|
const session = this.state.sessions[sessionId];
|
|
477
514
|
const pending = session?.pendingActivationCleanup;
|
|
478
515
|
if (!session || !pending)
|
|
479
516
|
return;
|
|
517
|
+
await quiesceRuntimeWriters(pending.activationId);
|
|
480
518
|
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
481
519
|
const activation = this.activationContexts.get(sessionId);
|
|
482
520
|
if (activation?.activationId === pending.activationId) {
|
|
@@ -505,9 +543,9 @@ export class AgentServiceSandboxClient {
|
|
|
505
543
|
this.currentTurnSessionId = null;
|
|
506
544
|
this.persist();
|
|
507
545
|
}
|
|
508
|
-
recoverPendingActivationCleanups() {
|
|
546
|
+
async recoverPendingActivationCleanups() {
|
|
509
547
|
for (const sessionId of Object.keys(this.state.sessions)) {
|
|
510
|
-
this.completePendingActivationCleanup(sessionId);
|
|
548
|
+
await this.completePendingActivationCleanup(sessionId);
|
|
511
549
|
}
|
|
512
550
|
}
|
|
513
551
|
async run() {
|
|
@@ -568,6 +606,11 @@ export class AgentServiceSandboxClient {
|
|
|
568
606
|
this.removeOperationalLogSink();
|
|
569
607
|
this.socket?.close(1000, "daemon_stopping");
|
|
570
608
|
}
|
|
609
|
+
async stopGracefully() {
|
|
610
|
+
this.dispatcher.cancelAll();
|
|
611
|
+
await quiesceRuntimeWriters(null);
|
|
612
|
+
this.stop();
|
|
613
|
+
}
|
|
571
614
|
async postEvent(agentRunId, event) {
|
|
572
615
|
const scope = this.turnScopes.get(agentRunId);
|
|
573
616
|
if (!scope)
|
|
@@ -578,6 +621,9 @@ export class AgentServiceSandboxClient {
|
|
|
578
621
|
if (!this.sandboxGeneration || !this.connectionEpoch) {
|
|
579
622
|
throw new Error("Agent Service sandbox is not authenticated");
|
|
580
623
|
}
|
|
624
|
+
// Terminal truth is never delayed by object storage. Course Service keeps the
|
|
625
|
+
// checkpoint obligation pending and, after this turn is ACKed and fully quiesced,
|
|
626
|
+
// re-delivers it as a session-scoped workspace.checkpoint command.
|
|
581
627
|
if (event.type === "run.block") {
|
|
582
628
|
await this.waitForEventCapacity();
|
|
583
629
|
}
|
|
@@ -782,6 +828,7 @@ export class AgentServiceSandboxClient {
|
|
|
782
828
|
if (frame.sandbox_generation !== this.state.sandboxGeneration) {
|
|
783
829
|
// Generation change wipes every session shard (contract §1.4).
|
|
784
830
|
this.dispatcher.cancelAll();
|
|
831
|
+
await quiesceRuntimeWriters(null);
|
|
785
832
|
this.rejectPending(new Error("Agent Service sandbox generation changed"));
|
|
786
833
|
for (const sessionId of Object.keys(this.state.sessions)) {
|
|
787
834
|
revokeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
|
|
@@ -800,7 +847,7 @@ export class AgentServiceSandboxClient {
|
|
|
800
847
|
this.connectionEpoch = frame.connection_epoch;
|
|
801
848
|
this.inboundSeq = frame.seq;
|
|
802
849
|
this.outboundSeq = frame.connection_epoch * SEQ_EPOCH_BASE;
|
|
803
|
-
this.recoverPendingActivationCleanups();
|
|
850
|
+
await this.recoverPendingActivationCleanups();
|
|
804
851
|
const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
|
|
805
852
|
this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
|
|
806
853
|
const staleSeconds = Number(frame.payload.stale_after_seconds ?? 45);
|
|
@@ -815,12 +862,16 @@ export class AgentServiceSandboxClient {
|
|
|
815
862
|
this.runtimeLogPolicy = runtimeLogPolicy(frame.payload.runtime_log_policy);
|
|
816
863
|
await cleanupRuntimeSessionWorkspaceCopyStaging();
|
|
817
864
|
this.persist();
|
|
865
|
+
assertRuntimeWorkspaceQuota(process.env.BOTLEARN_AGENT_SERVICE_WORKSPACE_ROOT ?? "");
|
|
818
866
|
await this.sendControlFrame("sandbox.ready", {
|
|
819
867
|
protocol_versions: [AGENT_SERVICE_WS_SCHEMA],
|
|
820
868
|
capabilities: [
|
|
821
869
|
WORKSPACE_FILE_READ_BINARY_CAPABILITY,
|
|
870
|
+
"workspace_snapshot_v1",
|
|
871
|
+
"workspace_restore_v1",
|
|
822
872
|
...(workspaceCopyV1Supported() ? ["workspace_copy_v1"] : []),
|
|
823
873
|
],
|
|
874
|
+
workspace_snapshot_policy: workspaceSnapshotPolicyCapability(),
|
|
824
875
|
daemon_version: this.options.daemonVersion,
|
|
825
876
|
runtime_versions: {
|
|
826
877
|
course_daemon: this.options.daemonVersion,
|
|
@@ -952,8 +1003,11 @@ export class AgentServiceSandboxClient {
|
|
|
952
1003
|
});
|
|
953
1004
|
});
|
|
954
1005
|
return;
|
|
1006
|
+
case "workspace.checkpoint":
|
|
1007
|
+
this.scheduleLifecycle("workspace.checkpoint", () => this.handleWorkspaceCheckpoint(frame));
|
|
1008
|
+
return;
|
|
955
1009
|
case "sandbox.shutdown":
|
|
956
|
-
this.
|
|
1010
|
+
await this.stopGracefully();
|
|
957
1011
|
return;
|
|
958
1012
|
case "sandbox.drain":
|
|
959
1013
|
await this.sendCommandAck(frame, "ok");
|
|
@@ -980,7 +1034,7 @@ export class AgentServiceSandboxClient {
|
|
|
980
1034
|
}
|
|
981
1035
|
const state = frame.payload.state;
|
|
982
1036
|
if (state === "shutdown") {
|
|
983
|
-
this.
|
|
1037
|
+
await this.stopGracefully();
|
|
984
1038
|
return;
|
|
985
1039
|
}
|
|
986
1040
|
if (state === "drain") {
|
|
@@ -998,6 +1052,7 @@ export class AgentServiceSandboxClient {
|
|
|
998
1052
|
const courseRunId = frame.payload.course_run_id;
|
|
999
1053
|
const requestedWorkspaceRef = frame.payload.workspace_ref;
|
|
1000
1054
|
const requestedCopy = frame.payload.workspace_copy;
|
|
1055
|
+
const requestedRestore = frame.payload.workspace_restore;
|
|
1001
1056
|
const requestedCopyRecord = requestedCopy && typeof requestedCopy === "object"
|
|
1002
1057
|
? requestedCopy
|
|
1003
1058
|
: undefined;
|
|
@@ -1018,7 +1073,11 @@ export class AgentServiceSandboxClient {
|
|
|
1018
1073
|
typeof copyId !== "string" ||
|
|
1019
1074
|
copyId.length < 1 ||
|
|
1020
1075
|
typeof requestedCopySource !== "string" ||
|
|
1021
|
-
requestedCopySource.length < 1))
|
|
1076
|
+
requestedCopySource.length < 1))
|
|
1077
|
+
|| (requestedRestore !== null && requestedRestore !== undefined &&
|
|
1078
|
+
(typeof requestedRestore !== "object" || Array.isArray(requestedRestore)))
|
|
1079
|
+
|| (requestedCopy !== null && requestedCopy !== undefined &&
|
|
1080
|
+
requestedRestore !== null && requestedRestore !== undefined)) {
|
|
1022
1081
|
throw new Error("invalid session.open payload");
|
|
1023
1082
|
}
|
|
1024
1083
|
if (session.courseRunId !== null && session.courseRunId !== courseRunId) {
|
|
@@ -1032,6 +1091,165 @@ export class AgentServiceSandboxClient {
|
|
|
1032
1091
|
const isNewSession = session.courseRunId === null;
|
|
1033
1092
|
let markerPath = null;
|
|
1034
1093
|
try {
|
|
1094
|
+
if (requestedRestore && typeof requestedRestore === "object") {
|
|
1095
|
+
const raw = requestedRestore;
|
|
1096
|
+
const descriptorRaw = raw.descriptor;
|
|
1097
|
+
const controlFence = raw.control_fence;
|
|
1098
|
+
let restoreControlToken = raw.control_token;
|
|
1099
|
+
if (!controlFence || typeof controlFence !== "object" || Array.isArray(controlFence)) {
|
|
1100
|
+
throw new Error("invalid workspace restore control fence");
|
|
1101
|
+
}
|
|
1102
|
+
if (typeof restoreControlToken !== "string" || restoreControlToken.length < 32) {
|
|
1103
|
+
throw new Error("invalid workspace restore control token");
|
|
1104
|
+
}
|
|
1105
|
+
if (!descriptorRaw || !Array.isArray(raw.entries)) {
|
|
1106
|
+
throw new Error("invalid workspace restore payload");
|
|
1107
|
+
}
|
|
1108
|
+
const descriptor = {
|
|
1109
|
+
sandboxId: this.options.sandboxId,
|
|
1110
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1111
|
+
runtimeSessionId: sessionId,
|
|
1112
|
+
continuityState: descriptorRaw.continuity_state,
|
|
1113
|
+
continuityErrorCode: descriptorRaw.continuity_error_code,
|
|
1114
|
+
snapshotId: descriptorRaw.snapshot_id,
|
|
1115
|
+
revision: Number(descriptorRaw.revision),
|
|
1116
|
+
contentSha256: descriptorRaw.content_sha256,
|
|
1117
|
+
expectedFileCount: Number(descriptorRaw.expected_file_count),
|
|
1118
|
+
expectedEntryCount: Number(descriptorRaw.expected_entry_count),
|
|
1119
|
+
expectedTotalBytes: Number(descriptorRaw.expected_total_bytes),
|
|
1120
|
+
};
|
|
1121
|
+
const workspaceDirectory = runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration);
|
|
1122
|
+
const requiredFreeBytes = Math.max(WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes, descriptor.expectedTotalBytes + WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes);
|
|
1123
|
+
// Fail before issuing or consuming any object grant. The download staging is a
|
|
1124
|
+
// sibling on this same bounded backing filesystem and is renamed, not copied,
|
|
1125
|
+
// into the atomic restore staging tree.
|
|
1126
|
+
assertWorkspaceRestoreCapacity({
|
|
1127
|
+
workspaceDirectory,
|
|
1128
|
+
descriptor,
|
|
1129
|
+
requiredFreeBytes,
|
|
1130
|
+
entryCopies: 2,
|
|
1131
|
+
});
|
|
1132
|
+
const entries = [];
|
|
1133
|
+
const predownloadRoot = path.join(path.dirname(workspaceDirectory), `.botlearn-predownload-${sessionId}-${randomUUID()}`);
|
|
1134
|
+
mkdirSync(predownloadRoot, { recursive: true, mode: 0o700 });
|
|
1135
|
+
try {
|
|
1136
|
+
let nextCursor = null;
|
|
1137
|
+
do {
|
|
1138
|
+
if (nextCursor !== null && nextCursor.length > 2048) {
|
|
1139
|
+
throw new Error("invalid workspace restore cursor");
|
|
1140
|
+
}
|
|
1141
|
+
const pageSignal = AbortSignal.timeout(60_000);
|
|
1142
|
+
const response = await fetch(`${agentServiceHttpBase(this.options.wsUrl)}/runtime-sessions/${sessionId}/restore-grants`, {
|
|
1143
|
+
method: "POST",
|
|
1144
|
+
headers: {
|
|
1145
|
+
Authorization: `Bearer ${restoreControlToken}`,
|
|
1146
|
+
"Content-Type": "application/json",
|
|
1147
|
+
},
|
|
1148
|
+
body: JSON.stringify({
|
|
1149
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
1150
|
+
fence: controlFence,
|
|
1151
|
+
cursor: nextCursor,
|
|
1152
|
+
// Fetch one canonical entry at a time so a grant is consumed as soon as
|
|
1153
|
+
// it is signed, including deployments using the minimum 60-second TTL.
|
|
1154
|
+
max_entries: 1,
|
|
1155
|
+
}),
|
|
1156
|
+
signal: pageSignal,
|
|
1157
|
+
});
|
|
1158
|
+
if (!response.ok) {
|
|
1159
|
+
let code = "workspace_restore_unavailable";
|
|
1160
|
+
let retryable = response.status >= 500 || response.status === 408 ||
|
|
1161
|
+
response.status === 429;
|
|
1162
|
+
try {
|
|
1163
|
+
const rejected = await response.json();
|
|
1164
|
+
const detail = rejected.detail;
|
|
1165
|
+
if (detail && typeof detail === "object" && !Array.isArray(detail)) {
|
|
1166
|
+
const stableCode = detail.code;
|
|
1167
|
+
if (typeof stableCode === "string" && stableCode.length <= 120) {
|
|
1168
|
+
code = stableCode;
|
|
1169
|
+
}
|
|
1170
|
+
if (typeof detail.retryable === "boolean") {
|
|
1171
|
+
retryable = detail.retryable;
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
catch {
|
|
1176
|
+
// Content-free fallback retains the stable local failure code.
|
|
1177
|
+
}
|
|
1178
|
+
throw new WorkspaceRestoreError(code, retryable);
|
|
1179
|
+
}
|
|
1180
|
+
const page = await response.json();
|
|
1181
|
+
if (!Array.isArray(page.entries))
|
|
1182
|
+
throw new Error("invalid workspace restore page");
|
|
1183
|
+
if (typeof page.control_token !== "string" || page.control_token.length < 32) {
|
|
1184
|
+
throw new Error("invalid refreshed workspace restore control token");
|
|
1185
|
+
}
|
|
1186
|
+
restoreControlToken = page.control_token;
|
|
1187
|
+
const pageFiles = [];
|
|
1188
|
+
for (const value of page.entries) {
|
|
1189
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1190
|
+
throw new Error("invalid workspace restore entry");
|
|
1191
|
+
}
|
|
1192
|
+
const item = value;
|
|
1193
|
+
if (item.entry_type === "directory") {
|
|
1194
|
+
entries.push({ type: "directory", path: String(item.path), mode: "0700" });
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
if (item.entry_type !== "file" || !item.grant || typeof item.grant !== "object") {
|
|
1198
|
+
throw new Error("invalid workspace restore file grant");
|
|
1199
|
+
}
|
|
1200
|
+
const entry = {
|
|
1201
|
+
type: "file",
|
|
1202
|
+
path: String(item.path),
|
|
1203
|
+
mode: item.mode,
|
|
1204
|
+
sizeBytes: Number(item.size_bytes),
|
|
1205
|
+
sha256: String(item.sha256),
|
|
1206
|
+
};
|
|
1207
|
+
entries.push(entry);
|
|
1208
|
+
pageFiles.push({ entry, grant: item.grant });
|
|
1209
|
+
}
|
|
1210
|
+
// Validate all paths and duplicates before using any page path locally.
|
|
1211
|
+
validateWorkspaceEntrySet({ schemaVersion: "agent-workspace-entry-set/1", entries }, WORKSPACE_SNAPSHOT_POLICY_V1.limits);
|
|
1212
|
+
for (const { entry, grant } of pageFiles) {
|
|
1213
|
+
if (typeof grant.url !== "string")
|
|
1214
|
+
throw new Error("workspace restore grant missing");
|
|
1215
|
+
const destination = path.join(predownloadRoot, ...entry.path.split("/"));
|
|
1216
|
+
mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
|
|
1217
|
+
const part = `${destination}.part`;
|
|
1218
|
+
const downloadSignal = AbortSignal.timeout(60_000);
|
|
1219
|
+
const download = await fetch(grant.url, {
|
|
1220
|
+
method: "GET",
|
|
1221
|
+
headers: grant.headers,
|
|
1222
|
+
signal: downloadSignal,
|
|
1223
|
+
});
|
|
1224
|
+
if (!download.ok)
|
|
1225
|
+
throw new Error(`workspace_restore_download_${download.status}`);
|
|
1226
|
+
if (!download.body)
|
|
1227
|
+
throw new Error("workspace_restore_download_body_missing");
|
|
1228
|
+
await pipeline(Readable.fromWeb(download.body), createWriteStream(part, { mode: 0o600 }), { signal: downloadSignal });
|
|
1229
|
+
renameSync(part, destination);
|
|
1230
|
+
}
|
|
1231
|
+
const rawNext = page.next_cursor;
|
|
1232
|
+
if (rawNext !== null && rawNext !== undefined && typeof rawNext !== "string") {
|
|
1233
|
+
throw new Error("invalid workspace restore cursor");
|
|
1234
|
+
}
|
|
1235
|
+
nextCursor = rawNext ?? null;
|
|
1236
|
+
} while (nextCursor !== null);
|
|
1237
|
+
await restoreWorkspaceSnapshot({
|
|
1238
|
+
workspaceDirectory,
|
|
1239
|
+
controlDirectory: path.join(ensureDaemonHome(), "agent-service-sandboxes", this.options.sandboxId, "workspace-control"),
|
|
1240
|
+
descriptor,
|
|
1241
|
+
entrySet: { schemaVersion: "agent-workspace-entry-set/1", entries },
|
|
1242
|
+
limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
|
|
1243
|
+
requiredFreeBytes,
|
|
1244
|
+
downloadFile: async (entry, partPath) => {
|
|
1245
|
+
renameSync(path.join(predownloadRoot, ...entry.path.split("/")), partPath);
|
|
1246
|
+
},
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
finally {
|
|
1250
|
+
rmSync(predownloadRoot, { recursive: true, force: true });
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1035
1253
|
if (typeof requestedCopySource === "string" && typeof copyId === "string") {
|
|
1036
1254
|
if (session.workspaceCopyReceipt !== null) {
|
|
1037
1255
|
const receipt = session.workspaceCopyReceipt;
|
|
@@ -1075,28 +1293,160 @@ export class AgentServiceSandboxClient {
|
|
|
1075
1293
|
});
|
|
1076
1294
|
return;
|
|
1077
1295
|
}
|
|
1296
|
+
if (requestedRestore) {
|
|
1297
|
+
const restoreError = error instanceof WorkspaceRestoreError ? error : null;
|
|
1298
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1299
|
+
error_code: restoreError?.code ?? "workspace_restore_unavailable",
|
|
1300
|
+
retryable: restoreError?.retryable ?? true,
|
|
1301
|
+
});
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1078
1304
|
throw error;
|
|
1079
1305
|
}
|
|
1080
1306
|
session.courseRunId = courseRunId;
|
|
1081
1307
|
session.workspaceRef = session.workspaceRef ?? (typeof requestedWorkspaceRef === "string"
|
|
1082
1308
|
? requestedWorkspaceRef
|
|
1083
1309
|
: `ws_${sessionId.replaceAll("-", "")}_g${this.sandboxGeneration}`);
|
|
1084
|
-
|
|
1310
|
+
// Replaying session.open is part of reconnect reconciliation. An already-active
|
|
1311
|
+
// runtime may still be creating files in this workspace, so never transiently revoke
|
|
1312
|
+
// its group write/execute access before the following session.activate arrives.
|
|
1313
|
+
if (session.workspaceCopyReceipt === null && (isNewSession || session.activationId === null)) {
|
|
1085
1314
|
ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
|
|
1086
1315
|
}
|
|
1087
|
-
this.state.sessions[sessionId] = session;
|
|
1088
|
-
this.persist();
|
|
1089
1316
|
if (markerPath !== null) {
|
|
1090
1317
|
await finalizeRuntimeSessionWorkspaceCopy(markerPath);
|
|
1091
1318
|
}
|
|
1319
|
+
try {
|
|
1320
|
+
await applyRuntimeWorkspaceQuota(sessionId, this.sandboxGeneration);
|
|
1321
|
+
}
|
|
1322
|
+
catch (error) {
|
|
1323
|
+
if (requestedRestore) {
|
|
1324
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1325
|
+
error_code: "workspace_restore_unavailable",
|
|
1326
|
+
});
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
if (requestedCopy) {
|
|
1330
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1331
|
+
error_code: "workspace_migration_io_failed",
|
|
1332
|
+
});
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
throw error;
|
|
1336
|
+
}
|
|
1337
|
+
this.state.sessions[sessionId] = session;
|
|
1338
|
+
this.persist();
|
|
1339
|
+
const materialization = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
|
|
1092
1340
|
await this.sendSessionFrame("session.opened", sessionId, {
|
|
1093
1341
|
workspace_ref: session.workspaceRef,
|
|
1094
1342
|
native_session_id: session.nativeSessionId,
|
|
1343
|
+
...(materialization
|
|
1344
|
+
? {
|
|
1345
|
+
workspace_marker_version: 1,
|
|
1346
|
+
workspace_continuity_state: materialization.workspaceContinuityState,
|
|
1347
|
+
workspace_snapshot_id: materialization.snapshotId,
|
|
1348
|
+
workspace_revision: materialization.revision,
|
|
1349
|
+
workspace_content_sha256: materialization.contentSha256,
|
|
1350
|
+
}
|
|
1351
|
+
: {}),
|
|
1095
1352
|
...(session.workspaceCopyReceipt !== null
|
|
1096
1353
|
? { workspace_copy_receipt: session.workspaceCopyReceipt }
|
|
1097
1354
|
: {}),
|
|
1098
1355
|
});
|
|
1099
1356
|
}
|
|
1357
|
+
async handleWorkspaceCheckpoint(frame) {
|
|
1358
|
+
const sessionId = frame.runtime_session_id;
|
|
1359
|
+
const session = this.state.sessions[sessionId];
|
|
1360
|
+
const checkpointId = frame.payload.checkpoint_id;
|
|
1361
|
+
const controlToken = frame.payload.control_token;
|
|
1362
|
+
const baseRevision = Number(frame.payload.base_revision);
|
|
1363
|
+
const agentRunId = frame.payload.agent_run_id;
|
|
1364
|
+
const workerAttempt = frame.payload.worker_attempt;
|
|
1365
|
+
const activationId = frame.payload.activation_id;
|
|
1366
|
+
const pendingCleanup = session?.pendingActivationCleanup;
|
|
1367
|
+
if (pendingCleanup &&
|
|
1368
|
+
pendingCleanup.agentRunId === agentRunId &&
|
|
1369
|
+
pendingCleanup.workerAttempt === workerAttempt &&
|
|
1370
|
+
pendingCleanup.activationId === activationId) {
|
|
1371
|
+
await this.completePendingActivationCleanup(sessionId);
|
|
1372
|
+
}
|
|
1373
|
+
if (!session ||
|
|
1374
|
+
this.activeSessionId === sessionId ||
|
|
1375
|
+
this.currentTurnSessionId === sessionId ||
|
|
1376
|
+
typeof checkpointId !== "string" ||
|
|
1377
|
+
typeof controlToken !== "string" ||
|
|
1378
|
+
controlToken.length < 32 ||
|
|
1379
|
+
frame.payload.policy_id !== "WorkspaceSnapshotPolicyV1" ||
|
|
1380
|
+
!Number.isSafeInteger(baseRevision) ||
|
|
1381
|
+
baseRevision < 0 ||
|
|
1382
|
+
!((agentRunId == null && workerAttempt == null && activationId == null) ||
|
|
1383
|
+
(typeof agentRunId === "string" &&
|
|
1384
|
+
Number.isSafeInteger(workerAttempt) &&
|
|
1385
|
+
Number(workerAttempt) >= 1 &&
|
|
1386
|
+
typeof activationId === "string" &&
|
|
1387
|
+
activationId.length > 0))) {
|
|
1388
|
+
await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
|
|
1389
|
+
return;
|
|
1390
|
+
}
|
|
1391
|
+
try {
|
|
1392
|
+
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1393
|
+
await quiesceRuntimeWriters(typeof activationId === "string" ? activationId : null);
|
|
1394
|
+
const quiescenceProof = issueWorkspaceQuiescenceProof({
|
|
1395
|
+
sandboxId: this.options.sandboxId,
|
|
1396
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1397
|
+
runtimeSessionId: sessionId,
|
|
1398
|
+
checkpointId,
|
|
1399
|
+
});
|
|
1400
|
+
await checkpointWorkspace({
|
|
1401
|
+
wsUrl: this.options.wsUrl,
|
|
1402
|
+
reconnectToken: controlToken,
|
|
1403
|
+
workspaceDirectory: runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration),
|
|
1404
|
+
scope: {
|
|
1405
|
+
checkpointId,
|
|
1406
|
+
baseRevision,
|
|
1407
|
+
sandboxId: this.options.sandboxId,
|
|
1408
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1409
|
+
connectionEpoch: this.connectionEpoch,
|
|
1410
|
+
runtimeSessionId: sessionId,
|
|
1411
|
+
...(typeof agentRunId === "string"
|
|
1412
|
+
? {
|
|
1413
|
+
agentRunId,
|
|
1414
|
+
workerAttempt: Number(workerAttempt),
|
|
1415
|
+
activationId: String(activationId),
|
|
1416
|
+
}
|
|
1417
|
+
: {}),
|
|
1418
|
+
},
|
|
1419
|
+
quiescenceProof,
|
|
1420
|
+
});
|
|
1421
|
+
const marker = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
|
|
1422
|
+
if (!marker)
|
|
1423
|
+
throw new Error("workspace checkpoint marker missing");
|
|
1424
|
+
await this.sendSessionFrame("session.opened", sessionId, {
|
|
1425
|
+
workspace_ref: session.workspaceRef,
|
|
1426
|
+
native_session_id: session.nativeSessionId,
|
|
1427
|
+
workspace_marker_version: 1,
|
|
1428
|
+
workspace_continuity_state: marker.workspaceContinuityState,
|
|
1429
|
+
workspace_snapshot_id: marker.snapshotId,
|
|
1430
|
+
workspace_revision: marker.revision,
|
|
1431
|
+
workspace_content_sha256: marker.contentSha256,
|
|
1432
|
+
});
|
|
1433
|
+
await this.sendCommandAck(frame, "ok");
|
|
1434
|
+
}
|
|
1435
|
+
catch (error) {
|
|
1436
|
+
this.log.error("workspace baseline checkpoint failed", {
|
|
1437
|
+
sandboxId: this.options.sandboxId,
|
|
1438
|
+
runtimeSessionId: sessionId,
|
|
1439
|
+
error: error instanceof Error ? redactSecretString(error.message) : "unexpected",
|
|
1440
|
+
});
|
|
1441
|
+
const deterministic = (error instanceof WorkspaceSnapshotStagingError && !error.retryable) ||
|
|
1442
|
+
error instanceof WorkspaceEntrySetError ||
|
|
1443
|
+
(error instanceof WorkspaceSnapshotControlError && !error.retryable);
|
|
1444
|
+
const errorCode = deterministic && "code" in error
|
|
1445
|
+
? String(error.code)
|
|
1446
|
+
: "workspace_checkpoint_retryable";
|
|
1447
|
+
await this.sendCommandAck(frame, "rejected", errorCode, !deterministic);
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1100
1450
|
async handleSessionActivate(frame, fence) {
|
|
1101
1451
|
const sessionId = frame.runtime_session_id;
|
|
1102
1452
|
const activationId = frame.activation_id;
|
|
@@ -1283,6 +1633,21 @@ export class AgentServiceSandboxClient {
|
|
|
1283
1633
|
await this.sendCommandAck(frame, "rejected", "activation_mismatch");
|
|
1284
1634
|
return;
|
|
1285
1635
|
}
|
|
1636
|
+
const checkpoint = frame.payload.workspace_checkpoint;
|
|
1637
|
+
if (checkpoint !== undefined) {
|
|
1638
|
+
if (!checkpoint || typeof checkpoint !== "object" || Array.isArray(checkpoint) ||
|
|
1639
|
+
typeof checkpoint.checkpoint_id !== "string" ||
|
|
1640
|
+
checkpoint.policy_id !== "WorkspaceSnapshotPolicyV1" ||
|
|
1641
|
+
!Number.isSafeInteger(checkpoint.base_revision) ||
|
|
1642
|
+
Number(checkpoint.base_revision) < 0) {
|
|
1643
|
+
await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
|
|
1644
|
+
return;
|
|
1645
|
+
}
|
|
1646
|
+
this.turnWorkspaceCheckpoints.set(runId, {
|
|
1647
|
+
checkpointId: String(checkpoint.checkpoint_id),
|
|
1648
|
+
baseRevision: Number(checkpoint.base_revision),
|
|
1649
|
+
});
|
|
1650
|
+
}
|
|
1286
1651
|
const payload = frame.payload;
|
|
1287
1652
|
if (!this.validTurnPayload(payload)) {
|
|
1288
1653
|
await this.sendCommandAck(frame, "rejected", "invalid_turn_payload");
|
|
@@ -1369,6 +1734,8 @@ export class AgentServiceSandboxClient {
|
|
|
1369
1734
|
.finally(() => {
|
|
1370
1735
|
this.inflightCommands.delete(commandId);
|
|
1371
1736
|
this.runProfiles.delete(runId);
|
|
1737
|
+
this.turnWorkspaceCheckpoints.delete(runId);
|
|
1738
|
+
this.completedWorkspaceCheckpoints.delete(runId);
|
|
1372
1739
|
});
|
|
1373
1740
|
}
|
|
1374
1741
|
async handleTurnCancel(frame) {
|
|
@@ -1419,7 +1786,7 @@ export class AgentServiceSandboxClient {
|
|
|
1419
1786
|
}
|
|
1420
1787
|
delete session.acceptedCommands[commandId];
|
|
1421
1788
|
this.persist();
|
|
1422
|
-
this.finishTurn({ agent_run_id: runId });
|
|
1789
|
+
await this.finishTurn({ agent_run_id: runId });
|
|
1423
1790
|
}
|
|
1424
1791
|
/**
|
|
1425
1792
|
* 幂等关闭一个 runtime session:终止其 runtime 子进程、删除 workspace/native state、
|
|
@@ -1478,7 +1845,7 @@ export class AgentServiceSandboxClient {
|
|
|
1478
1845
|
delete session.acceptedCommands[commandId];
|
|
1479
1846
|
this.persist();
|
|
1480
1847
|
}
|
|
1481
|
-
this.finishTurn({ agent_run_id: accepted.agentRunId });
|
|
1848
|
+
await this.finishTurn({ agent_run_id: accepted.agentRunId });
|
|
1482
1849
|
}
|
|
1483
1850
|
catch (error) {
|
|
1484
1851
|
this.log.error("failed to reconcile an interrupted accepted turn", {
|
|
@@ -1687,7 +2054,7 @@ export class AgentServiceSandboxClient {
|
|
|
1687
2054
|
spool_frames: this.spoolFrameCount(),
|
|
1688
2055
|
});
|
|
1689
2056
|
}
|
|
1690
|
-
async sendCommandAck(frame, status, error) {
|
|
2057
|
+
async sendCommandAck(frame, status, error, retryable) {
|
|
1691
2058
|
if (frame.sandbox_generation !== this.sandboxGeneration ||
|
|
1692
2059
|
frame.connection_epoch !== this.connectionEpoch)
|
|
1693
2060
|
return;
|
|
@@ -1695,6 +2062,7 @@ export class AgentServiceSandboxClient {
|
|
|
1695
2062
|
command_frame_id: frame.frame_id,
|
|
1696
2063
|
status,
|
|
1697
2064
|
...(error !== undefined ? { error } : {}),
|
|
2065
|
+
...(retryable !== undefined ? { retryable } : {}),
|
|
1698
2066
|
});
|
|
1699
2067
|
}
|
|
1700
2068
|
async handleWorkspaceFileRead(frame) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export declare const AGENT_SERVICE_WS_SCHEMA: "botlearn-agent-sandbox-ws/0.
|
|
2
|
-
export declare const AGENT_SERVICE_WS_SUBPROTOCOL: "botlearn-agent-sandbox.
|
|
1
|
+
export declare const AGENT_SERVICE_WS_SCHEMA: "botlearn-agent-sandbox-ws/0.3";
|
|
2
|
+
export declare const AGENT_SERVICE_WS_SUBPROTOCOL: "botlearn-agent-sandbox.v3";
|
|
3
3
|
export declare const WORKSPACE_FILE_READ_BINARY_CAPABILITY: "workspace_file_read_binary_v1";
|
|
4
4
|
export declare const WORKSPACE_FILE_CHUNK_BYTES: number;
|
|
5
5
|
export declare const WORKSPACE_FILE_CHUNK_HEADER_BYTES = 29;
|
|
6
|
-
export type SandboxFrameType = "sandbox.hello" | "sandbox.sync" | "session.open" | "session.activate" | "turn.start" | "turn.cancel" | "session.close" | "sandbox.drain" | "sandbox.shutdown" | "event.ack" | "auth.rotate" | "workspace.file.read" | "ping" | "sandbox.ready" | "sandbox.heartbeat" | "command.ack" | "session.opened" | "session.open_failed" | "session.closed" | "turn.event" | "turn.file.report" | "workspace.file.result" | "sandbox.drained" | "sandbox.log" | "pong" | "protocol.error";
|
|
6
|
+
export type SandboxFrameType = "sandbox.hello" | "sandbox.sync" | "session.open" | "session.activate" | "turn.start" | "turn.cancel" | "session.close" | "sandbox.drain" | "sandbox.shutdown" | "event.ack" | "auth.rotate" | "workspace.file.read" | "workspace.checkpoint" | "ping" | "sandbox.ready" | "sandbox.heartbeat" | "command.ack" | "session.opened" | "session.open_failed" | "session.closed" | "turn.event" | "turn.file.report" | "workspace.file.result" | "sandbox.drained" | "sandbox.log" | "pong" | "protocol.error";
|
|
7
7
|
export declare class UnsupportedSandboxProtocolError extends Error {
|
|
8
8
|
readonly schemaVersion: unknown;
|
|
9
9
|
constructor(schemaVersion: unknown);
|