@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,7 @@
1
- import { chmodSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
1
+ import { randomUUID } from "node:crypto";
2
+ import { chmodSync, closeSync, createWriteStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
3
+ import { Readable } from "node:stream";
4
+ import { pipeline } from "node:stream/promises";
2
5
  import path from "node:path";
3
6
  import { ensureDaemonHome } from "./auth-store.js";
4
7
  import { AGENT_SERVICE_WS_SCHEMA, AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, createWorkspaceFileChunk, parseSandboxFrame, UnsupportedSandboxProtocolError, WORKSPACE_FILE_READ_BINARY_CAPABILITY, } from "./agent-service-ws-protocol.js";
@@ -10,6 +13,14 @@ import { parseRuntimeSkillProviderGrantSet, prepareRuntimeSkillProvider, Runtime
10
13
  import { RunDispatcher, } from "./run-dispatcher.js";
11
14
  import { cleanupRuntimeSessionWorkspaceCopyStaging, copyRuntimeSessionWorkspace, ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, finalizeRuntimeSessionWorkspaceCopy, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, WORKSPACE_COPY_POLICY_V1, WorkspaceCopyError, workspaceCopyV1Supported, } from "./workspace.js";
12
15
  import { readWorkspaceFile, WorkspaceFileReadError } from "./workspace-file-read.js";
16
+ import { checkpointWorkspace, WorkspaceSnapshotControlError, } from "./workspace-snapshot-control.js";
17
+ import { assertWorkspaceWriterFreezeSupported, freezeRuntimeWriters, issueWorkspaceWriterFreezeProof, quiesceRuntimeWriters, thawRuntimeWriters, } from "./runtime-quiescence.js";
18
+ import { WorkspaceSnapshotStagingError } from "./workspace-snapshot-staging.js";
19
+ import { validateWorkspaceEntrySet, WorkspaceEntrySetError, } from "./workspace-entry-set.js";
20
+ import { WORKSPACE_SNAPSHOT_POLICY_V1, workspaceSnapshotPolicyCapability, } from "./workspace-snapshot-policy.js";
21
+ import { assertWorkspaceRestoreCapacity, restoreWorkspaceSnapshot, WorkspaceRestoreError, } from "./workspace-restore.js";
22
+ import { readWorkspaceMaterializationMarker } from "./workspace-materialization.js";
23
+ import { applyRuntimeWorkspaceQuota, assertRuntimeWorkspaceQuota } from "./workspace-quota.js";
13
24
  import { WebSocketClient, } from "./websocket-client.js";
14
25
  const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
15
26
  const MAX_SPOOL_FRAMES = 1024;
@@ -19,6 +30,14 @@ const LEGACY_EVENT_ACK_WINDOW = 1;
19
30
  const INPUT_ATTACHMENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
20
31
  /** 出站 seq 基址:seq = connection_epoch * SEQ_EPOCH_BASE + n,跨重连单调(合同 §1.1)。 */
21
32
  const SEQ_EPOCH_BASE = 1_000_000_000;
33
+ function agentServiceHttpBase(wsUrl) {
34
+ const url = new URL(wsUrl);
35
+ url.protocol = url.protocol === "wss:" ? "https:" : "http:";
36
+ url.pathname = "/internal/v1";
37
+ url.search = "";
38
+ url.hash = "";
39
+ return url.toString().replace(/\/$/, "");
40
+ }
22
41
  const RUNTIME_LOG_WINDOW_MS = 60_000;
23
42
  const WORKSPACE_FILE_TRANSFER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
24
43
  const DISABLED_RUNTIME_LOG_POLICY = {
@@ -346,6 +365,7 @@ export class AgentServiceSandboxClient {
346
365
  sessionProfiles = new Map();
347
366
  /** activation-scoped 模型短凭据;只保存在内存中,绝不进入 state.json。 */
348
367
  activationContexts = new Map();
368
+ activationCleanupTasks = new Map();
349
369
  pendingAcks = new Map();
350
370
  pendingFileReports = new Map();
351
371
  inflightCommands = new Set();
@@ -438,7 +458,7 @@ export class AgentServiceSandboxClient {
438
458
  session.nativeSessionId = nativeSessionId.trim() || null;
439
459
  this.persist();
440
460
  }
441
- finishTurn(payload) {
461
+ async finishTurn(payload) {
442
462
  const scope = this.turnScopes.get(payload.agent_run_id);
443
463
  if (!scope)
444
464
  return;
@@ -454,7 +474,7 @@ export class AgentServiceSandboxClient {
454
474
  };
455
475
  this.persist();
456
476
  try {
457
- this.completePendingActivationCleanup(scope.sessionId);
477
+ await this.completePendingActivationCleanup(scope.sessionId);
458
478
  }
459
479
  catch (error) {
460
480
  this.log.error("failed to revoke a terminal runtime session workspace", {
@@ -472,19 +492,31 @@ export class AgentServiceSandboxClient {
472
492
  this.currentTurnSessionId = null;
473
493
  this.persist();
474
494
  }
475
- completePendingActivationCleanup(sessionId) {
495
+ async completePendingActivationCleanup(sessionId) {
496
+ const existing = this.activationCleanupTasks.get(sessionId);
497
+ if (existing)
498
+ return await existing;
499
+ const task = this.runPendingActivationCleanup(sessionId);
500
+ this.activationCleanupTasks.set(sessionId, task);
501
+ try {
502
+ await task;
503
+ }
504
+ finally {
505
+ if (this.activationCleanupTasks.get(sessionId) === task) {
506
+ this.activationCleanupTasks.delete(sessionId);
507
+ }
508
+ }
509
+ }
510
+ async runPendingActivationCleanup(sessionId) {
476
511
  const session = this.state.sessions[sessionId];
477
512
  const pending = session?.pendingActivationCleanup;
478
513
  if (!session || !pending)
479
514
  return;
480
- revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
481
515
  const activation = this.activationContexts.get(sessionId);
482
516
  if (activation?.activationId === pending.activationId) {
483
517
  this.activationContexts.delete(sessionId);
484
518
  this.sessionProfiles.delete(sessionId);
485
519
  }
486
- if (session.activationId === pending.activationId)
487
- session.activationId = null;
488
520
  if (!session.completedActivations.includes(pending.activationId)) {
489
521
  // Session-close/generation-rotate are the retention bounds. Never evict an
490
522
  // activation tombstone while the session is alive: activation IDs are opaque, so
@@ -499,15 +531,13 @@ export class AgentServiceSandboxClient {
499
531
  delete session.acceptedCommands[commandId];
500
532
  session.pendingActivationCleanup = null;
501
533
  this.turnScopes.delete(pending.agentRunId);
502
- if (this.activeSessionId === sessionId)
503
- this.activeSessionId = null;
504
534
  if (this.currentTurnSessionId === sessionId)
505
535
  this.currentTurnSessionId = null;
506
536
  this.persist();
507
537
  }
508
- recoverPendingActivationCleanups() {
538
+ async recoverPendingActivationCleanups() {
509
539
  for (const sessionId of Object.keys(this.state.sessions)) {
510
- this.completePendingActivationCleanup(sessionId);
540
+ await this.completePendingActivationCleanup(sessionId);
511
541
  }
512
542
  }
513
543
  async run() {
@@ -568,6 +598,11 @@ export class AgentServiceSandboxClient {
568
598
  this.removeOperationalLogSink();
569
599
  this.socket?.close(1000, "daemon_stopping");
570
600
  }
601
+ async stopGracefully() {
602
+ this.dispatcher.cancelAll();
603
+ await quiesceRuntimeWriters(null);
604
+ this.stop();
605
+ }
571
606
  async postEvent(agentRunId, event) {
572
607
  const scope = this.turnScopes.get(agentRunId);
573
608
  if (!scope)
@@ -578,6 +613,8 @@ export class AgentServiceSandboxClient {
578
613
  if (!this.sandboxGeneration || !this.connectionEpoch) {
579
614
  throw new Error("Agent Service sandbox is not authenticated");
580
615
  }
616
+ // Terminal truth is never delayed by object storage. Durable workspace checkpointing
617
+ // is a separate lifecycle command issued only for pause/cleanup/rotation.
581
618
  if (event.type === "run.block") {
582
619
  await this.waitForEventCapacity();
583
620
  }
@@ -782,6 +819,7 @@ export class AgentServiceSandboxClient {
782
819
  if (frame.sandbox_generation !== this.state.sandboxGeneration) {
783
820
  // Generation change wipes every session shard (contract §1.4).
784
821
  this.dispatcher.cancelAll();
822
+ await quiesceRuntimeWriters(null);
785
823
  this.rejectPending(new Error("Agent Service sandbox generation changed"));
786
824
  for (const sessionId of Object.keys(this.state.sessions)) {
787
825
  revokeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
@@ -800,7 +838,7 @@ export class AgentServiceSandboxClient {
800
838
  this.connectionEpoch = frame.connection_epoch;
801
839
  this.inboundSeq = frame.seq;
802
840
  this.outboundSeq = frame.connection_epoch * SEQ_EPOCH_BASE;
803
- this.recoverPendingActivationCleanups();
841
+ await this.recoverPendingActivationCleanups();
804
842
  const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
805
843
  this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
806
844
  const staleSeconds = Number(frame.payload.stale_after_seconds ?? 45);
@@ -815,12 +853,18 @@ export class AgentServiceSandboxClient {
815
853
  this.runtimeLogPolicy = runtimeLogPolicy(frame.payload.runtime_log_policy);
816
854
  await cleanupRuntimeSessionWorkspaceCopyStaging();
817
855
  this.persist();
856
+ assertRuntimeWorkspaceQuota(process.env.BOTLEARN_AGENT_SERVICE_WORKSPACE_ROOT ?? "");
857
+ assertWorkspaceWriterFreezeSupported();
818
858
  await this.sendControlFrame("sandbox.ready", {
819
859
  protocol_versions: [AGENT_SERVICE_WS_SCHEMA],
820
860
  capabilities: [
821
861
  WORKSPACE_FILE_READ_BINARY_CAPABILITY,
862
+ "workspace_writer_freeze_v1",
863
+ "workspace_snapshot_v2",
864
+ "workspace_restore_v1",
822
865
  ...(workspaceCopyV1Supported() ? ["workspace_copy_v1"] : []),
823
866
  ],
867
+ workspace_snapshot_policy: workspaceSnapshotPolicyCapability(),
824
868
  daemon_version: this.options.daemonVersion,
825
869
  runtime_versions: {
826
870
  course_daemon: this.options.daemonVersion,
@@ -952,8 +996,15 @@ export class AgentServiceSandboxClient {
952
996
  });
953
997
  });
954
998
  return;
999
+ case "workspace.checkpoint":
1000
+ this.scheduleLifecycle("workspace.checkpoint", () => this.handleWorkspaceCheckpoint(frame));
1001
+ return;
1002
+ case "workspace.thaw":
1003
+ await thawRuntimeWriters();
1004
+ await this.sendCommandAck(frame, "ok");
1005
+ return;
955
1006
  case "sandbox.shutdown":
956
- this.stop();
1007
+ await this.stopGracefully();
957
1008
  return;
958
1009
  case "sandbox.drain":
959
1010
  await this.sendCommandAck(frame, "ok");
@@ -979,8 +1030,11 @@ export class AgentServiceSandboxClient {
979
1030
  this.scheduleLifecycle("sandbox.sync.close", (fence) => this.closeSession(sessionId, "sync_close", frame, fence));
980
1031
  }
981
1032
  const state = frame.payload.state;
1033
+ if (frame.payload.thaw_runtime_writers === true) {
1034
+ await thawRuntimeWriters();
1035
+ }
982
1036
  if (state === "shutdown") {
983
- this.stop();
1037
+ await this.stopGracefully();
984
1038
  return;
985
1039
  }
986
1040
  if (state === "drain") {
@@ -998,6 +1052,7 @@ export class AgentServiceSandboxClient {
998
1052
  const courseRunId = frame.payload.course_run_id;
999
1053
  const requestedWorkspaceRef = frame.payload.workspace_ref;
1000
1054
  const requestedCopy = frame.payload.workspace_copy;
1055
+ const requestedRestore = frame.payload.workspace_restore;
1001
1056
  const requestedCopyRecord = requestedCopy && typeof requestedCopy === "object"
1002
1057
  ? requestedCopy
1003
1058
  : undefined;
@@ -1018,7 +1073,11 @@ export class AgentServiceSandboxClient {
1018
1073
  typeof copyId !== "string" ||
1019
1074
  copyId.length < 1 ||
1020
1075
  typeof requestedCopySource !== "string" ||
1021
- requestedCopySource.length < 1))) {
1076
+ requestedCopySource.length < 1))
1077
+ || (requestedRestore !== null && requestedRestore !== undefined &&
1078
+ (typeof requestedRestore !== "object" || Array.isArray(requestedRestore)))
1079
+ || (requestedCopy !== null && requestedCopy !== undefined &&
1080
+ requestedRestore !== null && requestedRestore !== undefined)) {
1022
1081
  throw new Error("invalid session.open payload");
1023
1082
  }
1024
1083
  if (session.courseRunId !== null && session.courseRunId !== courseRunId) {
@@ -1032,6 +1091,165 @@ export class AgentServiceSandboxClient {
1032
1091
  const isNewSession = session.courseRunId === null;
1033
1092
  let markerPath = null;
1034
1093
  try {
1094
+ if (requestedRestore && typeof requestedRestore === "object") {
1095
+ const raw = requestedRestore;
1096
+ const descriptorRaw = raw.descriptor;
1097
+ const controlFence = raw.control_fence;
1098
+ let restoreControlToken = raw.control_token;
1099
+ if (!controlFence || typeof controlFence !== "object" || Array.isArray(controlFence)) {
1100
+ throw new Error("invalid workspace restore control fence");
1101
+ }
1102
+ if (typeof restoreControlToken !== "string" || restoreControlToken.length < 32) {
1103
+ throw new Error("invalid workspace restore control token");
1104
+ }
1105
+ if (!descriptorRaw || !Array.isArray(raw.entries)) {
1106
+ throw new Error("invalid workspace restore payload");
1107
+ }
1108
+ const descriptor = {
1109
+ sandboxId: this.options.sandboxId,
1110
+ sandboxGeneration: this.sandboxGeneration,
1111
+ runtimeSessionId: sessionId,
1112
+ continuityState: descriptorRaw.continuity_state,
1113
+ continuityErrorCode: descriptorRaw.continuity_error_code,
1114
+ snapshotId: descriptorRaw.snapshot_id,
1115
+ revision: Number(descriptorRaw.revision),
1116
+ contentSha256: descriptorRaw.content_sha256,
1117
+ expectedFileCount: Number(descriptorRaw.expected_file_count),
1118
+ expectedEntryCount: Number(descriptorRaw.expected_entry_count),
1119
+ expectedTotalBytes: Number(descriptorRaw.expected_total_bytes),
1120
+ };
1121
+ const workspaceDirectory = runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration);
1122
+ const requiredFreeBytes = Math.max(WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes, descriptor.expectedTotalBytes + WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes);
1123
+ // Fail before issuing or consuming any object grant. The download staging is a
1124
+ // sibling on this same bounded backing filesystem and is renamed, not copied,
1125
+ // into the atomic restore staging tree.
1126
+ assertWorkspaceRestoreCapacity({
1127
+ workspaceDirectory,
1128
+ descriptor,
1129
+ requiredFreeBytes,
1130
+ entryCopies: 2,
1131
+ });
1132
+ const entries = [];
1133
+ const predownloadRoot = path.join(path.dirname(workspaceDirectory), `.botlearn-predownload-${sessionId}-${randomUUID()}`);
1134
+ mkdirSync(predownloadRoot, { recursive: true, mode: 0o700 });
1135
+ try {
1136
+ let nextCursor = null;
1137
+ do {
1138
+ if (nextCursor !== null && nextCursor.length > 2048) {
1139
+ throw new Error("invalid workspace restore cursor");
1140
+ }
1141
+ const pageSignal = AbortSignal.timeout(60_000);
1142
+ const response = await fetch(`${agentServiceHttpBase(this.options.wsUrl)}/runtime-sessions/${sessionId}/restore-grants`, {
1143
+ method: "POST",
1144
+ headers: {
1145
+ Authorization: `Bearer ${restoreControlToken}`,
1146
+ "Content-Type": "application/json",
1147
+ },
1148
+ body: JSON.stringify({
1149
+ policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
1150
+ fence: controlFence,
1151
+ cursor: nextCursor,
1152
+ // Fetch one canonical entry at a time so a grant is consumed as soon as
1153
+ // it is signed, including deployments using the minimum 60-second TTL.
1154
+ max_entries: 1,
1155
+ }),
1156
+ signal: pageSignal,
1157
+ });
1158
+ if (!response.ok) {
1159
+ let code = "workspace_restore_unavailable";
1160
+ let retryable = response.status >= 500 || response.status === 408 ||
1161
+ response.status === 429;
1162
+ try {
1163
+ const rejected = await response.json();
1164
+ const detail = rejected.detail;
1165
+ if (detail && typeof detail === "object" && !Array.isArray(detail)) {
1166
+ const stableCode = detail.code;
1167
+ if (typeof stableCode === "string" && stableCode.length <= 120) {
1168
+ code = stableCode;
1169
+ }
1170
+ if (typeof detail.retryable === "boolean") {
1171
+ retryable = detail.retryable;
1172
+ }
1173
+ }
1174
+ }
1175
+ catch {
1176
+ // Content-free fallback retains the stable local failure code.
1177
+ }
1178
+ throw new WorkspaceRestoreError(code, retryable);
1179
+ }
1180
+ const page = await response.json();
1181
+ if (!Array.isArray(page.entries))
1182
+ throw new Error("invalid workspace restore page");
1183
+ if (typeof page.control_token !== "string" || page.control_token.length < 32) {
1184
+ throw new Error("invalid refreshed workspace restore control token");
1185
+ }
1186
+ restoreControlToken = page.control_token;
1187
+ const pageFiles = [];
1188
+ for (const value of page.entries) {
1189
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
1190
+ throw new Error("invalid workspace restore entry");
1191
+ }
1192
+ const item = value;
1193
+ if (item.entry_type === "directory") {
1194
+ entries.push({ type: "directory", path: String(item.path), mode: "0700" });
1195
+ continue;
1196
+ }
1197
+ if (item.entry_type !== "file" || !item.grant || typeof item.grant !== "object") {
1198
+ throw new Error("invalid workspace restore file grant");
1199
+ }
1200
+ const entry = {
1201
+ type: "file",
1202
+ path: String(item.path),
1203
+ mode: item.mode,
1204
+ sizeBytes: Number(item.size_bytes),
1205
+ sha256: String(item.sha256),
1206
+ };
1207
+ entries.push(entry);
1208
+ pageFiles.push({ entry, grant: item.grant });
1209
+ }
1210
+ // Validate all paths and duplicates before using any page path locally.
1211
+ validateWorkspaceEntrySet({ schemaVersion: "agent-workspace-entry-set/1", entries }, WORKSPACE_SNAPSHOT_POLICY_V1.limits);
1212
+ for (const { entry, grant } of pageFiles) {
1213
+ if (typeof grant.url !== "string")
1214
+ throw new Error("workspace restore grant missing");
1215
+ const destination = path.join(predownloadRoot, ...entry.path.split("/"));
1216
+ mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
1217
+ const part = `${destination}.part`;
1218
+ const downloadSignal = AbortSignal.timeout(60_000);
1219
+ const download = await fetch(grant.url, {
1220
+ method: "GET",
1221
+ headers: grant.headers,
1222
+ signal: downloadSignal,
1223
+ });
1224
+ if (!download.ok)
1225
+ throw new Error(`workspace_restore_download_${download.status}`);
1226
+ if (!download.body)
1227
+ throw new Error("workspace_restore_download_body_missing");
1228
+ await pipeline(Readable.fromWeb(download.body), createWriteStream(part, { mode: 0o600 }), { signal: downloadSignal });
1229
+ renameSync(part, destination);
1230
+ }
1231
+ const rawNext = page.next_cursor;
1232
+ if (rawNext !== null && rawNext !== undefined && typeof rawNext !== "string") {
1233
+ throw new Error("invalid workspace restore cursor");
1234
+ }
1235
+ nextCursor = rawNext ?? null;
1236
+ } while (nextCursor !== null);
1237
+ await restoreWorkspaceSnapshot({
1238
+ workspaceDirectory,
1239
+ controlDirectory: path.join(ensureDaemonHome(), "agent-service-sandboxes", this.options.sandboxId, "workspace-control"),
1240
+ descriptor,
1241
+ entrySet: { schemaVersion: "agent-workspace-entry-set/1", entries },
1242
+ limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
1243
+ requiredFreeBytes,
1244
+ downloadFile: async (entry, partPath) => {
1245
+ renameSync(path.join(predownloadRoot, ...entry.path.split("/")), partPath);
1246
+ },
1247
+ });
1248
+ }
1249
+ finally {
1250
+ rmSync(predownloadRoot, { recursive: true, force: true });
1251
+ }
1252
+ }
1035
1253
  if (typeof requestedCopySource === "string" && typeof copyId === "string") {
1036
1254
  if (session.workspaceCopyReceipt !== null) {
1037
1255
  const receipt = session.workspaceCopyReceipt;
@@ -1075,28 +1293,159 @@ export class AgentServiceSandboxClient {
1075
1293
  });
1076
1294
  return;
1077
1295
  }
1296
+ if (requestedRestore) {
1297
+ const restoreError = error instanceof WorkspaceRestoreError ? error : null;
1298
+ await this.sendSessionFrame("session.open_failed", sessionId, {
1299
+ error_code: restoreError?.code ?? "workspace_restore_unavailable",
1300
+ retryable: restoreError?.retryable ?? true,
1301
+ });
1302
+ return;
1303
+ }
1078
1304
  throw error;
1079
1305
  }
1080
1306
  session.courseRunId = courseRunId;
1081
1307
  session.workspaceRef = session.workspaceRef ?? (typeof requestedWorkspaceRef === "string"
1082
1308
  ? requestedWorkspaceRef
1083
1309
  : `ws_${sessionId.replaceAll("-", "")}_g${this.sandboxGeneration}`);
1084
- if (session.workspaceCopyReceipt === null) {
1310
+ // Replaying session.open is part of reconnect reconciliation. An already-active
1311
+ // runtime may still be creating files in this workspace, so never transiently revoke
1312
+ // its group write/execute access before the following session.activate arrives.
1313
+ if (session.workspaceCopyReceipt === null && (isNewSession || session.activationId === null)) {
1085
1314
  ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
1086
1315
  }
1087
- this.state.sessions[sessionId] = session;
1088
- this.persist();
1089
1316
  if (markerPath !== null) {
1090
1317
  await finalizeRuntimeSessionWorkspaceCopy(markerPath);
1091
1318
  }
1319
+ try {
1320
+ await applyRuntimeWorkspaceQuota(sessionId, this.sandboxGeneration);
1321
+ }
1322
+ catch (error) {
1323
+ if (requestedRestore) {
1324
+ await this.sendSessionFrame("session.open_failed", sessionId, {
1325
+ error_code: "workspace_restore_unavailable",
1326
+ });
1327
+ return;
1328
+ }
1329
+ if (requestedCopy) {
1330
+ await this.sendSessionFrame("session.open_failed", sessionId, {
1331
+ error_code: "workspace_migration_io_failed",
1332
+ });
1333
+ return;
1334
+ }
1335
+ throw error;
1336
+ }
1337
+ this.state.sessions[sessionId] = session;
1338
+ this.persist();
1339
+ const materialization = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
1092
1340
  await this.sendSessionFrame("session.opened", sessionId, {
1093
1341
  workspace_ref: session.workspaceRef,
1094
1342
  native_session_id: session.nativeSessionId,
1343
+ ...(materialization
1344
+ ? {
1345
+ workspace_marker_version: 1,
1346
+ workspace_continuity_state: materialization.workspaceContinuityState,
1347
+ workspace_snapshot_id: materialization.snapshotId,
1348
+ workspace_revision: materialization.revision,
1349
+ workspace_content_sha256: materialization.contentSha256,
1350
+ }
1351
+ : {}),
1095
1352
  ...(session.workspaceCopyReceipt !== null
1096
1353
  ? { workspace_copy_receipt: session.workspaceCopyReceipt }
1097
1354
  : {}),
1098
1355
  });
1099
1356
  }
1357
+ async handleWorkspaceCheckpoint(frame) {
1358
+ const sessionId = frame.runtime_session_id;
1359
+ const session = this.state.sessions[sessionId];
1360
+ const checkpointId = frame.payload.checkpoint_id;
1361
+ const controlToken = frame.payload.control_token;
1362
+ const baseRevision = Number(frame.payload.base_revision);
1363
+ const sourceKind = frame.payload.source_kind;
1364
+ if (!session ||
1365
+ this.currentTurnSessionId === sessionId ||
1366
+ typeof checkpointId !== "string" ||
1367
+ typeof controlToken !== "string" ||
1368
+ controlToken.length < 32 ||
1369
+ frame.payload.policy_id !== "WorkspaceSnapshotPolicyV1" ||
1370
+ ![
1371
+ "sandbox_pause",
1372
+ "sandbox_cleanup",
1373
+ "generation_rotation",
1374
+ "initial_migration",
1375
+ "live_workspace_copy",
1376
+ "maintenance_repair",
1377
+ ].includes(String(sourceKind)) ||
1378
+ !Number.isSafeInteger(baseRevision) ||
1379
+ baseRevision < 0 ||
1380
+ frame.payload.agent_run_id != null ||
1381
+ frame.payload.worker_attempt != null ||
1382
+ frame.payload.activation_id != null) {
1383
+ await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
1384
+ return;
1385
+ }
1386
+ const lifecycleCheckpoint = [
1387
+ "sandbox_pause",
1388
+ "sandbox_cleanup",
1389
+ "generation_rotation",
1390
+ ].includes(String(sourceKind));
1391
+ let checkpointCommitted = false;
1392
+ try {
1393
+ await freezeRuntimeWriters();
1394
+ const freezeProof = issueWorkspaceWriterFreezeProof({
1395
+ sandboxId: this.options.sandboxId,
1396
+ sandboxGeneration: this.sandboxGeneration,
1397
+ runtimeSessionId: sessionId,
1398
+ checkpointId,
1399
+ });
1400
+ await checkpointWorkspace({
1401
+ wsUrl: this.options.wsUrl,
1402
+ reconnectToken: controlToken,
1403
+ workspaceDirectory: runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration),
1404
+ scope: {
1405
+ checkpointId,
1406
+ baseRevision,
1407
+ sandboxId: this.options.sandboxId,
1408
+ sandboxGeneration: this.sandboxGeneration,
1409
+ connectionEpoch: this.connectionEpoch,
1410
+ runtimeSessionId: sessionId,
1411
+ },
1412
+ freezeProof,
1413
+ });
1414
+ checkpointCommitted = true;
1415
+ const marker = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
1416
+ if (!marker)
1417
+ throw new Error("workspace checkpoint marker missing");
1418
+ await this.sendSessionFrame("session.opened", sessionId, {
1419
+ workspace_ref: session.workspaceRef,
1420
+ native_session_id: session.nativeSessionId,
1421
+ workspace_marker_version: 1,
1422
+ workspace_continuity_state: marker.workspaceContinuityState,
1423
+ workspace_snapshot_id: marker.snapshotId,
1424
+ workspace_revision: marker.revision,
1425
+ workspace_content_sha256: marker.contentSha256,
1426
+ });
1427
+ await this.sendCommandAck(frame, "ok");
1428
+ if (!lifecycleCheckpoint)
1429
+ await thawRuntimeWriters();
1430
+ }
1431
+ catch (error) {
1432
+ if (!lifecycleCheckpoint || !checkpointCommitted) {
1433
+ await thawRuntimeWriters().catch(() => undefined);
1434
+ }
1435
+ this.log.error("workspace baseline checkpoint failed", {
1436
+ sandboxId: this.options.sandboxId,
1437
+ runtimeSessionId: sessionId,
1438
+ error: error instanceof Error ? redactSecretString(error.message) : "unexpected",
1439
+ });
1440
+ const deterministic = (error instanceof WorkspaceSnapshotStagingError && !error.retryable) ||
1441
+ error instanceof WorkspaceEntrySetError ||
1442
+ (error instanceof WorkspaceSnapshotControlError && !error.retryable);
1443
+ const errorCode = deterministic && "code" in error
1444
+ ? String(error.code)
1445
+ : "workspace_checkpoint_retryable";
1446
+ await this.sendCommandAck(frame, "rejected", errorCode, !deterministic);
1447
+ }
1448
+ }
1100
1449
  async handleSessionActivate(frame, fence) {
1101
1450
  const sessionId = frame.runtime_session_id;
1102
1451
  const activationId = frame.activation_id;
@@ -1233,7 +1582,7 @@ export class AgentServiceSandboxClient {
1233
1582
  const previousSessionId = this.activeSessionId;
1234
1583
  const changesActivation = previousSessionId !== null && (previousSessionId !== sessionId || session.activationId !== activationId);
1235
1584
  if (changesActivation) {
1236
- const stopped = await this.deactivateSession(previousSessionId);
1585
+ const stopped = await this.deactivateSession(previousSessionId, previousSessionId !== sessionId);
1237
1586
  if (fence && !this.isCurrentLifecycleFence(fence))
1238
1587
  return;
1239
1588
  if (!stopped) {
@@ -1419,7 +1768,7 @@ export class AgentServiceSandboxClient {
1419
1768
  }
1420
1769
  delete session.acceptedCommands[commandId];
1421
1770
  this.persist();
1422
- this.finishTurn({ agent_run_id: runId });
1771
+ await this.finishTurn({ agent_run_id: runId });
1423
1772
  }
1424
1773
  /**
1425
1774
  * 幂等关闭一个 runtime session:终止其 runtime 子进程、删除 workspace/native state、
@@ -1429,15 +1778,16 @@ export class AgentServiceSandboxClient {
1429
1778
  const session = this.state.sessions[sessionId];
1430
1779
  if (session) {
1431
1780
  const runIds = this.runIdsForSession(sessionId);
1432
- session.activationId = null;
1433
1781
  this.activationContexts.delete(sessionId);
1434
1782
  this.sessionProfiles.delete(sessionId);
1435
- revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1436
1783
  for (const runId of runIds)
1437
1784
  this.dispatcher.cancel(runId);
1438
1785
  if (!await this.dispatcher.waitForRuns(runIds, 10_000)) {
1439
1786
  throw new Error(`runtime session ${sessionId} did not stop before close`);
1440
1787
  }
1788
+ await this.terminateSessionWriters(session);
1789
+ session.activationId = null;
1790
+ revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1441
1791
  if (fence && !this.isCurrentLifecycleFence(fence))
1442
1792
  return;
1443
1793
  if (this.activeSessionId === sessionId)
@@ -1478,7 +1828,7 @@ export class AgentServiceSandboxClient {
1478
1828
  delete session.acceptedCommands[commandId];
1479
1829
  this.persist();
1480
1830
  }
1481
- this.finishTurn({ agent_run_id: accepted.agentRunId });
1831
+ await this.finishTurn({ agent_run_id: accepted.agentRunId });
1482
1832
  }
1483
1833
  catch (error) {
1484
1834
  this.log.error("failed to reconcile an interrupted accepted turn", {
@@ -1580,21 +1930,40 @@ export class AgentServiceSandboxClient {
1580
1930
  .filter(([, scope]) => scope.sessionId === sessionId)
1581
1931
  .map(([runId]) => runId);
1582
1932
  }
1583
- async deactivateSession(sessionId) {
1933
+ async deactivateSession(sessionId, terminateWriters = false) {
1584
1934
  const session = this.state.sessions[sessionId];
1585
1935
  if (!session)
1586
1936
  return true;
1587
1937
  const runIds = this.runIdsForSession(sessionId);
1588
- session.activationId = null;
1589
1938
  this.activationContexts.delete(sessionId);
1590
1939
  this.sessionProfiles.delete(sessionId);
1591
- revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1940
+ if (terminateWriters) {
1941
+ revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1942
+ }
1592
1943
  if (this.activeSessionId === sessionId)
1593
1944
  this.activeSessionId = null;
1594
1945
  for (const runId of runIds)
1595
1946
  this.dispatcher.cancel(runId);
1596
1947
  this.persist();
1597
- return this.dispatcher.waitForRuns(runIds, 10_000);
1948
+ const stopped = await this.dispatcher.waitForRuns(runIds, 10_000);
1949
+ if (stopped && terminateWriters) {
1950
+ await this.terminateSessionWriters(session);
1951
+ }
1952
+ session.activationId = null;
1953
+ this.persist();
1954
+ return stopped;
1955
+ }
1956
+ async terminateSessionWriters(session) {
1957
+ const activationIds = new Set([
1958
+ ...session.completedActivations,
1959
+ ...(session.activationId ? [session.activationId] : []),
1960
+ ...(session.pendingActivationCleanup?.activationId
1961
+ ? [session.pendingActivationCleanup.activationId]
1962
+ : []),
1963
+ ]);
1964
+ for (const activationId of activationIds) {
1965
+ await quiesceRuntimeWriters(activationId);
1966
+ }
1598
1967
  }
1599
1968
  sameTurnScope(left, right) {
1600
1969
  return left.sessionId === right.sessionId &&
@@ -1687,7 +2056,7 @@ export class AgentServiceSandboxClient {
1687
2056
  spool_frames: this.spoolFrameCount(),
1688
2057
  });
1689
2058
  }
1690
- async sendCommandAck(frame, status, error) {
2059
+ async sendCommandAck(frame, status, error, retryable) {
1691
2060
  if (frame.sandbox_generation !== this.sandboxGeneration ||
1692
2061
  frame.connection_epoch !== this.connectionEpoch)
1693
2062
  return;
@@ -1695,6 +2064,7 @@ export class AgentServiceSandboxClient {
1695
2064
  command_frame_id: frame.frame_id,
1696
2065
  status,
1697
2066
  ...(error !== undefined ? { error } : {}),
2067
+ ...(retryable !== undefined ? { retryable } : {}),
1698
2068
  });
1699
2069
  }
1700
2070
  async handleWorkspaceFileRead(frame) {
@@ -1,9 +1,9 @@
1
- export declare const AGENT_SERVICE_WS_SCHEMA: "botlearn-agent-sandbox-ws/0.2";
2
- export declare const AGENT_SERVICE_WS_SUBPROTOCOL: "botlearn-agent-sandbox.v2";
1
+ export declare const AGENT_SERVICE_WS_SCHEMA: "botlearn-agent-sandbox-ws/0.4";
2
+ export declare const AGENT_SERVICE_WS_SUBPROTOCOL: "botlearn-agent-sandbox.v4";
3
3
  export declare const WORKSPACE_FILE_READ_BINARY_CAPABILITY: "workspace_file_read_binary_v1";
4
4
  export declare const WORKSPACE_FILE_CHUNK_BYTES: number;
5
5
  export declare const WORKSPACE_FILE_CHUNK_HEADER_BYTES = 29;
6
- export type SandboxFrameType = "sandbox.hello" | "sandbox.sync" | "session.open" | "session.activate" | "turn.start" | "turn.cancel" | "session.close" | "sandbox.drain" | "sandbox.shutdown" | "event.ack" | "auth.rotate" | "workspace.file.read" | "ping" | "sandbox.ready" | "sandbox.heartbeat" | "command.ack" | "session.opened" | "session.open_failed" | "session.closed" | "turn.event" | "turn.file.report" | "workspace.file.result" | "sandbox.drained" | "sandbox.log" | "pong" | "protocol.error";
6
+ export type SandboxFrameType = "sandbox.hello" | "sandbox.sync" | "session.open" | "session.activate" | "turn.start" | "turn.cancel" | "session.close" | "sandbox.drain" | "sandbox.shutdown" | "event.ack" | "auth.rotate" | "workspace.file.read" | "workspace.checkpoint" | "workspace.thaw" | "ping" | "sandbox.ready" | "sandbox.heartbeat" | "command.ack" | "session.opened" | "session.open_failed" | "session.closed" | "turn.event" | "turn.file.report" | "workspace.file.result" | "sandbox.drained" | "sandbox.log" | "pong" | "protocol.error";
7
7
  export declare class UnsupportedSandboxProtocolError extends Error {
8
8
  readonly schemaVersion: unknown;
9
9
  constructor(schemaVersion: unknown);