@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,7 @@
1
- import { chmodSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
1
+ import { randomUUID } from "node:crypto";
2
+ import { chmodSync, closeSync, createWriteStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
3
+ import { Readable } from "node:stream";
4
+ import { pipeline } from "node:stream/promises";
2
5
  import path from "node:path";
3
6
  import { ensureDaemonHome } from "./auth-store.js";
4
7
  import { AGENT_SERVICE_WS_SCHEMA, AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, createWorkspaceFileChunk, parseSandboxFrame, UnsupportedSandboxProtocolError, WORKSPACE_FILE_READ_BINARY_CAPABILITY, } from "./agent-service-ws-protocol.js";
@@ -10,6 +13,15 @@ import { parseRuntimeSkillProviderGrantSet, prepareRuntimeSkillProvider, Runtime
10
13
  import { RunDispatcher, } from "./run-dispatcher.js";
11
14
  import { cleanupRuntimeSessionWorkspaceCopyStaging, copyRuntimeSessionWorkspace, ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, finalizeRuntimeSessionWorkspaceCopy, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, WORKSPACE_COPY_POLICY_V1, WorkspaceCopyError, workspaceCopyV1Supported, } from "./workspace.js";
12
15
  import { readWorkspaceFile, WorkspaceFileReadError } from "./workspace-file-read.js";
16
+ import { checkpointWorkspace, WorkspaceSnapshotControlError, } from "./workspace-snapshot-control.js";
17
+ import { assertWorkspaceWriterFreezeSupported, freezeRuntimeWriters, issueWorkspaceWriterFreezeProof, quiesceRuntimeWriters, thawRuntimeWriters, } from "./runtime-quiescence.js";
18
+ import { WorkspaceSnapshotStagingError } from "./workspace-snapshot-staging.js";
19
+ import { validateWorkspaceEntrySet, WorkspaceEntrySetError, } from "./workspace-entry-set.js";
20
+ import { WORKSPACE_SNAPSHOT_POLICY_V1, workspaceSnapshotPolicyCapability, } from "./workspace-snapshot-policy.js";
21
+ import { assertWorkspaceRestoreCapacity, restoreWorkspaceSnapshot, WorkspaceRestoreError, } from "./workspace-restore.js";
22
+ import { readWorkspaceMaterializationMarker } from "./workspace-materialization.js";
23
+ import { assertWorkspaceFilesystem, flushWorkspaceFilesystem, nasWorkspaceEnabled, } from "./workspace-filesystem.js";
24
+ import { applyRuntimeWorkspaceQuota } from "./workspace-quota.js";
13
25
  import { WebSocketClient, } from "./websocket-client.js";
14
26
  const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
15
27
  const MAX_SPOOL_FRAMES = 1024;
@@ -19,6 +31,14 @@ const LEGACY_EVENT_ACK_WINDOW = 1;
19
31
  const INPUT_ATTACHMENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
20
32
  /** 出站 seq 基址:seq = connection_epoch * SEQ_EPOCH_BASE + n,跨重连单调(合同 §1.1)。 */
21
33
  const SEQ_EPOCH_BASE = 1_000_000_000;
34
+ function agentServiceHttpBase(wsUrl) {
35
+ const url = new URL(wsUrl);
36
+ url.protocol = url.protocol === "wss:" ? "https:" : "http:";
37
+ url.pathname = "/internal/v1";
38
+ url.search = "";
39
+ url.hash = "";
40
+ return url.toString().replace(/\/$/, "");
41
+ }
22
42
  const RUNTIME_LOG_WINDOW_MS = 60_000;
23
43
  const WORKSPACE_FILE_TRANSFER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
24
44
  const DISABLED_RUNTIME_LOG_POLICY = {
@@ -346,6 +366,7 @@ export class AgentServiceSandboxClient {
346
366
  sessionProfiles = new Map();
347
367
  /** activation-scoped 模型短凭据;只保存在内存中,绝不进入 state.json。 */
348
368
  activationContexts = new Map();
369
+ activationCleanupTasks = new Map();
349
370
  pendingAcks = new Map();
350
371
  pendingFileReports = new Map();
351
372
  inflightCommands = new Set();
@@ -438,7 +459,7 @@ export class AgentServiceSandboxClient {
438
459
  session.nativeSessionId = nativeSessionId.trim() || null;
439
460
  this.persist();
440
461
  }
441
- finishTurn(payload) {
462
+ async finishTurn(payload) {
442
463
  const scope = this.turnScopes.get(payload.agent_run_id);
443
464
  if (!scope)
444
465
  return;
@@ -454,7 +475,7 @@ export class AgentServiceSandboxClient {
454
475
  };
455
476
  this.persist();
456
477
  try {
457
- this.completePendingActivationCleanup(scope.sessionId);
478
+ await this.completePendingActivationCleanup(scope.sessionId);
458
479
  }
459
480
  catch (error) {
460
481
  this.log.error("failed to revoke a terminal runtime session workspace", {
@@ -472,19 +493,31 @@ export class AgentServiceSandboxClient {
472
493
  this.currentTurnSessionId = null;
473
494
  this.persist();
474
495
  }
475
- completePendingActivationCleanup(sessionId) {
496
+ async completePendingActivationCleanup(sessionId) {
497
+ const existing = this.activationCleanupTasks.get(sessionId);
498
+ if (existing)
499
+ return await existing;
500
+ const task = this.runPendingActivationCleanup(sessionId);
501
+ this.activationCleanupTasks.set(sessionId, task);
502
+ try {
503
+ await task;
504
+ }
505
+ finally {
506
+ if (this.activationCleanupTasks.get(sessionId) === task) {
507
+ this.activationCleanupTasks.delete(sessionId);
508
+ }
509
+ }
510
+ }
511
+ async runPendingActivationCleanup(sessionId) {
476
512
  const session = this.state.sessions[sessionId];
477
513
  const pending = session?.pendingActivationCleanup;
478
514
  if (!session || !pending)
479
515
  return;
480
- revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
481
516
  const activation = this.activationContexts.get(sessionId);
482
517
  if (activation?.activationId === pending.activationId) {
483
518
  this.activationContexts.delete(sessionId);
484
519
  this.sessionProfiles.delete(sessionId);
485
520
  }
486
- if (session.activationId === pending.activationId)
487
- session.activationId = null;
488
521
  if (!session.completedActivations.includes(pending.activationId)) {
489
522
  // Session-close/generation-rotate are the retention bounds. Never evict an
490
523
  // activation tombstone while the session is alive: activation IDs are opaque, so
@@ -499,15 +532,13 @@ export class AgentServiceSandboxClient {
499
532
  delete session.acceptedCommands[commandId];
500
533
  session.pendingActivationCleanup = null;
501
534
  this.turnScopes.delete(pending.agentRunId);
502
- if (this.activeSessionId === sessionId)
503
- this.activeSessionId = null;
504
535
  if (this.currentTurnSessionId === sessionId)
505
536
  this.currentTurnSessionId = null;
506
537
  this.persist();
507
538
  }
508
- recoverPendingActivationCleanups() {
539
+ async recoverPendingActivationCleanups() {
509
540
  for (const sessionId of Object.keys(this.state.sessions)) {
510
- this.completePendingActivationCleanup(sessionId);
541
+ await this.completePendingActivationCleanup(sessionId);
511
542
  }
512
543
  }
513
544
  async run() {
@@ -568,6 +599,23 @@ export class AgentServiceSandboxClient {
568
599
  this.removeOperationalLogSink();
569
600
  this.socket?.close(1000, "daemon_stopping");
570
601
  }
602
+ async stopGracefully() {
603
+ this.dispatcher.cancelAll();
604
+ await quiesceRuntimeWriters(null);
605
+ await flushWorkspaceFilesystem();
606
+ this.stop();
607
+ }
608
+ async drainWorkspace(fence) {
609
+ if (!(await this.dispatcher.drain(10_000)) || !this.isCurrentLifecycleFence(fence)) {
610
+ return;
611
+ }
612
+ await quiesceRuntimeWriters(null);
613
+ await flushWorkspaceFilesystem();
614
+ if (!this.isCurrentLifecycleFence(fence))
615
+ return;
616
+ await this.sendControlFrame("sandbox.drained", {});
617
+ this.stop();
618
+ }
571
619
  async postEvent(agentRunId, event) {
572
620
  const scope = this.turnScopes.get(agentRunId);
573
621
  if (!scope)
@@ -578,6 +626,8 @@ export class AgentServiceSandboxClient {
578
626
  if (!this.sandboxGeneration || !this.connectionEpoch) {
579
627
  throw new Error("Agent Service sandbox is not authenticated");
580
628
  }
629
+ // Terminal truth is never delayed by object storage. Durable workspace checkpointing
630
+ // is a separate lifecycle command issued only for pause/cleanup/rotation.
581
631
  if (event.type === "run.block") {
582
632
  await this.waitForEventCapacity();
583
633
  }
@@ -782,10 +832,11 @@ export class AgentServiceSandboxClient {
782
832
  if (frame.sandbox_generation !== this.state.sandboxGeneration) {
783
833
  // Generation change wipes every session shard (contract §1.4).
784
834
  this.dispatcher.cancelAll();
835
+ await quiesceRuntimeWriters(null);
785
836
  this.rejectPending(new Error("Agent Service sandbox generation changed"));
786
837
  for (const sessionId of Object.keys(this.state.sessions)) {
787
838
  revokeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
788
- removeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
839
+ removeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration, nasWorkspaceEnabled());
789
840
  }
790
841
  this.state.sessions = {};
791
842
  this.state.sandboxGeneration = frame.sandbox_generation;
@@ -800,7 +851,7 @@ export class AgentServiceSandboxClient {
800
851
  this.connectionEpoch = frame.connection_epoch;
801
852
  this.inboundSeq = frame.seq;
802
853
  this.outboundSeq = frame.connection_epoch * SEQ_EPOCH_BASE;
803
- this.recoverPendingActivationCleanups();
854
+ await this.recoverPendingActivationCleanups();
804
855
  const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
805
856
  this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
806
857
  const staleSeconds = Number(frame.payload.stale_after_seconds ?? 45);
@@ -815,12 +866,19 @@ export class AgentServiceSandboxClient {
815
866
  this.runtimeLogPolicy = runtimeLogPolicy(frame.payload.runtime_log_policy);
816
867
  await cleanupRuntimeSessionWorkspaceCopyStaging();
817
868
  this.persist();
869
+ assertWorkspaceFilesystem(process.env.BOTLEARN_AGENT_SERVICE_WORKSPACE_ROOT ?? "");
870
+ assertWorkspaceWriterFreezeSupported();
818
871
  await this.sendControlFrame("sandbox.ready", {
819
872
  protocol_versions: [AGENT_SERVICE_WS_SCHEMA],
820
873
  capabilities: [
821
874
  WORKSPACE_FILE_READ_BINARY_CAPABILITY,
875
+ "workspace_writer_freeze_v1",
876
+ "workspace_snapshot_v2",
877
+ "workspace_restore_v1",
878
+ ...(nasWorkspaceEnabled() ? ["workspace_nas_v1"] : []),
822
879
  ...(workspaceCopyV1Supported() ? ["workspace_copy_v1"] : []),
823
880
  ],
881
+ workspace_snapshot_policy: workspaceSnapshotPolicyCapability(),
824
882
  daemon_version: this.options.daemonVersion,
825
883
  runtime_versions: {
826
884
  course_daemon: this.options.daemonVersion,
@@ -952,16 +1010,19 @@ export class AgentServiceSandboxClient {
952
1010
  });
953
1011
  });
954
1012
  return;
1013
+ case "workspace.checkpoint":
1014
+ this.scheduleLifecycle("workspace.checkpoint", () => this.handleWorkspaceCheckpoint(frame));
1015
+ return;
1016
+ case "workspace.thaw":
1017
+ await thawRuntimeWriters();
1018
+ await this.sendCommandAck(frame, "ok");
1019
+ return;
955
1020
  case "sandbox.shutdown":
956
- this.stop();
1021
+ await this.stopGracefully();
957
1022
  return;
958
1023
  case "sandbox.drain":
959
1024
  await this.sendCommandAck(frame, "ok");
960
- this.scheduleLifecycle("sandbox.drain", async (fence) => {
961
- if (await this.dispatcher.drain(10_000) && this.isCurrentLifecycleFence(fence)) {
962
- await this.sendControlFrame("sandbox.drained", {});
963
- }
964
- });
1025
+ this.scheduleLifecycle("sandbox.drain", (fence) => this.drainWorkspace(fence));
965
1026
  return;
966
1027
  case "protocol.error":
967
1028
  throw new Error(`server protocol error: ${String(frame.payload.code ?? "unknown")}`);
@@ -979,16 +1040,15 @@ export class AgentServiceSandboxClient {
979
1040
  this.scheduleLifecycle("sandbox.sync.close", (fence) => this.closeSession(sessionId, "sync_close", frame, fence));
980
1041
  }
981
1042
  const state = frame.payload.state;
1043
+ if (frame.payload.thaw_runtime_writers === true) {
1044
+ await thawRuntimeWriters();
1045
+ }
982
1046
  if (state === "shutdown") {
983
- this.stop();
1047
+ await this.stopGracefully();
984
1048
  return;
985
1049
  }
986
1050
  if (state === "drain") {
987
- this.scheduleLifecycle("sandbox.sync.drain", async (fence) => {
988
- if (await this.dispatcher.drain(10_000) && this.isCurrentLifecycleFence(fence)) {
989
- await this.sendControlFrame("sandbox.drained", {});
990
- }
991
- });
1051
+ this.scheduleLifecycle("sandbox.sync.drain", (fence) => this.drainWorkspace(fence));
992
1052
  }
993
1053
  // idle/run/cancel:实际工作全部由显式命令帧下发(合同 §1.3)。
994
1054
  }
@@ -998,6 +1058,7 @@ export class AgentServiceSandboxClient {
998
1058
  const courseRunId = frame.payload.course_run_id;
999
1059
  const requestedWorkspaceRef = frame.payload.workspace_ref;
1000
1060
  const requestedCopy = frame.payload.workspace_copy;
1061
+ const requestedRestore = frame.payload.workspace_restore;
1001
1062
  const requestedCopyRecord = requestedCopy && typeof requestedCopy === "object"
1002
1063
  ? requestedCopy
1003
1064
  : undefined;
@@ -1018,7 +1079,11 @@ export class AgentServiceSandboxClient {
1018
1079
  typeof copyId !== "string" ||
1019
1080
  copyId.length < 1 ||
1020
1081
  typeof requestedCopySource !== "string" ||
1021
- requestedCopySource.length < 1))) {
1082
+ requestedCopySource.length < 1))
1083
+ || (requestedRestore !== null && requestedRestore !== undefined &&
1084
+ (typeof requestedRestore !== "object" || Array.isArray(requestedRestore)))
1085
+ || (requestedCopy !== null && requestedCopy !== undefined &&
1086
+ requestedRestore !== null && requestedRestore !== undefined)) {
1022
1087
  throw new Error("invalid session.open payload");
1023
1088
  }
1024
1089
  if (session.courseRunId !== null && session.courseRunId !== courseRunId) {
@@ -1032,6 +1097,165 @@ export class AgentServiceSandboxClient {
1032
1097
  const isNewSession = session.courseRunId === null;
1033
1098
  let markerPath = null;
1034
1099
  try {
1100
+ if (requestedRestore && typeof requestedRestore === "object") {
1101
+ const raw = requestedRestore;
1102
+ const descriptorRaw = raw.descriptor;
1103
+ const controlFence = raw.control_fence;
1104
+ let restoreControlToken = raw.control_token;
1105
+ if (!controlFence || typeof controlFence !== "object" || Array.isArray(controlFence)) {
1106
+ throw new Error("invalid workspace restore control fence");
1107
+ }
1108
+ if (typeof restoreControlToken !== "string" || restoreControlToken.length < 32) {
1109
+ throw new Error("invalid workspace restore control token");
1110
+ }
1111
+ if (!descriptorRaw || !Array.isArray(raw.entries)) {
1112
+ throw new Error("invalid workspace restore payload");
1113
+ }
1114
+ const descriptor = {
1115
+ sandboxId: this.options.sandboxId,
1116
+ sandboxGeneration: this.sandboxGeneration,
1117
+ runtimeSessionId: sessionId,
1118
+ continuityState: descriptorRaw.continuity_state,
1119
+ continuityErrorCode: descriptorRaw.continuity_error_code,
1120
+ snapshotId: descriptorRaw.snapshot_id,
1121
+ revision: Number(descriptorRaw.revision),
1122
+ contentSha256: descriptorRaw.content_sha256,
1123
+ expectedFileCount: Number(descriptorRaw.expected_file_count),
1124
+ expectedEntryCount: Number(descriptorRaw.expected_entry_count),
1125
+ expectedTotalBytes: Number(descriptorRaw.expected_total_bytes),
1126
+ };
1127
+ const workspaceDirectory = runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration);
1128
+ const requiredFreeBytes = Math.max(WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes, descriptor.expectedTotalBytes + WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes);
1129
+ // Fail before issuing or consuming any object grant. The download staging is a
1130
+ // sibling on this same bounded backing filesystem and is renamed, not copied,
1131
+ // into the atomic restore staging tree.
1132
+ assertWorkspaceRestoreCapacity({
1133
+ workspaceDirectory,
1134
+ descriptor,
1135
+ requiredFreeBytes,
1136
+ entryCopies: 2,
1137
+ });
1138
+ const entries = [];
1139
+ const predownloadRoot = path.join(path.dirname(workspaceDirectory), `.botlearn-predownload-${sessionId}-${randomUUID()}`);
1140
+ mkdirSync(predownloadRoot, { recursive: true, mode: 0o700 });
1141
+ try {
1142
+ let nextCursor = null;
1143
+ do {
1144
+ if (nextCursor !== null && nextCursor.length > 2048) {
1145
+ throw new Error("invalid workspace restore cursor");
1146
+ }
1147
+ const pageSignal = AbortSignal.timeout(60_000);
1148
+ const response = await fetch(`${agentServiceHttpBase(this.options.wsUrl)}/runtime-sessions/${sessionId}/restore-grants`, {
1149
+ method: "POST",
1150
+ headers: {
1151
+ Authorization: `Bearer ${restoreControlToken}`,
1152
+ "Content-Type": "application/json",
1153
+ },
1154
+ body: JSON.stringify({
1155
+ policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
1156
+ fence: controlFence,
1157
+ cursor: nextCursor,
1158
+ // Fetch one canonical entry at a time so a grant is consumed as soon as
1159
+ // it is signed, including deployments using the minimum 60-second TTL.
1160
+ max_entries: 1,
1161
+ }),
1162
+ signal: pageSignal,
1163
+ });
1164
+ if (!response.ok) {
1165
+ let code = "workspace_restore_unavailable";
1166
+ let retryable = response.status >= 500 || response.status === 408 ||
1167
+ response.status === 429;
1168
+ try {
1169
+ const rejected = await response.json();
1170
+ const detail = rejected.detail;
1171
+ if (detail && typeof detail === "object" && !Array.isArray(detail)) {
1172
+ const stableCode = detail.code;
1173
+ if (typeof stableCode === "string" && stableCode.length <= 120) {
1174
+ code = stableCode;
1175
+ }
1176
+ if (typeof detail.retryable === "boolean") {
1177
+ retryable = detail.retryable;
1178
+ }
1179
+ }
1180
+ }
1181
+ catch {
1182
+ // Content-free fallback retains the stable local failure code.
1183
+ }
1184
+ throw new WorkspaceRestoreError(code, retryable);
1185
+ }
1186
+ const page = await response.json();
1187
+ if (!Array.isArray(page.entries))
1188
+ throw new Error("invalid workspace restore page");
1189
+ if (typeof page.control_token !== "string" || page.control_token.length < 32) {
1190
+ throw new Error("invalid refreshed workspace restore control token");
1191
+ }
1192
+ restoreControlToken = page.control_token;
1193
+ const pageFiles = [];
1194
+ for (const value of page.entries) {
1195
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
1196
+ throw new Error("invalid workspace restore entry");
1197
+ }
1198
+ const item = value;
1199
+ if (item.entry_type === "directory") {
1200
+ entries.push({ type: "directory", path: String(item.path), mode: "0700" });
1201
+ continue;
1202
+ }
1203
+ if (item.entry_type !== "file" || !item.grant || typeof item.grant !== "object") {
1204
+ throw new Error("invalid workspace restore file grant");
1205
+ }
1206
+ const entry = {
1207
+ type: "file",
1208
+ path: String(item.path),
1209
+ mode: item.mode,
1210
+ sizeBytes: Number(item.size_bytes),
1211
+ sha256: String(item.sha256),
1212
+ };
1213
+ entries.push(entry);
1214
+ pageFiles.push({ entry, grant: item.grant });
1215
+ }
1216
+ // Validate all paths and duplicates before using any page path locally.
1217
+ validateWorkspaceEntrySet({ schemaVersion: "agent-workspace-entry-set/1", entries }, WORKSPACE_SNAPSHOT_POLICY_V1.limits);
1218
+ for (const { entry, grant } of pageFiles) {
1219
+ if (typeof grant.url !== "string")
1220
+ throw new Error("workspace restore grant missing");
1221
+ const destination = path.join(predownloadRoot, ...entry.path.split("/"));
1222
+ mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
1223
+ const part = `${destination}.part`;
1224
+ const downloadSignal = AbortSignal.timeout(60_000);
1225
+ const download = await fetch(grant.url, {
1226
+ method: "GET",
1227
+ headers: grant.headers,
1228
+ signal: downloadSignal,
1229
+ });
1230
+ if (!download.ok)
1231
+ throw new Error(`workspace_restore_download_${download.status}`);
1232
+ if (!download.body)
1233
+ throw new Error("workspace_restore_download_body_missing");
1234
+ await pipeline(Readable.fromWeb(download.body), createWriteStream(part, { mode: 0o600 }), { signal: downloadSignal });
1235
+ renameSync(part, destination);
1236
+ }
1237
+ const rawNext = page.next_cursor;
1238
+ if (rawNext !== null && rawNext !== undefined && typeof rawNext !== "string") {
1239
+ throw new Error("invalid workspace restore cursor");
1240
+ }
1241
+ nextCursor = rawNext ?? null;
1242
+ } while (nextCursor !== null);
1243
+ await restoreWorkspaceSnapshot({
1244
+ workspaceDirectory,
1245
+ controlDirectory: path.join(ensureDaemonHome(), "agent-service-sandboxes", this.options.sandboxId, "workspace-control"),
1246
+ descriptor,
1247
+ entrySet: { schemaVersion: "agent-workspace-entry-set/1", entries },
1248
+ limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
1249
+ requiredFreeBytes,
1250
+ downloadFile: async (entry, partPath) => {
1251
+ renameSync(path.join(predownloadRoot, ...entry.path.split("/")), partPath);
1252
+ },
1253
+ });
1254
+ }
1255
+ finally {
1256
+ rmSync(predownloadRoot, { recursive: true, force: true });
1257
+ }
1258
+ }
1035
1259
  if (typeof requestedCopySource === "string" && typeof copyId === "string") {
1036
1260
  if (session.workspaceCopyReceipt !== null) {
1037
1261
  const receipt = session.workspaceCopyReceipt;
@@ -1075,28 +1299,161 @@ export class AgentServiceSandboxClient {
1075
1299
  });
1076
1300
  return;
1077
1301
  }
1302
+ if (requestedRestore) {
1303
+ const restoreError = error instanceof WorkspaceRestoreError ? error : null;
1304
+ await this.sendSessionFrame("session.open_failed", sessionId, {
1305
+ error_code: restoreError?.code ?? "workspace_restore_unavailable",
1306
+ retryable: restoreError?.retryable ?? true,
1307
+ });
1308
+ return;
1309
+ }
1078
1310
  throw error;
1079
1311
  }
1080
1312
  session.courseRunId = courseRunId;
1081
1313
  session.workspaceRef = session.workspaceRef ?? (typeof requestedWorkspaceRef === "string"
1082
1314
  ? requestedWorkspaceRef
1083
1315
  : `ws_${sessionId.replaceAll("-", "")}_g${this.sandboxGeneration}`);
1084
- if (session.workspaceCopyReceipt === null) {
1316
+ // Replaying session.open is part of reconnect reconciliation. An already-active
1317
+ // runtime may still be creating files in this workspace, so never transiently revoke
1318
+ // its group write/execute access before the following session.activate arrives.
1319
+ if (session.workspaceCopyReceipt === null && (isNewSession || session.activationId === null)) {
1085
1320
  ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
1086
1321
  }
1087
- this.state.sessions[sessionId] = session;
1088
- this.persist();
1089
1322
  if (markerPath !== null) {
1090
1323
  await finalizeRuntimeSessionWorkspaceCopy(markerPath);
1091
1324
  }
1325
+ try {
1326
+ if (!nasWorkspaceEnabled()) {
1327
+ await applyRuntimeWorkspaceQuota(sessionId, this.sandboxGeneration);
1328
+ }
1329
+ }
1330
+ catch (error) {
1331
+ if (requestedRestore) {
1332
+ await this.sendSessionFrame("session.open_failed", sessionId, {
1333
+ error_code: "workspace_restore_unavailable",
1334
+ });
1335
+ return;
1336
+ }
1337
+ if (requestedCopy) {
1338
+ await this.sendSessionFrame("session.open_failed", sessionId, {
1339
+ error_code: "workspace_migration_io_failed",
1340
+ });
1341
+ return;
1342
+ }
1343
+ throw error;
1344
+ }
1345
+ this.state.sessions[sessionId] = session;
1346
+ this.persist();
1347
+ const materialization = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
1092
1348
  await this.sendSessionFrame("session.opened", sessionId, {
1093
1349
  workspace_ref: session.workspaceRef,
1094
1350
  native_session_id: session.nativeSessionId,
1351
+ ...(materialization
1352
+ ? {
1353
+ workspace_marker_version: 1,
1354
+ workspace_continuity_state: materialization.workspaceContinuityState,
1355
+ workspace_snapshot_id: materialization.snapshotId,
1356
+ workspace_revision: materialization.revision,
1357
+ workspace_content_sha256: materialization.contentSha256,
1358
+ }
1359
+ : {}),
1095
1360
  ...(session.workspaceCopyReceipt !== null
1096
1361
  ? { workspace_copy_receipt: session.workspaceCopyReceipt }
1097
1362
  : {}),
1098
1363
  });
1099
1364
  }
1365
+ async handleWorkspaceCheckpoint(frame) {
1366
+ const sessionId = frame.runtime_session_id;
1367
+ const session = this.state.sessions[sessionId];
1368
+ const checkpointId = frame.payload.checkpoint_id;
1369
+ const controlToken = frame.payload.control_token;
1370
+ const baseRevision = Number(frame.payload.base_revision);
1371
+ const sourceKind = frame.payload.source_kind;
1372
+ if (!session ||
1373
+ this.currentTurnSessionId === sessionId ||
1374
+ typeof checkpointId !== "string" ||
1375
+ typeof controlToken !== "string" ||
1376
+ controlToken.length < 32 ||
1377
+ frame.payload.policy_id !== "WorkspaceSnapshotPolicyV1" ||
1378
+ ![
1379
+ "sandbox_pause",
1380
+ "sandbox_cleanup",
1381
+ "generation_rotation",
1382
+ "initial_migration",
1383
+ "live_workspace_copy",
1384
+ "maintenance_repair",
1385
+ ].includes(String(sourceKind)) ||
1386
+ !Number.isSafeInteger(baseRevision) ||
1387
+ baseRevision < 0 ||
1388
+ frame.payload.agent_run_id != null ||
1389
+ frame.payload.worker_attempt != null ||
1390
+ frame.payload.activation_id != null) {
1391
+ await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
1392
+ return;
1393
+ }
1394
+ const lifecycleCheckpoint = [
1395
+ "sandbox_pause",
1396
+ "sandbox_cleanup",
1397
+ "generation_rotation",
1398
+ ].includes(String(sourceKind));
1399
+ let checkpointCommitted = false;
1400
+ try {
1401
+ await freezeRuntimeWriters();
1402
+ const freezeProof = issueWorkspaceWriterFreezeProof({
1403
+ sandboxId: this.options.sandboxId,
1404
+ sandboxGeneration: this.sandboxGeneration,
1405
+ runtimeSessionId: sessionId,
1406
+ checkpointId,
1407
+ });
1408
+ await checkpointWorkspace({
1409
+ wsUrl: this.options.wsUrl,
1410
+ reconnectToken: controlToken,
1411
+ workspaceDirectory: runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration),
1412
+ scope: {
1413
+ checkpointId,
1414
+ baseRevision,
1415
+ sandboxId: this.options.sandboxId,
1416
+ sandboxGeneration: this.sandboxGeneration,
1417
+ connectionEpoch: this.connectionEpoch,
1418
+ runtimeSessionId: sessionId,
1419
+ },
1420
+ freezeProof,
1421
+ });
1422
+ checkpointCommitted = true;
1423
+ const marker = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
1424
+ if (!marker)
1425
+ throw new Error("workspace checkpoint marker missing");
1426
+ await this.sendSessionFrame("session.opened", sessionId, {
1427
+ workspace_ref: session.workspaceRef,
1428
+ native_session_id: session.nativeSessionId,
1429
+ workspace_marker_version: 1,
1430
+ workspace_continuity_state: marker.workspaceContinuityState,
1431
+ workspace_snapshot_id: marker.snapshotId,
1432
+ workspace_revision: marker.revision,
1433
+ workspace_content_sha256: marker.contentSha256,
1434
+ });
1435
+ await this.sendCommandAck(frame, "ok");
1436
+ if (!lifecycleCheckpoint)
1437
+ await thawRuntimeWriters();
1438
+ }
1439
+ catch (error) {
1440
+ if (!lifecycleCheckpoint || !checkpointCommitted) {
1441
+ await thawRuntimeWriters().catch(() => undefined);
1442
+ }
1443
+ this.log.error("workspace baseline checkpoint failed", {
1444
+ sandboxId: this.options.sandboxId,
1445
+ runtimeSessionId: sessionId,
1446
+ error: error instanceof Error ? redactSecretString(error.message) : "unexpected",
1447
+ });
1448
+ const deterministic = (error instanceof WorkspaceSnapshotStagingError && !error.retryable) ||
1449
+ error instanceof WorkspaceEntrySetError ||
1450
+ (error instanceof WorkspaceSnapshotControlError && !error.retryable);
1451
+ const errorCode = deterministic && "code" in error
1452
+ ? String(error.code)
1453
+ : "workspace_checkpoint_retryable";
1454
+ await this.sendCommandAck(frame, "rejected", errorCode, !deterministic);
1455
+ }
1456
+ }
1100
1457
  async handleSessionActivate(frame, fence) {
1101
1458
  const sessionId = frame.runtime_session_id;
1102
1459
  const activationId = frame.activation_id;
@@ -1233,7 +1590,7 @@ export class AgentServiceSandboxClient {
1233
1590
  const previousSessionId = this.activeSessionId;
1234
1591
  const changesActivation = previousSessionId !== null && (previousSessionId !== sessionId || session.activationId !== activationId);
1235
1592
  if (changesActivation) {
1236
- const stopped = await this.deactivateSession(previousSessionId);
1593
+ const stopped = await this.deactivateSession(previousSessionId, previousSessionId !== sessionId);
1237
1594
  if (fence && !this.isCurrentLifecycleFence(fence))
1238
1595
  return;
1239
1596
  if (!stopped) {
@@ -1419,7 +1776,7 @@ export class AgentServiceSandboxClient {
1419
1776
  }
1420
1777
  delete session.acceptedCommands[commandId];
1421
1778
  this.persist();
1422
- this.finishTurn({ agent_run_id: runId });
1779
+ await this.finishTurn({ agent_run_id: runId });
1423
1780
  }
1424
1781
  /**
1425
1782
  * 幂等关闭一个 runtime session:终止其 runtime 子进程、删除 workspace/native state、
@@ -1429,15 +1786,16 @@ export class AgentServiceSandboxClient {
1429
1786
  const session = this.state.sessions[sessionId];
1430
1787
  if (session) {
1431
1788
  const runIds = this.runIdsForSession(sessionId);
1432
- session.activationId = null;
1433
1789
  this.activationContexts.delete(sessionId);
1434
1790
  this.sessionProfiles.delete(sessionId);
1435
- revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1436
1791
  for (const runId of runIds)
1437
1792
  this.dispatcher.cancel(runId);
1438
1793
  if (!await this.dispatcher.waitForRuns(runIds, 10_000)) {
1439
1794
  throw new Error(`runtime session ${sessionId} did not stop before close`);
1440
1795
  }
1796
+ await this.terminateSessionWriters(session);
1797
+ session.activationId = null;
1798
+ revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1441
1799
  if (fence && !this.isCurrentLifecycleFence(fence))
1442
1800
  return;
1443
1801
  if (this.activeSessionId === sessionId)
@@ -1478,7 +1836,7 @@ export class AgentServiceSandboxClient {
1478
1836
  delete session.acceptedCommands[commandId];
1479
1837
  this.persist();
1480
1838
  }
1481
- this.finishTurn({ agent_run_id: accepted.agentRunId });
1839
+ await this.finishTurn({ agent_run_id: accepted.agentRunId });
1482
1840
  }
1483
1841
  catch (error) {
1484
1842
  this.log.error("failed to reconcile an interrupted accepted turn", {
@@ -1580,21 +1938,40 @@ export class AgentServiceSandboxClient {
1580
1938
  .filter(([, scope]) => scope.sessionId === sessionId)
1581
1939
  .map(([runId]) => runId);
1582
1940
  }
1583
- async deactivateSession(sessionId) {
1941
+ async deactivateSession(sessionId, terminateWriters = false) {
1584
1942
  const session = this.state.sessions[sessionId];
1585
1943
  if (!session)
1586
1944
  return true;
1587
1945
  const runIds = this.runIdsForSession(sessionId);
1588
- session.activationId = null;
1589
1946
  this.activationContexts.delete(sessionId);
1590
1947
  this.sessionProfiles.delete(sessionId);
1591
- revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1948
+ if (terminateWriters) {
1949
+ revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
1950
+ }
1592
1951
  if (this.activeSessionId === sessionId)
1593
1952
  this.activeSessionId = null;
1594
1953
  for (const runId of runIds)
1595
1954
  this.dispatcher.cancel(runId);
1596
1955
  this.persist();
1597
- return this.dispatcher.waitForRuns(runIds, 10_000);
1956
+ const stopped = await this.dispatcher.waitForRuns(runIds, 10_000);
1957
+ if (stopped && terminateWriters) {
1958
+ await this.terminateSessionWriters(session);
1959
+ }
1960
+ session.activationId = null;
1961
+ this.persist();
1962
+ return stopped;
1963
+ }
1964
+ async terminateSessionWriters(session) {
1965
+ const activationIds = new Set([
1966
+ ...session.completedActivations,
1967
+ ...(session.activationId ? [session.activationId] : []),
1968
+ ...(session.pendingActivationCleanup?.activationId
1969
+ ? [session.pendingActivationCleanup.activationId]
1970
+ : []),
1971
+ ]);
1972
+ for (const activationId of activationIds) {
1973
+ await quiesceRuntimeWriters(activationId);
1974
+ }
1598
1975
  }
1599
1976
  sameTurnScope(left, right) {
1600
1977
  return left.sessionId === right.sessionId &&
@@ -1687,7 +2064,7 @@ export class AgentServiceSandboxClient {
1687
2064
  spool_frames: this.spoolFrameCount(),
1688
2065
  });
1689
2066
  }
1690
- async sendCommandAck(frame, status, error) {
2067
+ async sendCommandAck(frame, status, error, retryable) {
1691
2068
  if (frame.sandbox_generation !== this.sandboxGeneration ||
1692
2069
  frame.connection_epoch !== this.connectionEpoch)
1693
2070
  return;
@@ -1695,6 +2072,7 @@ export class AgentServiceSandboxClient {
1695
2072
  command_frame_id: frame.frame_id,
1696
2073
  status,
1697
2074
  ...(error !== undefined ? { error } : {}),
2075
+ ...(retryable !== undefined ? { retryable } : {}),
1698
2076
  });
1699
2077
  }
1700
2078
  async handleWorkspaceFileRead(frame) {