@botlearn-course/daemon 0.0.19 → 0.0.20-beta.2

package/dist/agent-service-sandbox.js

@@ -1,4 +1,7 @@
-import { chmodSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { chmodSync, closeSync, createWriteStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import path from "node:path";
 import { ensureDaemonHome } from "./auth-store.js";
 import { AGENT_SERVICE_WS_SCHEMA, AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, createWorkspaceFileChunk, parseSandboxFrame, UnsupportedSandboxProtocolError, WORKSPACE_FILE_READ_BINARY_CAPABILITY, } from "./agent-service-ws-protocol.js";
@@ -8,8 +11,16 @@ import { activationRuntimeEnv, runtimeChildEnv } from "./runtime-env.js";
 import { redactSecretString } from "./redaction.js";
 import { parseRuntimeSkillProviderGrantSet, prepareRuntimeSkillProvider, RuntimeSkillProviderError, } from "./runtime-skills.js";
 import { RunDispatcher, } from "./run-dispatcher.js";
-import { ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, } from "./workspace.js";
+import { cleanupRuntimeSessionWorkspaceCopyStaging, copyRuntimeSessionWorkspace, ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, finalizeRuntimeSessionWorkspaceCopy, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, WORKSPACE_COPY_POLICY_V1, WorkspaceCopyError, workspaceCopyV1Supported, } from "./workspace.js";
 import { readWorkspaceFile, WorkspaceFileReadError } from "./workspace-file-read.js";
+import { checkpointWorkspace, WorkspaceSnapshotControlError, } from "./workspace-snapshot-control.js";
+import { issueWorkspaceQuiescenceProof, quiesceRuntimeWriters, } from "./runtime-quiescence.js";
+import { WorkspaceSnapshotStagingError } from "./workspace-snapshot-staging.js";
+import { validateWorkspaceEntrySet, WorkspaceEntrySetError, } from "./workspace-entry-set.js";
+import { WORKSPACE_SNAPSHOT_POLICY_V1, workspaceSnapshotPolicyCapability, } from "./workspace-snapshot-policy.js";
+import { assertWorkspaceRestoreCapacity, restoreWorkspaceSnapshot, WorkspaceRestoreError, } from "./workspace-restore.js";
+import { readWorkspaceMaterializationMarker } from "./workspace-materialization.js";
+import { applyRuntimeWorkspaceQuota, assertRuntimeWorkspaceQuota } from "./workspace-quota.js";
 import { WebSocketClient, } from "./websocket-client.js";
 const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
 const MAX_SPOOL_FRAMES = 1024;
@@ -19,6 +30,14 @@ const LEGACY_EVENT_ACK_WINDOW = 1;
 const INPUT_ATTACHMENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
 /** 出站 seq 基址：seq = connection_epoch * SEQ_EPOCH_BASE + n，跨重连单调（合同 §1.1）。 */
 const SEQ_EPOCH_BASE = 1_000_000_000;
+function agentServiceHttpBase(wsUrl) {
+    const url = new URL(wsUrl);
+    url.protocol = url.protocol === "wss:" ? "https:" : "http:";
+    url.pathname = "/internal/v1";
+    url.search = "";
+    url.hash = "";
+    return url.toString().replace(/\/$/, "");
+}
 const RUNTIME_LOG_WINDOW_MS = 60_000;
 const WORKSPACE_FILE_TRANSFER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
 const DISABLED_RUNTIME_LOG_POLICY = {
@@ -212,6 +231,7 @@ function emptySessionState() {
     return {
         courseRunId: null,
         workspaceRef: null,
+        workspaceCopyReceipt: null,
         runtimeId: null,
         nativeSessionId: null,
         contextRevision: 0,
@@ -230,6 +250,7 @@ function normalizeSessionState(value) {
     return {
         courseRunId: typeof value.courseRunId === "string" ? value.courseRunId : null,
         workspaceRef: typeof value.workspaceRef === "string" ? value.workspaceRef : null,
+        workspaceCopyReceipt: (value.workspaceCopyReceipt && typeof value.workspaceCopyReceipt === "object") ? value.workspaceCopyReceipt : null,
         runtimeId: typeof value.runtimeId === "string" ? value.runtimeId : null,
         nativeSessionId: typeof value.nativeSessionId === "string" ? value.nativeSessionId : null,
         contextRevision: Number(value.contextRevision ?? 0),
@@ -338,12 +359,15 @@ export class AgentServiceSandboxClient {
     dispatcher;
     /** agent_run_id → TURN scope（session/attempt/activation），事件与文件帧路由用。 */
     turnScopes = new Map();
+    turnWorkspaceCheckpoints = new Map();
+    completedWorkspaceCheckpoints = new Set();
     /** 每 run 内嵌 profile（turn payload 自带时优先）。 */
     runProfiles = new Map();
     /** activate 时装配的 per-session profile，turn.start 复用。 */
     sessionProfiles = new Map();
     /** activation-scoped 模型短凭据；只保存在内存中，绝不进入 state.json。 */
     activationContexts = new Map();
+    activationCleanupTasks = new Map();
     pendingAcks = new Map();
     pendingFileReports = new Map();
     inflightCommands = new Set();
@@ -436,7 +460,7 @@ export class AgentServiceSandboxClient {
         session.nativeSessionId = nativeSessionId.trim() || null;
         this.persist();
     }
-    finishTurn(payload) {
+    async finishTurn(payload) {
         const scope = this.turnScopes.get(payload.agent_run_id);
         if (!scope)
             return;
@@ -452,7 +476,7 @@ export class AgentServiceSandboxClient {
             };
             this.persist();
             try {
-                this.completePendingActivationCleanup(scope.sessionId);
+                await this.completePendingActivationCleanup(scope.sessionId);
             }
             catch (error) {
                 this.log.error("failed to revoke a terminal runtime session workspace", {
@@ -470,11 +494,27 @@ export class AgentServiceSandboxClient {
             this.currentTurnSessionId = null;
         this.persist();
     }
-    completePendingActivationCleanup(sessionId) {
+    async completePendingActivationCleanup(sessionId) {
+        const existing = this.activationCleanupTasks.get(sessionId);
+        if (existing)
+            return await existing;
+        const task = this.runPendingActivationCleanup(sessionId);
+        this.activationCleanupTasks.set(sessionId, task);
+        try {
+            await task;
+        }
+        finally {
+            if (this.activationCleanupTasks.get(sessionId) === task) {
+                this.activationCleanupTasks.delete(sessionId);
+            }
+        }
+    }
+    async runPendingActivationCleanup(sessionId) {
         const session = this.state.sessions[sessionId];
         const pending = session?.pendingActivationCleanup;
         if (!session || !pending)
             return;
+        await quiesceRuntimeWriters(pending.activationId);
         revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
         const activation = this.activationContexts.get(sessionId);
         if (activation?.activationId === pending.activationId) {
@@ -503,9 +543,9 @@ export class AgentServiceSandboxClient {
             this.currentTurnSessionId = null;
         this.persist();
     }
-    recoverPendingActivationCleanups() {
+    async recoverPendingActivationCleanups() {
         for (const sessionId of Object.keys(this.state.sessions)) {
-            this.completePendingActivationCleanup(sessionId);
+            await this.completePendingActivationCleanup(sessionId);
         }
     }
     async run() {
@@ -566,6 +606,11 @@ export class AgentServiceSandboxClient {
         this.removeOperationalLogSink();
         this.socket?.close(1000, "daemon_stopping");
     }
+    async stopGracefully() {
+        this.dispatcher.cancelAll();
+        await quiesceRuntimeWriters(null);
+        this.stop();
+    }
     async postEvent(agentRunId, event) {
         const scope = this.turnScopes.get(agentRunId);
         if (!scope)
@@ -576,6 +621,9 @@ export class AgentServiceSandboxClient {
         if (!this.sandboxGeneration || !this.connectionEpoch) {
             throw new Error("Agent Service sandbox is not authenticated");
         }
+        // Terminal truth is never delayed by object storage. Course Service keeps the
+        // checkpoint obligation pending and, after this turn is ACKed and fully quiesced,
+        // re-delivers it as a session-scoped workspace.checkpoint command.
         if (event.type === "run.block") {
             await this.waitForEventCapacity();
         }
@@ -655,6 +703,29 @@ export class AgentServiceSandboxClient {
         await this.sendFrame(frame);
         return await reported;
     }
+    async recordWorkspaceScanMeasurement(agentRunId, measurement) {
+        if (!this.turnScopes.has(agentRunId)) {
+            throw new Error("Agent Service sandbox scan measurement has no active turn scope");
+        }
+        await this.sendFrame(createSandboxFrame({
+            type: "sandbox.log",
+            sandboxId: this.options.sandboxId,
+            sandboxGeneration: this.sandboxGeneration,
+            connectionEpoch: this.connectionEpoch,
+            seq: this.nextOutboundSeq(),
+            payload: {
+                level: "info",
+                stream: "daemon",
+                message: "workspace.scan.measurement",
+                fields: {
+                    file_count: measurement.fileCount,
+                    total_bytes: measurement.totalBytes,
+                    duration_ms: measurement.durationMs,
+                    result: measurement.truncated ? "truncated" : "complete",
+                },
+            },
+        }));
+    }
     async getRunRuntimeProfile(agentRunId) {
         const embedded = this.runProfiles.get(agentRunId);
         if (embedded)
@@ -744,6 +815,10 @@ export class AgentServiceSandboxClient {
         }
     }
     async handleHello(frame) {
+        // A reconnect can race a copy that started on the previous socket. Wait for
+        // the serialized lifecycle attempt to converge before deleting crash-left
+        // staging and advertising capability on the new connection.
+        await this.lifecycleChain;
         if (frame.sandbox_id !== this.options.sandboxId) {
             throw new SandboxClosedError(4403, "sandbox_mismatch");
         }
@@ -753,6 +828,7 @@ export class AgentServiceSandboxClient {
         if (frame.sandbox_generation !== this.state.sandboxGeneration) {
             // Generation change wipes every session shard (contract §1.4).
             this.dispatcher.cancelAll();
+            await quiesceRuntimeWriters(null);
             this.rejectPending(new Error("Agent Service sandbox generation changed"));
             for (const sessionId of Object.keys(this.state.sessions)) {
                 revokeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
@@ -771,7 +847,7 @@ export class AgentServiceSandboxClient {
         this.connectionEpoch = frame.connection_epoch;
         this.inboundSeq = frame.seq;
         this.outboundSeq = frame.connection_epoch * SEQ_EPOCH_BASE;
-        this.recoverPendingActivationCleanups();
+        await this.recoverPendingActivationCleanups();
         const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
         this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
         const staleSeconds = Number(frame.payload.stale_after_seconds ?? 45);
@@ -784,10 +860,18 @@ export class AgentServiceSandboxClient {
             : LEGACY_EVENT_ACK_WINDOW;
         this.maxUnackedEvents = Math.min(MAX_EVENT_ACK_WINDOW, ackWindow);
         this.runtimeLogPolicy = runtimeLogPolicy(frame.payload.runtime_log_policy);
+        await cleanupRuntimeSessionWorkspaceCopyStaging();
         this.persist();
+        assertRuntimeWorkspaceQuota(process.env.BOTLEARN_AGENT_SERVICE_WORKSPACE_ROOT ?? "");
         await this.sendControlFrame("sandbox.ready", {
             protocol_versions: [AGENT_SERVICE_WS_SCHEMA],
-            capabilities: [WORKSPACE_FILE_READ_BINARY_CAPABILITY],
+            capabilities: [
+                WORKSPACE_FILE_READ_BINARY_CAPABILITY,
+                "workspace_snapshot_v1",
+                "workspace_restore_v1",
+                ...(workspaceCopyV1Supported() ? ["workspace_copy_v1"] : []),
+            ],
+            workspace_snapshot_policy: workspaceSnapshotPolicyCapability(),
             daemon_version: this.options.daemonVersion,
             runtime_versions: {
                 course_daemon: this.options.daemonVersion,
@@ -919,8 +1003,11 @@ export class AgentServiceSandboxClient {
                     });
                 });
                 return;
+            case "workspace.checkpoint":
+                this.scheduleLifecycle("workspace.checkpoint", () => this.handleWorkspaceCheckpoint(frame));
+                return;
             case "sandbox.shutdown":
-                this.stop();
+                await this.stopGracefully();
                 return;
             case "sandbox.drain":
                 await this.sendCommandAck(frame, "ok");
@@ -947,7 +1034,7 @@ export class AgentServiceSandboxClient {
         }
         const state = frame.payload.state;
         if (state === "shutdown") {
-            this.stop();
+            await this.stopGracefully();
             return;
         }
         if (state === "drain") {
@@ -964,11 +1051,33 @@ export class AgentServiceSandboxClient {
         const session = this.state.sessions[sessionId] ?? emptySessionState();
         const courseRunId = frame.payload.course_run_id;
         const requestedWorkspaceRef = frame.payload.workspace_ref;
+        const requestedCopy = frame.payload.workspace_copy;
+        const requestedRestore = frame.payload.workspace_restore;
+        const requestedCopyRecord = requestedCopy && typeof requestedCopy === "object"
+            ? requestedCopy
+            : undefined;
+        const copyId = requestedCopyRecord
+            ? requestedCopyRecord.copy_id
+            : undefined;
+        const requestedCopySource = requestedCopyRecord
+            ? requestedCopyRecord.source_runtime_session_id
+            : undefined;
         if (typeof courseRunId !== "string" ||
             courseRunId.length < 1 ||
             (requestedWorkspaceRef !== null &&
                 requestedWorkspaceRef !== undefined &&
-                (typeof requestedWorkspaceRef !== "string" || requestedWorkspaceRef.length < 1))) {
+                (typeof requestedWorkspaceRef !== "string" || requestedWorkspaceRef.length < 1)) ||
+            (requestedCopy !== null &&
+                requestedCopy !== undefined &&
+                (typeof requestedCopy !== "object" ||
+                    typeof copyId !== "string" ||
+                    copyId.length < 1 ||
+                    typeof requestedCopySource !== "string" ||
+                    requestedCopySource.length < 1))
+            || (requestedRestore !== null && requestedRestore !== undefined &&
+                (typeof requestedRestore !== "object" || Array.isArray(requestedRestore)))
+            || (requestedCopy !== null && requestedCopy !== undefined &&
+                requestedRestore !== null && requestedRestore !== undefined)) {
             throw new Error("invalid session.open payload");
         }
         if (session.courseRunId !== null && session.courseRunId !== courseRunId) {
@@ -979,18 +1088,365 @@ export class AgentServiceSandboxClient {
             session.workspaceRef !== requestedWorkspaceRef) {
             throw new Error("runtime session workspace_ref cannot be rebound");
         }
+        const isNewSession = session.courseRunId === null;
+        let markerPath = null;
+        try {
+            if (requestedRestore && typeof requestedRestore === "object") {
+                const raw = requestedRestore;
+                const descriptorRaw = raw.descriptor;
+                const controlFence = raw.control_fence;
+                let restoreControlToken = raw.control_token;
+                if (!controlFence || typeof controlFence !== "object" || Array.isArray(controlFence)) {
+                    throw new Error("invalid workspace restore control fence");
+                }
+                if (typeof restoreControlToken !== "string" || restoreControlToken.length < 32) {
+                    throw new Error("invalid workspace restore control token");
+                }
+                if (!descriptorRaw || !Array.isArray(raw.entries)) {
+                    throw new Error("invalid workspace restore payload");
+                }
+                const descriptor = {
+                    sandboxId: this.options.sandboxId,
+                    sandboxGeneration: this.sandboxGeneration,
+                    runtimeSessionId: sessionId,
+                    continuityState: descriptorRaw.continuity_state,
+                    continuityErrorCode: descriptorRaw.continuity_error_code,
+                    snapshotId: descriptorRaw.snapshot_id,
+                    revision: Number(descriptorRaw.revision),
+                    contentSha256: descriptorRaw.content_sha256,
+                    expectedFileCount: Number(descriptorRaw.expected_file_count),
+                    expectedEntryCount: Number(descriptorRaw.expected_entry_count),
+                    expectedTotalBytes: Number(descriptorRaw.expected_total_bytes),
+                };
+                const workspaceDirectory = runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration);
+                const requiredFreeBytes = Math.max(WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes, descriptor.expectedTotalBytes + WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes);
+                // Fail before issuing or consuming any object grant. The download staging is a
+                // sibling on this same bounded backing filesystem and is renamed, not copied,
+                // into the atomic restore staging tree.
+                assertWorkspaceRestoreCapacity({
+                    workspaceDirectory,
+                    descriptor,
+                    requiredFreeBytes,
+                    entryCopies: 2,
+                });
+                const entries = [];
+                const predownloadRoot = path.join(path.dirname(workspaceDirectory), `.botlearn-predownload-${sessionId}-${randomUUID()}`);
+                mkdirSync(predownloadRoot, { recursive: true, mode: 0o700 });
+                try {
+                    let nextCursor = null;
+                    do {
+                        if (nextCursor !== null && nextCursor.length > 2048) {
+                            throw new Error("invalid workspace restore cursor");
+                        }
+                        const pageSignal = AbortSignal.timeout(60_000);
+                        const response = await fetch(`${agentServiceHttpBase(this.options.wsUrl)}/runtime-sessions/${sessionId}/restore-grants`, {
+                            method: "POST",
+                            headers: {
+                                Authorization: `Bearer ${restoreControlToken}`,
+                                "Content-Type": "application/json",
+                            },
+                            body: JSON.stringify({
+                                policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
+                                fence: controlFence,
+                                cursor: nextCursor,
+                                // Fetch one canonical entry at a time so a grant is consumed as soon as
+                                // it is signed, including deployments using the minimum 60-second TTL.
+                                max_entries: 1,
+                            }),
+                            signal: pageSignal,
+                        });
+                        if (!response.ok) {
+                            let code = "workspace_restore_unavailable";
+                            let retryable = response.status >= 500 || response.status === 408 ||
+                                response.status === 429;
+                            try {
+                                const rejected = await response.json();
+                                const detail = rejected.detail;
+                                if (detail && typeof detail === "object" && !Array.isArray(detail)) {
+                                    const stableCode = detail.code;
+                                    if (typeof stableCode === "string" && stableCode.length <= 120) {
+                                        code = stableCode;
+                                    }
+                                    if (typeof detail.retryable === "boolean") {
+                                        retryable = detail.retryable;
+                                    }
+                                }
+                            }
+                            catch {
+                                // Content-free fallback retains the stable local failure code.
+                            }
+                            throw new WorkspaceRestoreError(code, retryable);
+                        }
+                        const page = await response.json();
+                        if (!Array.isArray(page.entries))
+                            throw new Error("invalid workspace restore page");
+                        if (typeof page.control_token !== "string" || page.control_token.length < 32) {
+                            throw new Error("invalid refreshed workspace restore control token");
+                        }
+                        restoreControlToken = page.control_token;
+                        const pageFiles = [];
+                        for (const value of page.entries) {
+                            if (!value || typeof value !== "object" || Array.isArray(value)) {
+                                throw new Error("invalid workspace restore entry");
+                            }
+                            const item = value;
+                            if (item.entry_type === "directory") {
+                                entries.push({ type: "directory", path: String(item.path), mode: "0700" });
+                                continue;
+                            }
+                            if (item.entry_type !== "file" || !item.grant || typeof item.grant !== "object") {
+                                throw new Error("invalid workspace restore file grant");
+                            }
+                            const entry = {
+                                type: "file",
+                                path: String(item.path),
+                                mode: item.mode,
+                                sizeBytes: Number(item.size_bytes),
+                                sha256: String(item.sha256),
+                            };
+                            entries.push(entry);
+                            pageFiles.push({ entry, grant: item.grant });
+                        }
+                        // Validate all paths and duplicates before using any page path locally.
+                        validateWorkspaceEntrySet({ schemaVersion: "agent-workspace-entry-set/1", entries }, WORKSPACE_SNAPSHOT_POLICY_V1.limits);
+                        for (const { entry, grant } of pageFiles) {
+                            if (typeof grant.url !== "string")
+                                throw new Error("workspace restore grant missing");
+                            const destination = path.join(predownloadRoot, ...entry.path.split("/"));
+                            mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
+                            const part = `${destination}.part`;
+                            const downloadSignal = AbortSignal.timeout(60_000);
+                            const download = await fetch(grant.url, {
+                                method: "GET",
+                                headers: grant.headers,
+                                signal: downloadSignal,
+                            });
+                            if (!download.ok)
+                                throw new Error(`workspace_restore_download_${download.status}`);
+                            if (!download.body)
+                                throw new Error("workspace_restore_download_body_missing");
+                            await pipeline(Readable.fromWeb(download.body), createWriteStream(part, { mode: 0o600 }), { signal: downloadSignal });
+                            renameSync(part, destination);
+                        }
+                        const rawNext = page.next_cursor;
+                        if (rawNext !== null && rawNext !== undefined && typeof rawNext !== "string") {
+                            throw new Error("invalid workspace restore cursor");
+                        }
+                        nextCursor = rawNext ?? null;
+                    } while (nextCursor !== null);
+                    await restoreWorkspaceSnapshot({
+                        workspaceDirectory,
+                        controlDirectory: path.join(ensureDaemonHome(), "agent-service-sandboxes", this.options.sandboxId, "workspace-control"),
+                        descriptor,
+                        entrySet: { schemaVersion: "agent-workspace-entry-set/1", entries },
+                        limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
+                        requiredFreeBytes,
+                        downloadFile: async (entry, partPath) => {
+                            renameSync(path.join(predownloadRoot, ...entry.path.split("/")), partPath);
+                        },
+                    });
+                }
+                finally {
+                    rmSync(predownloadRoot, { recursive: true, force: true });
+                }
+            }
+            if (typeof requestedCopySource === "string" && typeof copyId === "string") {
+                if (session.workspaceCopyReceipt !== null) {
+                    const receipt = session.workspaceCopyReceipt;
+                    if (receipt.copy_id !== copyId ||
+                        receipt.source_runtime_session_id !== requestedCopySource ||
+                        receipt.target_runtime_session_id !== sessionId ||
+                        receipt.source_sandbox_id !== this.options.sandboxId ||
+                        receipt.source_generation !== this.sandboxGeneration) {
+                        throw new WorkspaceCopyError("workspace_migration_receipt_mismatch");
+                    }
+                    markerPath = path.join(runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration), WORKSPACE_COPY_POLICY_V1.markerName);
+                }
+                else if (!isNewSession) {
+                    throw new WorkspaceCopyError("workspace_migration_receipt_mismatch");
+                }
+            }
+            if (isNewSession && typeof requestedCopySource === "string" && typeof copyId === "string") {
+                const sourceSession = this.state.sessions[requestedCopySource];
+                if (!sourceSession ||
+                    sourceSession.courseRunId === null ||
+                    this.activeSessionId === requestedCopySource ||
+                    this.currentTurnSessionId === requestedCopySource) {
+                    throw new WorkspaceCopyError("workspace_migration_source_unavailable");
+                }
+                revokeRuntimeSessionWorkspace(requestedCopySource, this.sandboxGeneration);
+                const copied = await copyRuntimeSessionWorkspace({
+                    copyId,
+                    sourceRuntimeSessionId: requestedCopySource,
+                    targetRuntimeSessionId: sessionId,
+                    sandboxId: this.options.sandboxId,
+                    sandboxGeneration: this.sandboxGeneration,
+                });
+                session.workspaceCopyReceipt = copied.receipt;
+                markerPath = copied.markerPath;
+            }
+        }
+        catch (error) {
+            if (error instanceof WorkspaceCopyError) {
+                await this.sendSessionFrame("session.open_failed", sessionId, {
+                    error_code: error.code,
+                });
+                return;
+            }
+            if (requestedRestore) {
+                const restoreError = error instanceof WorkspaceRestoreError ? error : null;
+                await this.sendSessionFrame("session.open_failed", sessionId, {
+                    error_code: restoreError?.code ?? "workspace_restore_unavailable",
+                    retryable: restoreError?.retryable ?? true,
+                });
+                return;
+            }
+            throw error;
+        }
         session.courseRunId = courseRunId;
         session.workspaceRef = session.workspaceRef ?? (typeof requestedWorkspaceRef === "string"
             ? requestedWorkspaceRef
             : `ws_${sessionId.replaceAll("-", "")}_g${this.sandboxGeneration}`);
-        ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
+        // Replaying session.open is part of reconnect reconciliation.  An already-active
+        // runtime may still be creating files in this workspace, so never transiently revoke
+        // its group write/execute access before the following session.activate arrives.
+        if (session.workspaceCopyReceipt === null && (isNewSession || session.activationId === null)) {
+            ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
+        }
+        if (markerPath !== null) {
+            await finalizeRuntimeSessionWorkspaceCopy(markerPath);
+        }
+        try {
+            await applyRuntimeWorkspaceQuota(sessionId, this.sandboxGeneration);
+        }
+        catch (error) {
+            if (requestedRestore) {
+                await this.sendSessionFrame("session.open_failed", sessionId, {
+                    error_code: "workspace_restore_unavailable",
+                });
+                return;
+            }
+            if (requestedCopy) {
+                await this.sendSessionFrame("session.open_failed", sessionId, {
+                    error_code: "workspace_migration_io_failed",
+                });
+                return;
+            }
+            throw error;
+        }
         this.state.sessions[sessionId] = session;
         this.persist();
+        const materialization = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
         await this.sendSessionFrame("session.opened", sessionId, {
             workspace_ref: session.workspaceRef,
             native_session_id: session.nativeSessionId,
+            ...(materialization
+                ? {
+                    workspace_marker_version: 1,
+                    workspace_continuity_state: materialization.workspaceContinuityState,
+                    workspace_snapshot_id: materialization.snapshotId,
+                    workspace_revision: materialization.revision,
+                    workspace_content_sha256: materialization.contentSha256,
+                }
+                : {}),
+            ...(session.workspaceCopyReceipt !== null
+                ? { workspace_copy_receipt: session.workspaceCopyReceipt }
+                : {}),
         });
     }
+    async handleWorkspaceCheckpoint(frame) {
+        const sessionId = frame.runtime_session_id;
+        const session = this.state.sessions[sessionId];
+        const checkpointId = frame.payload.checkpoint_id;
+        const controlToken = frame.payload.control_token;
+        const baseRevision = Number(frame.payload.base_revision);
+        const agentRunId = frame.payload.agent_run_id;
+        const workerAttempt = frame.payload.worker_attempt;
+        const activationId = frame.payload.activation_id;
+        const pendingCleanup = session?.pendingActivationCleanup;
+        if (pendingCleanup &&
+            pendingCleanup.agentRunId === agentRunId &&
+            pendingCleanup.workerAttempt === workerAttempt &&
+            pendingCleanup.activationId === activationId) {
+            await this.completePendingActivationCleanup(sessionId);
+        }
+        if (!session ||
+            this.activeSessionId === sessionId ||
+            this.currentTurnSessionId === sessionId ||
+            typeof checkpointId !== "string" ||
+            typeof controlToken !== "string" ||
+            controlToken.length < 32 ||
+            frame.payload.policy_id !== "WorkspaceSnapshotPolicyV1" ||
+            !Number.isSafeInteger(baseRevision) ||
+            baseRevision < 0 ||
+            !((agentRunId == null && workerAttempt == null && activationId == null) ||
+                (typeof agentRunId === "string" &&
+                    Number.isSafeInteger(workerAttempt) &&
+                    Number(workerAttempt) >= 1 &&
+                    typeof activationId === "string" &&
+                    activationId.length > 0))) {
+            await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
+            return;
+        }
+        try {
+            revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
+            await quiesceRuntimeWriters(typeof activationId === "string" ? activationId : null);
+            const quiescenceProof = issueWorkspaceQuiescenceProof({
+                sandboxId: this.options.sandboxId,
+                sandboxGeneration: this.sandboxGeneration,
+                runtimeSessionId: sessionId,
+                checkpointId,
+            });
+            await checkpointWorkspace({
+                wsUrl: this.options.wsUrl,
+                reconnectToken: controlToken,
+                workspaceDirectory: runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration),
+                scope: {
+                    checkpointId,
+                    baseRevision,
+                    sandboxId: this.options.sandboxId,
+                    sandboxGeneration: this.sandboxGeneration,
+                    connectionEpoch: this.connectionEpoch,
+                    runtimeSessionId: sessionId,
+                    ...(typeof agentRunId === "string"
+                        ? {
+                            agentRunId,
+                            workerAttempt: Number(workerAttempt),
+                            activationId: String(activationId),
+                        }
+                        : {}),
+                },
+                quiescenceProof,
+            });
+            const marker = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
+            if (!marker)
+                throw new Error("workspace checkpoint marker missing");
+            await this.sendSessionFrame("session.opened", sessionId, {
+                workspace_ref: session.workspaceRef,
+                native_session_id: session.nativeSessionId,
+                workspace_marker_version: 1,
+                workspace_continuity_state: marker.workspaceContinuityState,
+                workspace_snapshot_id: marker.snapshotId,
+                workspace_revision: marker.revision,
+                workspace_content_sha256: marker.contentSha256,
+            });
+            await this.sendCommandAck(frame, "ok");
+        }
+        catch (error) {
+            this.log.error("workspace baseline checkpoint failed", {
+                sandboxId: this.options.sandboxId,
+                runtimeSessionId: sessionId,
+                error: error instanceof Error ? redactSecretString(error.message) : "unexpected",
+            });
+            const deterministic = (error instanceof WorkspaceSnapshotStagingError && !error.retryable) ||
+                error instanceof WorkspaceEntrySetError ||
+                (error instanceof WorkspaceSnapshotControlError && !error.retryable);
+            const errorCode = deterministic && "code" in error
+                ? String(error.code)
+                : "workspace_checkpoint_retryable";
+            await this.sendCommandAck(frame, "rejected", errorCode, !deterministic);
+        }
+    }
     async handleSessionActivate(frame, fence) {
         const sessionId = frame.runtime_session_id;
         const activationId = frame.activation_id;
@@ -1177,6 +1633,21 @@ export class AgentServiceSandboxClient {
             await this.sendCommandAck(frame, "rejected", "activation_mismatch");
             return;
         }
+        const checkpoint = frame.payload.workspace_checkpoint;
+        if (checkpoint !== undefined) {
+            if (!checkpoint || typeof checkpoint !== "object" || Array.isArray(checkpoint) ||
+                typeof checkpoint.checkpoint_id !== "string" ||
+                checkpoint.policy_id !== "WorkspaceSnapshotPolicyV1" ||
+                !Number.isSafeInteger(checkpoint.base_revision) ||
+                Number(checkpoint.base_revision) < 0) {
+                await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
+                return;
+            }
+            this.turnWorkspaceCheckpoints.set(runId, {
+                checkpointId: String(checkpoint.checkpoint_id),
+                baseRevision: Number(checkpoint.base_revision),
+            });
+        }
         const payload = frame.payload;
         if (!this.validTurnPayload(payload)) {
             await this.sendCommandAck(frame, "rejected", "invalid_turn_payload");
@@ -1263,6 +1734,8 @@ export class AgentServiceSandboxClient {
             .finally(() => {
             this.inflightCommands.delete(commandId);
             this.runProfiles.delete(runId);
+            this.turnWorkspaceCheckpoints.delete(runId);
+            this.completedWorkspaceCheckpoints.delete(runId);
         });
     }
     async handleTurnCancel(frame) {
@@ -1313,7 +1786,7 @@ export class AgentServiceSandboxClient {
         }
         delete session.acceptedCommands[commandId];
         this.persist();
-        this.finishTurn({ agent_run_id: runId });
+        await this.finishTurn({ agent_run_id: runId });
     }
     /**
      * 幂等关闭一个 runtime session：终止其 runtime 子进程、删除 workspace/native state、
@@ -1372,7 +1845,7 @@ export class AgentServiceSandboxClient {
                 delete session.acceptedCommands[commandId];
                 this.persist();
             }
-            this.finishTurn({ agent_run_id: accepted.agentRunId });
+            await this.finishTurn({ agent_run_id: accepted.agentRunId });
         }
         catch (error) {
             this.log.error("failed to reconcile an interrupted accepted turn", {
@@ -1581,7 +2054,7 @@ export class AgentServiceSandboxClient {
             spool_frames: this.spoolFrameCount(),
         });
     }
-    async sendCommandAck(frame, status, error) {
+    async sendCommandAck(frame, status, error, retryable) {
         if (frame.sandbox_generation !== this.sandboxGeneration ||
             frame.connection_epoch !== this.connectionEpoch)
             return;
@@ -1589,6 +2062,7 @@ export class AgentServiceSandboxClient {
             command_frame_id: frame.frame_id,
             status,
             ...(error !== undefined ? { error } : {}),
+            ...(retryable !== undefined ? { retryable } : {}),
         });
     }
     async handleWorkspaceFileRead(frame) {