@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-service-sandbox.d.ts +6 -1
- package/dist/agent-service-sandbox.js +398 -28
- package/dist/agent-service-ws-protocol.d.ts +3 -3
- package/dist/agent-service-ws-protocol.js +5 -2
- package/dist/cli.js +19 -1
- package/dist/file-candidates.js +46 -39
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/run-dispatcher.d.ts +1 -1
- package/dist/run-dispatcher.js +1 -1
- package/dist/runtime-env.js +4 -4
- package/dist/runtime-quiescence.d.ts +19 -0
- package/dist/runtime-quiescence.js +110 -0
- package/dist/tool-observation.d.ts +6 -5
- package/dist/tool-observation.js +6 -5
- package/dist/trace-projection.d.ts +1 -1
- package/dist/trace-projection.js +1 -1
- package/dist/workspace-quota.d.ts +4 -0
- package/dist/workspace-quota.js +42 -0
- package/dist/workspace-restore.d.ts +9 -1
- package/dist/workspace-restore.js +30 -16
- package/dist/workspace-snapshot-control.d.ts +29 -0
- package/dist/workspace-snapshot-control.js +169 -0
- package/dist/workspace-snapshot-policy.d.ts +24 -0
- package/dist/workspace-snapshot-policy.js +45 -0
- package/dist/workspace-snapshot-staging.d.ts +2 -9
- package/dist/workspace-snapshot-staging.js +59 -25
- package/package.json +1 -1
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, closeSync, createWriteStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import { Readable } from "node:stream";
|
|
4
|
+
import { pipeline } from "node:stream/promises";
|
|
2
5
|
import path from "node:path";
|
|
3
6
|
import { ensureDaemonHome } from "./auth-store.js";
|
|
4
7
|
import { AGENT_SERVICE_WS_SCHEMA, AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, createWorkspaceFileChunk, parseSandboxFrame, UnsupportedSandboxProtocolError, WORKSPACE_FILE_READ_BINARY_CAPABILITY, } from "./agent-service-ws-protocol.js";
|
|
@@ -10,6 +13,14 @@ import { parseRuntimeSkillProviderGrantSet, prepareRuntimeSkillProvider, Runtime
|
|
|
10
13
|
import { RunDispatcher, } from "./run-dispatcher.js";
|
|
11
14
|
import { cleanupRuntimeSessionWorkspaceCopyStaging, copyRuntimeSessionWorkspace, ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, finalizeRuntimeSessionWorkspaceCopy, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, WORKSPACE_COPY_POLICY_V1, WorkspaceCopyError, workspaceCopyV1Supported, } from "./workspace.js";
|
|
12
15
|
import { readWorkspaceFile, WorkspaceFileReadError } from "./workspace-file-read.js";
|
|
16
|
+
import { checkpointWorkspace, WorkspaceSnapshotControlError, } from "./workspace-snapshot-control.js";
|
|
17
|
+
import { assertWorkspaceWriterFreezeSupported, freezeRuntimeWriters, issueWorkspaceWriterFreezeProof, quiesceRuntimeWriters, thawRuntimeWriters, } from "./runtime-quiescence.js";
|
|
18
|
+
import { WorkspaceSnapshotStagingError } from "./workspace-snapshot-staging.js";
|
|
19
|
+
import { validateWorkspaceEntrySet, WorkspaceEntrySetError, } from "./workspace-entry-set.js";
|
|
20
|
+
import { WORKSPACE_SNAPSHOT_POLICY_V1, workspaceSnapshotPolicyCapability, } from "./workspace-snapshot-policy.js";
|
|
21
|
+
import { assertWorkspaceRestoreCapacity, restoreWorkspaceSnapshot, WorkspaceRestoreError, } from "./workspace-restore.js";
|
|
22
|
+
import { readWorkspaceMaterializationMarker } from "./workspace-materialization.js";
|
|
23
|
+
import { applyRuntimeWorkspaceQuota, assertRuntimeWorkspaceQuota } from "./workspace-quota.js";
|
|
13
24
|
import { WebSocketClient, } from "./websocket-client.js";
|
|
14
25
|
const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
|
|
15
26
|
const MAX_SPOOL_FRAMES = 1024;
|
|
@@ -19,6 +30,14 @@ const LEGACY_EVENT_ACK_WINDOW = 1;
|
|
|
19
30
|
const INPUT_ATTACHMENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
20
31
|
/** 出站 seq 基址:seq = connection_epoch * SEQ_EPOCH_BASE + n,跨重连单调(合同 §1.1)。 */
|
|
21
32
|
const SEQ_EPOCH_BASE = 1_000_000_000;
|
|
33
|
+
function agentServiceHttpBase(wsUrl) {
|
|
34
|
+
const url = new URL(wsUrl);
|
|
35
|
+
url.protocol = url.protocol === "wss:" ? "https:" : "http:";
|
|
36
|
+
url.pathname = "/internal/v1";
|
|
37
|
+
url.search = "";
|
|
38
|
+
url.hash = "";
|
|
39
|
+
return url.toString().replace(/\/$/, "");
|
|
40
|
+
}
|
|
22
41
|
const RUNTIME_LOG_WINDOW_MS = 60_000;
|
|
23
42
|
const WORKSPACE_FILE_TRANSFER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
24
43
|
const DISABLED_RUNTIME_LOG_POLICY = {
|
|
@@ -346,6 +365,7 @@ export class AgentServiceSandboxClient {
|
|
|
346
365
|
sessionProfiles = new Map();
|
|
347
366
|
/** activation-scoped 模型短凭据;只保存在内存中,绝不进入 state.json。 */
|
|
348
367
|
activationContexts = new Map();
|
|
368
|
+
activationCleanupTasks = new Map();
|
|
349
369
|
pendingAcks = new Map();
|
|
350
370
|
pendingFileReports = new Map();
|
|
351
371
|
inflightCommands = new Set();
|
|
@@ -438,7 +458,7 @@ export class AgentServiceSandboxClient {
|
|
|
438
458
|
session.nativeSessionId = nativeSessionId.trim() || null;
|
|
439
459
|
this.persist();
|
|
440
460
|
}
|
|
441
|
-
finishTurn(payload) {
|
|
461
|
+
async finishTurn(payload) {
|
|
442
462
|
const scope = this.turnScopes.get(payload.agent_run_id);
|
|
443
463
|
if (!scope)
|
|
444
464
|
return;
|
|
@@ -454,7 +474,7 @@ export class AgentServiceSandboxClient {
|
|
|
454
474
|
};
|
|
455
475
|
this.persist();
|
|
456
476
|
try {
|
|
457
|
-
this.completePendingActivationCleanup(scope.sessionId);
|
|
477
|
+
await this.completePendingActivationCleanup(scope.sessionId);
|
|
458
478
|
}
|
|
459
479
|
catch (error) {
|
|
460
480
|
this.log.error("failed to revoke a terminal runtime session workspace", {
|
|
@@ -472,19 +492,31 @@ export class AgentServiceSandboxClient {
|
|
|
472
492
|
this.currentTurnSessionId = null;
|
|
473
493
|
this.persist();
|
|
474
494
|
}
|
|
475
|
-
completePendingActivationCleanup(sessionId) {
|
|
495
|
+
async completePendingActivationCleanup(sessionId) {
|
|
496
|
+
const existing = this.activationCleanupTasks.get(sessionId);
|
|
497
|
+
if (existing)
|
|
498
|
+
return await existing;
|
|
499
|
+
const task = this.runPendingActivationCleanup(sessionId);
|
|
500
|
+
this.activationCleanupTasks.set(sessionId, task);
|
|
501
|
+
try {
|
|
502
|
+
await task;
|
|
503
|
+
}
|
|
504
|
+
finally {
|
|
505
|
+
if (this.activationCleanupTasks.get(sessionId) === task) {
|
|
506
|
+
this.activationCleanupTasks.delete(sessionId);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
async runPendingActivationCleanup(sessionId) {
|
|
476
511
|
const session = this.state.sessions[sessionId];
|
|
477
512
|
const pending = session?.pendingActivationCleanup;
|
|
478
513
|
if (!session || !pending)
|
|
479
514
|
return;
|
|
480
|
-
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
481
515
|
const activation = this.activationContexts.get(sessionId);
|
|
482
516
|
if (activation?.activationId === pending.activationId) {
|
|
483
517
|
this.activationContexts.delete(sessionId);
|
|
484
518
|
this.sessionProfiles.delete(sessionId);
|
|
485
519
|
}
|
|
486
|
-
if (session.activationId === pending.activationId)
|
|
487
|
-
session.activationId = null;
|
|
488
520
|
if (!session.completedActivations.includes(pending.activationId)) {
|
|
489
521
|
// Session-close/generation-rotate are the retention bounds. Never evict an
|
|
490
522
|
// activation tombstone while the session is alive: activation IDs are opaque, so
|
|
@@ -499,15 +531,13 @@ export class AgentServiceSandboxClient {
|
|
|
499
531
|
delete session.acceptedCommands[commandId];
|
|
500
532
|
session.pendingActivationCleanup = null;
|
|
501
533
|
this.turnScopes.delete(pending.agentRunId);
|
|
502
|
-
if (this.activeSessionId === sessionId)
|
|
503
|
-
this.activeSessionId = null;
|
|
504
534
|
if (this.currentTurnSessionId === sessionId)
|
|
505
535
|
this.currentTurnSessionId = null;
|
|
506
536
|
this.persist();
|
|
507
537
|
}
|
|
508
|
-
recoverPendingActivationCleanups() {
|
|
538
|
+
async recoverPendingActivationCleanups() {
|
|
509
539
|
for (const sessionId of Object.keys(this.state.sessions)) {
|
|
510
|
-
this.completePendingActivationCleanup(sessionId);
|
|
540
|
+
await this.completePendingActivationCleanup(sessionId);
|
|
511
541
|
}
|
|
512
542
|
}
|
|
513
543
|
async run() {
|
|
@@ -568,6 +598,11 @@ export class AgentServiceSandboxClient {
|
|
|
568
598
|
this.removeOperationalLogSink();
|
|
569
599
|
this.socket?.close(1000, "daemon_stopping");
|
|
570
600
|
}
|
|
601
|
+
async stopGracefully() {
|
|
602
|
+
this.dispatcher.cancelAll();
|
|
603
|
+
await quiesceRuntimeWriters(null);
|
|
604
|
+
this.stop();
|
|
605
|
+
}
|
|
571
606
|
async postEvent(agentRunId, event) {
|
|
572
607
|
const scope = this.turnScopes.get(agentRunId);
|
|
573
608
|
if (!scope)
|
|
@@ -578,6 +613,8 @@ export class AgentServiceSandboxClient {
|
|
|
578
613
|
if (!this.sandboxGeneration || !this.connectionEpoch) {
|
|
579
614
|
throw new Error("Agent Service sandbox is not authenticated");
|
|
580
615
|
}
|
|
616
|
+
// Terminal truth is never delayed by object storage. Durable workspace checkpointing
|
|
617
|
+
// is a separate lifecycle command issued only for pause/cleanup/rotation.
|
|
581
618
|
if (event.type === "run.block") {
|
|
582
619
|
await this.waitForEventCapacity();
|
|
583
620
|
}
|
|
@@ -782,6 +819,7 @@ export class AgentServiceSandboxClient {
|
|
|
782
819
|
if (frame.sandbox_generation !== this.state.sandboxGeneration) {
|
|
783
820
|
// Generation change wipes every session shard (contract §1.4).
|
|
784
821
|
this.dispatcher.cancelAll();
|
|
822
|
+
await quiesceRuntimeWriters(null);
|
|
785
823
|
this.rejectPending(new Error("Agent Service sandbox generation changed"));
|
|
786
824
|
for (const sessionId of Object.keys(this.state.sessions)) {
|
|
787
825
|
revokeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
|
|
@@ -800,7 +838,7 @@ export class AgentServiceSandboxClient {
|
|
|
800
838
|
this.connectionEpoch = frame.connection_epoch;
|
|
801
839
|
this.inboundSeq = frame.seq;
|
|
802
840
|
this.outboundSeq = frame.connection_epoch * SEQ_EPOCH_BASE;
|
|
803
|
-
this.recoverPendingActivationCleanups();
|
|
841
|
+
await this.recoverPendingActivationCleanups();
|
|
804
842
|
const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
|
|
805
843
|
this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
|
|
806
844
|
const staleSeconds = Number(frame.payload.stale_after_seconds ?? 45);
|
|
@@ -815,12 +853,18 @@ export class AgentServiceSandboxClient {
|
|
|
815
853
|
this.runtimeLogPolicy = runtimeLogPolicy(frame.payload.runtime_log_policy);
|
|
816
854
|
await cleanupRuntimeSessionWorkspaceCopyStaging();
|
|
817
855
|
this.persist();
|
|
856
|
+
assertRuntimeWorkspaceQuota(process.env.BOTLEARN_AGENT_SERVICE_WORKSPACE_ROOT ?? "");
|
|
857
|
+
assertWorkspaceWriterFreezeSupported();
|
|
818
858
|
await this.sendControlFrame("sandbox.ready", {
|
|
819
859
|
protocol_versions: [AGENT_SERVICE_WS_SCHEMA],
|
|
820
860
|
capabilities: [
|
|
821
861
|
WORKSPACE_FILE_READ_BINARY_CAPABILITY,
|
|
862
|
+
"workspace_writer_freeze_v1",
|
|
863
|
+
"workspace_snapshot_v2",
|
|
864
|
+
"workspace_restore_v1",
|
|
822
865
|
...(workspaceCopyV1Supported() ? ["workspace_copy_v1"] : []),
|
|
823
866
|
],
|
|
867
|
+
workspace_snapshot_policy: workspaceSnapshotPolicyCapability(),
|
|
824
868
|
daemon_version: this.options.daemonVersion,
|
|
825
869
|
runtime_versions: {
|
|
826
870
|
course_daemon: this.options.daemonVersion,
|
|
@@ -952,8 +996,15 @@ export class AgentServiceSandboxClient {
|
|
|
952
996
|
});
|
|
953
997
|
});
|
|
954
998
|
return;
|
|
999
|
+
case "workspace.checkpoint":
|
|
1000
|
+
this.scheduleLifecycle("workspace.checkpoint", () => this.handleWorkspaceCheckpoint(frame));
|
|
1001
|
+
return;
|
|
1002
|
+
case "workspace.thaw":
|
|
1003
|
+
await thawRuntimeWriters();
|
|
1004
|
+
await this.sendCommandAck(frame, "ok");
|
|
1005
|
+
return;
|
|
955
1006
|
case "sandbox.shutdown":
|
|
956
|
-
this.
|
|
1007
|
+
await this.stopGracefully();
|
|
957
1008
|
return;
|
|
958
1009
|
case "sandbox.drain":
|
|
959
1010
|
await this.sendCommandAck(frame, "ok");
|
|
@@ -979,8 +1030,11 @@ export class AgentServiceSandboxClient {
|
|
|
979
1030
|
this.scheduleLifecycle("sandbox.sync.close", (fence) => this.closeSession(sessionId, "sync_close", frame, fence));
|
|
980
1031
|
}
|
|
981
1032
|
const state = frame.payload.state;
|
|
1033
|
+
if (frame.payload.thaw_runtime_writers === true) {
|
|
1034
|
+
await thawRuntimeWriters();
|
|
1035
|
+
}
|
|
982
1036
|
if (state === "shutdown") {
|
|
983
|
-
this.
|
|
1037
|
+
await this.stopGracefully();
|
|
984
1038
|
return;
|
|
985
1039
|
}
|
|
986
1040
|
if (state === "drain") {
|
|
@@ -998,6 +1052,7 @@ export class AgentServiceSandboxClient {
|
|
|
998
1052
|
const courseRunId = frame.payload.course_run_id;
|
|
999
1053
|
const requestedWorkspaceRef = frame.payload.workspace_ref;
|
|
1000
1054
|
const requestedCopy = frame.payload.workspace_copy;
|
|
1055
|
+
const requestedRestore = frame.payload.workspace_restore;
|
|
1001
1056
|
const requestedCopyRecord = requestedCopy && typeof requestedCopy === "object"
|
|
1002
1057
|
? requestedCopy
|
|
1003
1058
|
: undefined;
|
|
@@ -1018,7 +1073,11 @@ export class AgentServiceSandboxClient {
|
|
|
1018
1073
|
typeof copyId !== "string" ||
|
|
1019
1074
|
copyId.length < 1 ||
|
|
1020
1075
|
typeof requestedCopySource !== "string" ||
|
|
1021
|
-
requestedCopySource.length < 1))
|
|
1076
|
+
requestedCopySource.length < 1))
|
|
1077
|
+
|| (requestedRestore !== null && requestedRestore !== undefined &&
|
|
1078
|
+
(typeof requestedRestore !== "object" || Array.isArray(requestedRestore)))
|
|
1079
|
+
|| (requestedCopy !== null && requestedCopy !== undefined &&
|
|
1080
|
+
requestedRestore !== null && requestedRestore !== undefined)) {
|
|
1022
1081
|
throw new Error("invalid session.open payload");
|
|
1023
1082
|
}
|
|
1024
1083
|
if (session.courseRunId !== null && session.courseRunId !== courseRunId) {
|
|
@@ -1032,6 +1091,165 @@ export class AgentServiceSandboxClient {
|
|
|
1032
1091
|
const isNewSession = session.courseRunId === null;
|
|
1033
1092
|
let markerPath = null;
|
|
1034
1093
|
try {
|
|
1094
|
+
if (requestedRestore && typeof requestedRestore === "object") {
|
|
1095
|
+
const raw = requestedRestore;
|
|
1096
|
+
const descriptorRaw = raw.descriptor;
|
|
1097
|
+
const controlFence = raw.control_fence;
|
|
1098
|
+
let restoreControlToken = raw.control_token;
|
|
1099
|
+
if (!controlFence || typeof controlFence !== "object" || Array.isArray(controlFence)) {
|
|
1100
|
+
throw new Error("invalid workspace restore control fence");
|
|
1101
|
+
}
|
|
1102
|
+
if (typeof restoreControlToken !== "string" || restoreControlToken.length < 32) {
|
|
1103
|
+
throw new Error("invalid workspace restore control token");
|
|
1104
|
+
}
|
|
1105
|
+
if (!descriptorRaw || !Array.isArray(raw.entries)) {
|
|
1106
|
+
throw new Error("invalid workspace restore payload");
|
|
1107
|
+
}
|
|
1108
|
+
const descriptor = {
|
|
1109
|
+
sandboxId: this.options.sandboxId,
|
|
1110
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1111
|
+
runtimeSessionId: sessionId,
|
|
1112
|
+
continuityState: descriptorRaw.continuity_state,
|
|
1113
|
+
continuityErrorCode: descriptorRaw.continuity_error_code,
|
|
1114
|
+
snapshotId: descriptorRaw.snapshot_id,
|
|
1115
|
+
revision: Number(descriptorRaw.revision),
|
|
1116
|
+
contentSha256: descriptorRaw.content_sha256,
|
|
1117
|
+
expectedFileCount: Number(descriptorRaw.expected_file_count),
|
|
1118
|
+
expectedEntryCount: Number(descriptorRaw.expected_entry_count),
|
|
1119
|
+
expectedTotalBytes: Number(descriptorRaw.expected_total_bytes),
|
|
1120
|
+
};
|
|
1121
|
+
const workspaceDirectory = runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration);
|
|
1122
|
+
const requiredFreeBytes = Math.max(WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes, descriptor.expectedTotalBytes + WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes);
|
|
1123
|
+
// Fail before issuing or consuming any object grant. The download staging is a
|
|
1124
|
+
// sibling on this same bounded backing filesystem and is renamed, not copied,
|
|
1125
|
+
// into the atomic restore staging tree.
|
|
1126
|
+
assertWorkspaceRestoreCapacity({
|
|
1127
|
+
workspaceDirectory,
|
|
1128
|
+
descriptor,
|
|
1129
|
+
requiredFreeBytes,
|
|
1130
|
+
entryCopies: 2,
|
|
1131
|
+
});
|
|
1132
|
+
const entries = [];
|
|
1133
|
+
const predownloadRoot = path.join(path.dirname(workspaceDirectory), `.botlearn-predownload-${sessionId}-${randomUUID()}`);
|
|
1134
|
+
mkdirSync(predownloadRoot, { recursive: true, mode: 0o700 });
|
|
1135
|
+
try {
|
|
1136
|
+
let nextCursor = null;
|
|
1137
|
+
do {
|
|
1138
|
+
if (nextCursor !== null && nextCursor.length > 2048) {
|
|
1139
|
+
throw new Error("invalid workspace restore cursor");
|
|
1140
|
+
}
|
|
1141
|
+
const pageSignal = AbortSignal.timeout(60_000);
|
|
1142
|
+
const response = await fetch(`${agentServiceHttpBase(this.options.wsUrl)}/runtime-sessions/${sessionId}/restore-grants`, {
|
|
1143
|
+
method: "POST",
|
|
1144
|
+
headers: {
|
|
1145
|
+
Authorization: `Bearer ${restoreControlToken}`,
|
|
1146
|
+
"Content-Type": "application/json",
|
|
1147
|
+
},
|
|
1148
|
+
body: JSON.stringify({
|
|
1149
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
1150
|
+
fence: controlFence,
|
|
1151
|
+
cursor: nextCursor,
|
|
1152
|
+
// Fetch one canonical entry at a time so a grant is consumed as soon as
|
|
1153
|
+
// it is signed, including deployments using the minimum 60-second TTL.
|
|
1154
|
+
max_entries: 1,
|
|
1155
|
+
}),
|
|
1156
|
+
signal: pageSignal,
|
|
1157
|
+
});
|
|
1158
|
+
if (!response.ok) {
|
|
1159
|
+
let code = "workspace_restore_unavailable";
|
|
1160
|
+
let retryable = response.status >= 500 || response.status === 408 ||
|
|
1161
|
+
response.status === 429;
|
|
1162
|
+
try {
|
|
1163
|
+
const rejected = await response.json();
|
|
1164
|
+
const detail = rejected.detail;
|
|
1165
|
+
if (detail && typeof detail === "object" && !Array.isArray(detail)) {
|
|
1166
|
+
const stableCode = detail.code;
|
|
1167
|
+
if (typeof stableCode === "string" && stableCode.length <= 120) {
|
|
1168
|
+
code = stableCode;
|
|
1169
|
+
}
|
|
1170
|
+
if (typeof detail.retryable === "boolean") {
|
|
1171
|
+
retryable = detail.retryable;
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
catch {
|
|
1176
|
+
// Content-free fallback retains the stable local failure code.
|
|
1177
|
+
}
|
|
1178
|
+
throw new WorkspaceRestoreError(code, retryable);
|
|
1179
|
+
}
|
|
1180
|
+
const page = await response.json();
|
|
1181
|
+
if (!Array.isArray(page.entries))
|
|
1182
|
+
throw new Error("invalid workspace restore page");
|
|
1183
|
+
if (typeof page.control_token !== "string" || page.control_token.length < 32) {
|
|
1184
|
+
throw new Error("invalid refreshed workspace restore control token");
|
|
1185
|
+
}
|
|
1186
|
+
restoreControlToken = page.control_token;
|
|
1187
|
+
const pageFiles = [];
|
|
1188
|
+
for (const value of page.entries) {
|
|
1189
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1190
|
+
throw new Error("invalid workspace restore entry");
|
|
1191
|
+
}
|
|
1192
|
+
const item = value;
|
|
1193
|
+
if (item.entry_type === "directory") {
|
|
1194
|
+
entries.push({ type: "directory", path: String(item.path), mode: "0700" });
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
if (item.entry_type !== "file" || !item.grant || typeof item.grant !== "object") {
|
|
1198
|
+
throw new Error("invalid workspace restore file grant");
|
|
1199
|
+
}
|
|
1200
|
+
const entry = {
|
|
1201
|
+
type: "file",
|
|
1202
|
+
path: String(item.path),
|
|
1203
|
+
mode: item.mode,
|
|
1204
|
+
sizeBytes: Number(item.size_bytes),
|
|
1205
|
+
sha256: String(item.sha256),
|
|
1206
|
+
};
|
|
1207
|
+
entries.push(entry);
|
|
1208
|
+
pageFiles.push({ entry, grant: item.grant });
|
|
1209
|
+
}
|
|
1210
|
+
// Validate all paths and duplicates before using any page path locally.
|
|
1211
|
+
validateWorkspaceEntrySet({ schemaVersion: "agent-workspace-entry-set/1", entries }, WORKSPACE_SNAPSHOT_POLICY_V1.limits);
|
|
1212
|
+
for (const { entry, grant } of pageFiles) {
|
|
1213
|
+
if (typeof grant.url !== "string")
|
|
1214
|
+
throw new Error("workspace restore grant missing");
|
|
1215
|
+
const destination = path.join(predownloadRoot, ...entry.path.split("/"));
|
|
1216
|
+
mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
|
|
1217
|
+
const part = `${destination}.part`;
|
|
1218
|
+
const downloadSignal = AbortSignal.timeout(60_000);
|
|
1219
|
+
const download = await fetch(grant.url, {
|
|
1220
|
+
method: "GET",
|
|
1221
|
+
headers: grant.headers,
|
|
1222
|
+
signal: downloadSignal,
|
|
1223
|
+
});
|
|
1224
|
+
if (!download.ok)
|
|
1225
|
+
throw new Error(`workspace_restore_download_${download.status}`);
|
|
1226
|
+
if (!download.body)
|
|
1227
|
+
throw new Error("workspace_restore_download_body_missing");
|
|
1228
|
+
await pipeline(Readable.fromWeb(download.body), createWriteStream(part, { mode: 0o600 }), { signal: downloadSignal });
|
|
1229
|
+
renameSync(part, destination);
|
|
1230
|
+
}
|
|
1231
|
+
const rawNext = page.next_cursor;
|
|
1232
|
+
if (rawNext !== null && rawNext !== undefined && typeof rawNext !== "string") {
|
|
1233
|
+
throw new Error("invalid workspace restore cursor");
|
|
1234
|
+
}
|
|
1235
|
+
nextCursor = rawNext ?? null;
|
|
1236
|
+
} while (nextCursor !== null);
|
|
1237
|
+
await restoreWorkspaceSnapshot({
|
|
1238
|
+
workspaceDirectory,
|
|
1239
|
+
controlDirectory: path.join(ensureDaemonHome(), "agent-service-sandboxes", this.options.sandboxId, "workspace-control"),
|
|
1240
|
+
descriptor,
|
|
1241
|
+
entrySet: { schemaVersion: "agent-workspace-entry-set/1", entries },
|
|
1242
|
+
limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
|
|
1243
|
+
requiredFreeBytes,
|
|
1244
|
+
downloadFile: async (entry, partPath) => {
|
|
1245
|
+
renameSync(path.join(predownloadRoot, ...entry.path.split("/")), partPath);
|
|
1246
|
+
},
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
finally {
|
|
1250
|
+
rmSync(predownloadRoot, { recursive: true, force: true });
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1035
1253
|
if (typeof requestedCopySource === "string" && typeof copyId === "string") {
|
|
1036
1254
|
if (session.workspaceCopyReceipt !== null) {
|
|
1037
1255
|
const receipt = session.workspaceCopyReceipt;
|
|
@@ -1075,28 +1293,159 @@ export class AgentServiceSandboxClient {
|
|
|
1075
1293
|
});
|
|
1076
1294
|
return;
|
|
1077
1295
|
}
|
|
1296
|
+
if (requestedRestore) {
|
|
1297
|
+
const restoreError = error instanceof WorkspaceRestoreError ? error : null;
|
|
1298
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1299
|
+
error_code: restoreError?.code ?? "workspace_restore_unavailable",
|
|
1300
|
+
retryable: restoreError?.retryable ?? true,
|
|
1301
|
+
});
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1078
1304
|
throw error;
|
|
1079
1305
|
}
|
|
1080
1306
|
session.courseRunId = courseRunId;
|
|
1081
1307
|
session.workspaceRef = session.workspaceRef ?? (typeof requestedWorkspaceRef === "string"
|
|
1082
1308
|
? requestedWorkspaceRef
|
|
1083
1309
|
: `ws_${sessionId.replaceAll("-", "")}_g${this.sandboxGeneration}`);
|
|
1084
|
-
|
|
1310
|
+
// Replaying session.open is part of reconnect reconciliation. An already-active
|
|
1311
|
+
// runtime may still be creating files in this workspace, so never transiently revoke
|
|
1312
|
+
// its group write/execute access before the following session.activate arrives.
|
|
1313
|
+
if (session.workspaceCopyReceipt === null && (isNewSession || session.activationId === null)) {
|
|
1085
1314
|
ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
|
|
1086
1315
|
}
|
|
1087
|
-
this.state.sessions[sessionId] = session;
|
|
1088
|
-
this.persist();
|
|
1089
1316
|
if (markerPath !== null) {
|
|
1090
1317
|
await finalizeRuntimeSessionWorkspaceCopy(markerPath);
|
|
1091
1318
|
}
|
|
1319
|
+
try {
|
|
1320
|
+
await applyRuntimeWorkspaceQuota(sessionId, this.sandboxGeneration);
|
|
1321
|
+
}
|
|
1322
|
+
catch (error) {
|
|
1323
|
+
if (requestedRestore) {
|
|
1324
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1325
|
+
error_code: "workspace_restore_unavailable",
|
|
1326
|
+
});
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
if (requestedCopy) {
|
|
1330
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1331
|
+
error_code: "workspace_migration_io_failed",
|
|
1332
|
+
});
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
throw error;
|
|
1336
|
+
}
|
|
1337
|
+
this.state.sessions[sessionId] = session;
|
|
1338
|
+
this.persist();
|
|
1339
|
+
const materialization = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
|
|
1092
1340
|
await this.sendSessionFrame("session.opened", sessionId, {
|
|
1093
1341
|
workspace_ref: session.workspaceRef,
|
|
1094
1342
|
native_session_id: session.nativeSessionId,
|
|
1343
|
+
...(materialization
|
|
1344
|
+
? {
|
|
1345
|
+
workspace_marker_version: 1,
|
|
1346
|
+
workspace_continuity_state: materialization.workspaceContinuityState,
|
|
1347
|
+
workspace_snapshot_id: materialization.snapshotId,
|
|
1348
|
+
workspace_revision: materialization.revision,
|
|
1349
|
+
workspace_content_sha256: materialization.contentSha256,
|
|
1350
|
+
}
|
|
1351
|
+
: {}),
|
|
1095
1352
|
...(session.workspaceCopyReceipt !== null
|
|
1096
1353
|
? { workspace_copy_receipt: session.workspaceCopyReceipt }
|
|
1097
1354
|
: {}),
|
|
1098
1355
|
});
|
|
1099
1356
|
}
|
|
1357
|
+
async handleWorkspaceCheckpoint(frame) {
|
|
1358
|
+
const sessionId = frame.runtime_session_id;
|
|
1359
|
+
const session = this.state.sessions[sessionId];
|
|
1360
|
+
const checkpointId = frame.payload.checkpoint_id;
|
|
1361
|
+
const controlToken = frame.payload.control_token;
|
|
1362
|
+
const baseRevision = Number(frame.payload.base_revision);
|
|
1363
|
+
const sourceKind = frame.payload.source_kind;
|
|
1364
|
+
if (!session ||
|
|
1365
|
+
this.currentTurnSessionId === sessionId ||
|
|
1366
|
+
typeof checkpointId !== "string" ||
|
|
1367
|
+
typeof controlToken !== "string" ||
|
|
1368
|
+
controlToken.length < 32 ||
|
|
1369
|
+
frame.payload.policy_id !== "WorkspaceSnapshotPolicyV1" ||
|
|
1370
|
+
![
|
|
1371
|
+
"sandbox_pause",
|
|
1372
|
+
"sandbox_cleanup",
|
|
1373
|
+
"generation_rotation",
|
|
1374
|
+
"initial_migration",
|
|
1375
|
+
"live_workspace_copy",
|
|
1376
|
+
"maintenance_repair",
|
|
1377
|
+
].includes(String(sourceKind)) ||
|
|
1378
|
+
!Number.isSafeInteger(baseRevision) ||
|
|
1379
|
+
baseRevision < 0 ||
|
|
1380
|
+
frame.payload.agent_run_id != null ||
|
|
1381
|
+
frame.payload.worker_attempt != null ||
|
|
1382
|
+
frame.payload.activation_id != null) {
|
|
1383
|
+
await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
|
|
1384
|
+
return;
|
|
1385
|
+
}
|
|
1386
|
+
const lifecycleCheckpoint = [
|
|
1387
|
+
"sandbox_pause",
|
|
1388
|
+
"sandbox_cleanup",
|
|
1389
|
+
"generation_rotation",
|
|
1390
|
+
].includes(String(sourceKind));
|
|
1391
|
+
let checkpointCommitted = false;
|
|
1392
|
+
try {
|
|
1393
|
+
await freezeRuntimeWriters();
|
|
1394
|
+
const freezeProof = issueWorkspaceWriterFreezeProof({
|
|
1395
|
+
sandboxId: this.options.sandboxId,
|
|
1396
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1397
|
+
runtimeSessionId: sessionId,
|
|
1398
|
+
checkpointId,
|
|
1399
|
+
});
|
|
1400
|
+
await checkpointWorkspace({
|
|
1401
|
+
wsUrl: this.options.wsUrl,
|
|
1402
|
+
reconnectToken: controlToken,
|
|
1403
|
+
workspaceDirectory: runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration),
|
|
1404
|
+
scope: {
|
|
1405
|
+
checkpointId,
|
|
1406
|
+
baseRevision,
|
|
1407
|
+
sandboxId: this.options.sandboxId,
|
|
1408
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1409
|
+
connectionEpoch: this.connectionEpoch,
|
|
1410
|
+
runtimeSessionId: sessionId,
|
|
1411
|
+
},
|
|
1412
|
+
freezeProof,
|
|
1413
|
+
});
|
|
1414
|
+
checkpointCommitted = true;
|
|
1415
|
+
const marker = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
|
|
1416
|
+
if (!marker)
|
|
1417
|
+
throw new Error("workspace checkpoint marker missing");
|
|
1418
|
+
await this.sendSessionFrame("session.opened", sessionId, {
|
|
1419
|
+
workspace_ref: session.workspaceRef,
|
|
1420
|
+
native_session_id: session.nativeSessionId,
|
|
1421
|
+
workspace_marker_version: 1,
|
|
1422
|
+
workspace_continuity_state: marker.workspaceContinuityState,
|
|
1423
|
+
workspace_snapshot_id: marker.snapshotId,
|
|
1424
|
+
workspace_revision: marker.revision,
|
|
1425
|
+
workspace_content_sha256: marker.contentSha256,
|
|
1426
|
+
});
|
|
1427
|
+
await this.sendCommandAck(frame, "ok");
|
|
1428
|
+
if (!lifecycleCheckpoint)
|
|
1429
|
+
await thawRuntimeWriters();
|
|
1430
|
+
}
|
|
1431
|
+
catch (error) {
|
|
1432
|
+
if (!lifecycleCheckpoint || !checkpointCommitted) {
|
|
1433
|
+
await thawRuntimeWriters().catch(() => undefined);
|
|
1434
|
+
}
|
|
1435
|
+
this.log.error("workspace baseline checkpoint failed", {
|
|
1436
|
+
sandboxId: this.options.sandboxId,
|
|
1437
|
+
runtimeSessionId: sessionId,
|
|
1438
|
+
error: error instanceof Error ? redactSecretString(error.message) : "unexpected",
|
|
1439
|
+
});
|
|
1440
|
+
const deterministic = (error instanceof WorkspaceSnapshotStagingError && !error.retryable) ||
|
|
1441
|
+
error instanceof WorkspaceEntrySetError ||
|
|
1442
|
+
(error instanceof WorkspaceSnapshotControlError && !error.retryable);
|
|
1443
|
+
const errorCode = deterministic && "code" in error
|
|
1444
|
+
? String(error.code)
|
|
1445
|
+
: "workspace_checkpoint_retryable";
|
|
1446
|
+
await this.sendCommandAck(frame, "rejected", errorCode, !deterministic);
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1100
1449
|
async handleSessionActivate(frame, fence) {
|
|
1101
1450
|
const sessionId = frame.runtime_session_id;
|
|
1102
1451
|
const activationId = frame.activation_id;
|
|
@@ -1233,7 +1582,7 @@ export class AgentServiceSandboxClient {
|
|
|
1233
1582
|
const previousSessionId = this.activeSessionId;
|
|
1234
1583
|
const changesActivation = previousSessionId !== null && (previousSessionId !== sessionId || session.activationId !== activationId);
|
|
1235
1584
|
if (changesActivation) {
|
|
1236
|
-
const stopped = await this.deactivateSession(previousSessionId);
|
|
1585
|
+
const stopped = await this.deactivateSession(previousSessionId, previousSessionId !== sessionId);
|
|
1237
1586
|
if (fence && !this.isCurrentLifecycleFence(fence))
|
|
1238
1587
|
return;
|
|
1239
1588
|
if (!stopped) {
|
|
@@ -1419,7 +1768,7 @@ export class AgentServiceSandboxClient {
|
|
|
1419
1768
|
}
|
|
1420
1769
|
delete session.acceptedCommands[commandId];
|
|
1421
1770
|
this.persist();
|
|
1422
|
-
this.finishTurn({ agent_run_id: runId });
|
|
1771
|
+
await this.finishTurn({ agent_run_id: runId });
|
|
1423
1772
|
}
|
|
1424
1773
|
/**
|
|
1425
1774
|
* 幂等关闭一个 runtime session:终止其 runtime 子进程、删除 workspace/native state、
|
|
@@ -1429,15 +1778,16 @@ export class AgentServiceSandboxClient {
|
|
|
1429
1778
|
const session = this.state.sessions[sessionId];
|
|
1430
1779
|
if (session) {
|
|
1431
1780
|
const runIds = this.runIdsForSession(sessionId);
|
|
1432
|
-
session.activationId = null;
|
|
1433
1781
|
this.activationContexts.delete(sessionId);
|
|
1434
1782
|
this.sessionProfiles.delete(sessionId);
|
|
1435
|
-
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1436
1783
|
for (const runId of runIds)
|
|
1437
1784
|
this.dispatcher.cancel(runId);
|
|
1438
1785
|
if (!await this.dispatcher.waitForRuns(runIds, 10_000)) {
|
|
1439
1786
|
throw new Error(`runtime session ${sessionId} did not stop before close`);
|
|
1440
1787
|
}
|
|
1788
|
+
await this.terminateSessionWriters(session);
|
|
1789
|
+
session.activationId = null;
|
|
1790
|
+
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1441
1791
|
if (fence && !this.isCurrentLifecycleFence(fence))
|
|
1442
1792
|
return;
|
|
1443
1793
|
if (this.activeSessionId === sessionId)
|
|
@@ -1478,7 +1828,7 @@ export class AgentServiceSandboxClient {
|
|
|
1478
1828
|
delete session.acceptedCommands[commandId];
|
|
1479
1829
|
this.persist();
|
|
1480
1830
|
}
|
|
1481
|
-
this.finishTurn({ agent_run_id: accepted.agentRunId });
|
|
1831
|
+
await this.finishTurn({ agent_run_id: accepted.agentRunId });
|
|
1482
1832
|
}
|
|
1483
1833
|
catch (error) {
|
|
1484
1834
|
this.log.error("failed to reconcile an interrupted accepted turn", {
|
|
@@ -1580,21 +1930,40 @@ export class AgentServiceSandboxClient {
|
|
|
1580
1930
|
.filter(([, scope]) => scope.sessionId === sessionId)
|
|
1581
1931
|
.map(([runId]) => runId);
|
|
1582
1932
|
}
|
|
1583
|
-
async deactivateSession(sessionId) {
|
|
1933
|
+
async deactivateSession(sessionId, terminateWriters = false) {
|
|
1584
1934
|
const session = this.state.sessions[sessionId];
|
|
1585
1935
|
if (!session)
|
|
1586
1936
|
return true;
|
|
1587
1937
|
const runIds = this.runIdsForSession(sessionId);
|
|
1588
|
-
session.activationId = null;
|
|
1589
1938
|
this.activationContexts.delete(sessionId);
|
|
1590
1939
|
this.sessionProfiles.delete(sessionId);
|
|
1591
|
-
|
|
1940
|
+
if (terminateWriters) {
|
|
1941
|
+
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1942
|
+
}
|
|
1592
1943
|
if (this.activeSessionId === sessionId)
|
|
1593
1944
|
this.activeSessionId = null;
|
|
1594
1945
|
for (const runId of runIds)
|
|
1595
1946
|
this.dispatcher.cancel(runId);
|
|
1596
1947
|
this.persist();
|
|
1597
|
-
|
|
1948
|
+
const stopped = await this.dispatcher.waitForRuns(runIds, 10_000);
|
|
1949
|
+
if (stopped && terminateWriters) {
|
|
1950
|
+
await this.terminateSessionWriters(session);
|
|
1951
|
+
}
|
|
1952
|
+
session.activationId = null;
|
|
1953
|
+
this.persist();
|
|
1954
|
+
return stopped;
|
|
1955
|
+
}
|
|
1956
|
+
async terminateSessionWriters(session) {
|
|
1957
|
+
const activationIds = new Set([
|
|
1958
|
+
...session.completedActivations,
|
|
1959
|
+
...(session.activationId ? [session.activationId] : []),
|
|
1960
|
+
...(session.pendingActivationCleanup?.activationId
|
|
1961
|
+
? [session.pendingActivationCleanup.activationId]
|
|
1962
|
+
: []),
|
|
1963
|
+
]);
|
|
1964
|
+
for (const activationId of activationIds) {
|
|
1965
|
+
await quiesceRuntimeWriters(activationId);
|
|
1966
|
+
}
|
|
1598
1967
|
}
|
|
1599
1968
|
sameTurnScope(left, right) {
|
|
1600
1969
|
return left.sessionId === right.sessionId &&
|
|
@@ -1687,7 +2056,7 @@ export class AgentServiceSandboxClient {
|
|
|
1687
2056
|
spool_frames: this.spoolFrameCount(),
|
|
1688
2057
|
});
|
|
1689
2058
|
}
|
|
1690
|
-
async sendCommandAck(frame, status, error) {
|
|
2059
|
+
async sendCommandAck(frame, status, error, retryable) {
|
|
1691
2060
|
if (frame.sandbox_generation !== this.sandboxGeneration ||
|
|
1692
2061
|
frame.connection_epoch !== this.connectionEpoch)
|
|
1693
2062
|
return;
|
|
@@ -1695,6 +2064,7 @@ export class AgentServiceSandboxClient {
|
|
|
1695
2064
|
command_frame_id: frame.frame_id,
|
|
1696
2065
|
status,
|
|
1697
2066
|
...(error !== undefined ? { error } : {}),
|
|
2067
|
+
...(retryable !== undefined ? { retryable } : {}),
|
|
1698
2068
|
});
|
|
1699
2069
|
}
|
|
1700
2070
|
async handleWorkspaceFileRead(frame) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export declare const AGENT_SERVICE_WS_SCHEMA: "botlearn-agent-sandbox-ws/0.
|
|
2
|
-
export declare const AGENT_SERVICE_WS_SUBPROTOCOL: "botlearn-agent-sandbox.
|
|
1
|
+
export declare const AGENT_SERVICE_WS_SCHEMA: "botlearn-agent-sandbox-ws/0.4";
|
|
2
|
+
export declare const AGENT_SERVICE_WS_SUBPROTOCOL: "botlearn-agent-sandbox.v4";
|
|
3
3
|
export declare const WORKSPACE_FILE_READ_BINARY_CAPABILITY: "workspace_file_read_binary_v1";
|
|
4
4
|
export declare const WORKSPACE_FILE_CHUNK_BYTES: number;
|
|
5
5
|
export declare const WORKSPACE_FILE_CHUNK_HEADER_BYTES = 29;
|
|
6
|
-
export type SandboxFrameType = "sandbox.hello" | "sandbox.sync" | "session.open" | "session.activate" | "turn.start" | "turn.cancel" | "session.close" | "sandbox.drain" | "sandbox.shutdown" | "event.ack" | "auth.rotate" | "workspace.file.read" | "ping" | "sandbox.ready" | "sandbox.heartbeat" | "command.ack" | "session.opened" | "session.open_failed" | "session.closed" | "turn.event" | "turn.file.report" | "workspace.file.result" | "sandbox.drained" | "sandbox.log" | "pong" | "protocol.error";
|
|
6
|
+
export type SandboxFrameType = "sandbox.hello" | "sandbox.sync" | "session.open" | "session.activate" | "turn.start" | "turn.cancel" | "session.close" | "sandbox.drain" | "sandbox.shutdown" | "event.ack" | "auth.rotate" | "workspace.file.read" | "workspace.checkpoint" | "workspace.thaw" | "ping" | "sandbox.ready" | "sandbox.heartbeat" | "command.ack" | "session.opened" | "session.open_failed" | "session.closed" | "turn.event" | "turn.file.report" | "workspace.file.result" | "sandbox.drained" | "sandbox.log" | "pong" | "protocol.error";
|
|
7
7
|
export declare class UnsupportedSandboxProtocolError extends Error {
|
|
8
8
|
readonly schemaVersion: unknown;
|
|
9
9
|
constructor(schemaVersion: unknown);
|