@botlearn-course/daemon 0.0.20-beta.1 → 0.0.20-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/agent-service-sandbox.d.ts +7 -1
- package/dist/agent-service-sandbox.js +417 -39
- package/dist/agent-service-ws-protocol.d.ts +3 -3
- package/dist/agent-service-ws-protocol.js +5 -2
- package/dist/cli.js +19 -1
- package/dist/file-candidates.js +46 -39
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/run-dispatcher.d.ts +1 -1
- package/dist/run-dispatcher.js +10 -7
- package/dist/runtime-env.js +4 -4
- package/dist/runtime-quiescence.d.ts +19 -0
- package/dist/runtime-quiescence.js +130 -0
- package/dist/runtimes/deepseek-tui.js +11 -0
- package/dist/tool-observation.d.ts +6 -5
- package/dist/tool-observation.js +6 -5
- package/dist/trace-projection.d.ts +1 -1
- package/dist/trace-projection.js +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/workspace-filesystem.d.ts +20 -0
- package/dist/workspace-filesystem.js +120 -0
- package/dist/workspace-quota.d.ts +4 -0
- package/dist/workspace-quota.js +42 -0
- package/dist/workspace-restore.d.ts +9 -1
- package/dist/workspace-restore.js +30 -16
- package/dist/workspace-snapshot-control.d.ts +29 -0
- package/dist/workspace-snapshot-control.js +169 -0
- package/dist/workspace-snapshot-policy.d.ts +24 -0
- package/dist/workspace-snapshot-policy.js +45 -0
- package/dist/workspace-snapshot-staging.d.ts +2 -9
- package/dist/workspace-snapshot-staging.js +59 -25
- package/dist/workspace.d.ts +4 -4
- package/dist/workspace.js +26 -11
- package/package.json +1 -1
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, closeSync, createWriteStream, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import { Readable } from "node:stream";
|
|
4
|
+
import { pipeline } from "node:stream/promises";
|
|
2
5
|
import path from "node:path";
|
|
3
6
|
import { ensureDaemonHome } from "./auth-store.js";
|
|
4
7
|
import { AGENT_SERVICE_WS_SCHEMA, AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, createWorkspaceFileChunk, parseSandboxFrame, UnsupportedSandboxProtocolError, WORKSPACE_FILE_READ_BINARY_CAPABILITY, } from "./agent-service-ws-protocol.js";
|
|
@@ -10,6 +13,15 @@ import { parseRuntimeSkillProviderGrantSet, prepareRuntimeSkillProvider, Runtime
|
|
|
10
13
|
import { RunDispatcher, } from "./run-dispatcher.js";
|
|
11
14
|
import { cleanupRuntimeSessionWorkspaceCopyStaging, copyRuntimeSessionWorkspace, ensureRuntimeSessionDirectories, ensureRuntimeSessionWorkspace, exposeRuntimeSessionWorkspace, finalizeRuntimeSessionWorkspaceCopy, removeRuntimeSessionWorkspace, revokeRuntimeSessionWorkspace, runtimeSessionWorkspaceDir, WORKSPACE_COPY_POLICY_V1, WorkspaceCopyError, workspaceCopyV1Supported, } from "./workspace.js";
|
|
12
15
|
import { readWorkspaceFile, WorkspaceFileReadError } from "./workspace-file-read.js";
|
|
16
|
+
import { checkpointWorkspace, WorkspaceSnapshotControlError, } from "./workspace-snapshot-control.js";
|
|
17
|
+
import { assertWorkspaceWriterFreezeSupported, freezeRuntimeWriters, issueWorkspaceWriterFreezeProof, quiesceRuntimeWriters, thawRuntimeWriters, } from "./runtime-quiescence.js";
|
|
18
|
+
import { WorkspaceSnapshotStagingError } from "./workspace-snapshot-staging.js";
|
|
19
|
+
import { validateWorkspaceEntrySet, WorkspaceEntrySetError, } from "./workspace-entry-set.js";
|
|
20
|
+
import { WORKSPACE_SNAPSHOT_POLICY_V1, workspaceSnapshotPolicyCapability, } from "./workspace-snapshot-policy.js";
|
|
21
|
+
import { assertWorkspaceRestoreCapacity, restoreWorkspaceSnapshot, WorkspaceRestoreError, } from "./workspace-restore.js";
|
|
22
|
+
import { readWorkspaceMaterializationMarker } from "./workspace-materialization.js";
|
|
23
|
+
import { assertWorkspaceFilesystem, flushWorkspaceFilesystem, nasWorkspaceEnabled, } from "./workspace-filesystem.js";
|
|
24
|
+
import { applyRuntimeWorkspaceQuota } from "./workspace-quota.js";
|
|
13
25
|
import { WebSocketClient, } from "./websocket-client.js";
|
|
14
26
|
const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
|
|
15
27
|
const MAX_SPOOL_FRAMES = 1024;
|
|
@@ -19,6 +31,14 @@ const LEGACY_EVENT_ACK_WINDOW = 1;
|
|
|
19
31
|
const INPUT_ATTACHMENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
20
32
|
/** 出站 seq 基址:seq = connection_epoch * SEQ_EPOCH_BASE + n,跨重连单调(合同 §1.1)。 */
|
|
21
33
|
const SEQ_EPOCH_BASE = 1_000_000_000;
|
|
34
|
+
function agentServiceHttpBase(wsUrl) {
|
|
35
|
+
const url = new URL(wsUrl);
|
|
36
|
+
url.protocol = url.protocol === "wss:" ? "https:" : "http:";
|
|
37
|
+
url.pathname = "/internal/v1";
|
|
38
|
+
url.search = "";
|
|
39
|
+
url.hash = "";
|
|
40
|
+
return url.toString().replace(/\/$/, "");
|
|
41
|
+
}
|
|
22
42
|
const RUNTIME_LOG_WINDOW_MS = 60_000;
|
|
23
43
|
const WORKSPACE_FILE_TRANSFER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
24
44
|
const DISABLED_RUNTIME_LOG_POLICY = {
|
|
@@ -346,6 +366,7 @@ export class AgentServiceSandboxClient {
|
|
|
346
366
|
sessionProfiles = new Map();
|
|
347
367
|
/** activation-scoped 模型短凭据;只保存在内存中,绝不进入 state.json。 */
|
|
348
368
|
activationContexts = new Map();
|
|
369
|
+
activationCleanupTasks = new Map();
|
|
349
370
|
pendingAcks = new Map();
|
|
350
371
|
pendingFileReports = new Map();
|
|
351
372
|
inflightCommands = new Set();
|
|
@@ -438,7 +459,7 @@ export class AgentServiceSandboxClient {
|
|
|
438
459
|
session.nativeSessionId = nativeSessionId.trim() || null;
|
|
439
460
|
this.persist();
|
|
440
461
|
}
|
|
441
|
-
finishTurn(payload) {
|
|
462
|
+
async finishTurn(payload) {
|
|
442
463
|
const scope = this.turnScopes.get(payload.agent_run_id);
|
|
443
464
|
if (!scope)
|
|
444
465
|
return;
|
|
@@ -454,7 +475,7 @@ export class AgentServiceSandboxClient {
|
|
|
454
475
|
};
|
|
455
476
|
this.persist();
|
|
456
477
|
try {
|
|
457
|
-
this.completePendingActivationCleanup(scope.sessionId);
|
|
478
|
+
await this.completePendingActivationCleanup(scope.sessionId);
|
|
458
479
|
}
|
|
459
480
|
catch (error) {
|
|
460
481
|
this.log.error("failed to revoke a terminal runtime session workspace", {
|
|
@@ -472,19 +493,31 @@ export class AgentServiceSandboxClient {
|
|
|
472
493
|
this.currentTurnSessionId = null;
|
|
473
494
|
this.persist();
|
|
474
495
|
}
|
|
475
|
-
completePendingActivationCleanup(sessionId) {
|
|
496
|
+
async completePendingActivationCleanup(sessionId) {
|
|
497
|
+
const existing = this.activationCleanupTasks.get(sessionId);
|
|
498
|
+
if (existing)
|
|
499
|
+
return await existing;
|
|
500
|
+
const task = this.runPendingActivationCleanup(sessionId);
|
|
501
|
+
this.activationCleanupTasks.set(sessionId, task);
|
|
502
|
+
try {
|
|
503
|
+
await task;
|
|
504
|
+
}
|
|
505
|
+
finally {
|
|
506
|
+
if (this.activationCleanupTasks.get(sessionId) === task) {
|
|
507
|
+
this.activationCleanupTasks.delete(sessionId);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
async runPendingActivationCleanup(sessionId) {
|
|
476
512
|
const session = this.state.sessions[sessionId];
|
|
477
513
|
const pending = session?.pendingActivationCleanup;
|
|
478
514
|
if (!session || !pending)
|
|
479
515
|
return;
|
|
480
|
-
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
481
516
|
const activation = this.activationContexts.get(sessionId);
|
|
482
517
|
if (activation?.activationId === pending.activationId) {
|
|
483
518
|
this.activationContexts.delete(sessionId);
|
|
484
519
|
this.sessionProfiles.delete(sessionId);
|
|
485
520
|
}
|
|
486
|
-
if (session.activationId === pending.activationId)
|
|
487
|
-
session.activationId = null;
|
|
488
521
|
if (!session.completedActivations.includes(pending.activationId)) {
|
|
489
522
|
// Session-close/generation-rotate are the retention bounds. Never evict an
|
|
490
523
|
// activation tombstone while the session is alive: activation IDs are opaque, so
|
|
@@ -499,15 +532,13 @@ export class AgentServiceSandboxClient {
|
|
|
499
532
|
delete session.acceptedCommands[commandId];
|
|
500
533
|
session.pendingActivationCleanup = null;
|
|
501
534
|
this.turnScopes.delete(pending.agentRunId);
|
|
502
|
-
if (this.activeSessionId === sessionId)
|
|
503
|
-
this.activeSessionId = null;
|
|
504
535
|
if (this.currentTurnSessionId === sessionId)
|
|
505
536
|
this.currentTurnSessionId = null;
|
|
506
537
|
this.persist();
|
|
507
538
|
}
|
|
508
|
-
recoverPendingActivationCleanups() {
|
|
539
|
+
async recoverPendingActivationCleanups() {
|
|
509
540
|
for (const sessionId of Object.keys(this.state.sessions)) {
|
|
510
|
-
this.completePendingActivationCleanup(sessionId);
|
|
541
|
+
await this.completePendingActivationCleanup(sessionId);
|
|
511
542
|
}
|
|
512
543
|
}
|
|
513
544
|
async run() {
|
|
@@ -568,6 +599,23 @@ export class AgentServiceSandboxClient {
|
|
|
568
599
|
this.removeOperationalLogSink();
|
|
569
600
|
this.socket?.close(1000, "daemon_stopping");
|
|
570
601
|
}
|
|
602
|
+
async stopGracefully() {
|
|
603
|
+
this.dispatcher.cancelAll();
|
|
604
|
+
await quiesceRuntimeWriters(null);
|
|
605
|
+
await flushWorkspaceFilesystem();
|
|
606
|
+
this.stop();
|
|
607
|
+
}
|
|
608
|
+
async drainWorkspace(fence) {
|
|
609
|
+
if (!(await this.dispatcher.drain(10_000)) || !this.isCurrentLifecycleFence(fence)) {
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
612
|
+
await quiesceRuntimeWriters(null);
|
|
613
|
+
await flushWorkspaceFilesystem();
|
|
614
|
+
if (!this.isCurrentLifecycleFence(fence))
|
|
615
|
+
return;
|
|
616
|
+
await this.sendControlFrame("sandbox.drained", {});
|
|
617
|
+
this.stop();
|
|
618
|
+
}
|
|
571
619
|
async postEvent(agentRunId, event) {
|
|
572
620
|
const scope = this.turnScopes.get(agentRunId);
|
|
573
621
|
if (!scope)
|
|
@@ -578,6 +626,8 @@ export class AgentServiceSandboxClient {
|
|
|
578
626
|
if (!this.sandboxGeneration || !this.connectionEpoch) {
|
|
579
627
|
throw new Error("Agent Service sandbox is not authenticated");
|
|
580
628
|
}
|
|
629
|
+
// Terminal truth is never delayed by object storage. Durable workspace checkpointing
|
|
630
|
+
// is a separate lifecycle command issued only for pause/cleanup/rotation.
|
|
581
631
|
if (event.type === "run.block") {
|
|
582
632
|
await this.waitForEventCapacity();
|
|
583
633
|
}
|
|
@@ -782,10 +832,11 @@ export class AgentServiceSandboxClient {
|
|
|
782
832
|
if (frame.sandbox_generation !== this.state.sandboxGeneration) {
|
|
783
833
|
// Generation change wipes every session shard (contract §1.4).
|
|
784
834
|
this.dispatcher.cancelAll();
|
|
835
|
+
await quiesceRuntimeWriters(null);
|
|
785
836
|
this.rejectPending(new Error("Agent Service sandbox generation changed"));
|
|
786
837
|
for (const sessionId of Object.keys(this.state.sessions)) {
|
|
787
838
|
revokeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
|
|
788
|
-
removeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration);
|
|
839
|
+
removeRuntimeSessionWorkspace(sessionId, this.state.sandboxGeneration, nasWorkspaceEnabled());
|
|
789
840
|
}
|
|
790
841
|
this.state.sessions = {};
|
|
791
842
|
this.state.sandboxGeneration = frame.sandbox_generation;
|
|
@@ -800,7 +851,7 @@ export class AgentServiceSandboxClient {
|
|
|
800
851
|
this.connectionEpoch = frame.connection_epoch;
|
|
801
852
|
this.inboundSeq = frame.seq;
|
|
802
853
|
this.outboundSeq = frame.connection_epoch * SEQ_EPOCH_BASE;
|
|
803
|
-
this.recoverPendingActivationCleanups();
|
|
854
|
+
await this.recoverPendingActivationCleanups();
|
|
804
855
|
const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
|
|
805
856
|
this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
|
|
806
857
|
const staleSeconds = Number(frame.payload.stale_after_seconds ?? 45);
|
|
@@ -815,12 +866,19 @@ export class AgentServiceSandboxClient {
|
|
|
815
866
|
this.runtimeLogPolicy = runtimeLogPolicy(frame.payload.runtime_log_policy);
|
|
816
867
|
await cleanupRuntimeSessionWorkspaceCopyStaging();
|
|
817
868
|
this.persist();
|
|
869
|
+
assertWorkspaceFilesystem(process.env.BOTLEARN_AGENT_SERVICE_WORKSPACE_ROOT ?? "");
|
|
870
|
+
assertWorkspaceWriterFreezeSupported();
|
|
818
871
|
await this.sendControlFrame("sandbox.ready", {
|
|
819
872
|
protocol_versions: [AGENT_SERVICE_WS_SCHEMA],
|
|
820
873
|
capabilities: [
|
|
821
874
|
WORKSPACE_FILE_READ_BINARY_CAPABILITY,
|
|
875
|
+
"workspace_writer_freeze_v1",
|
|
876
|
+
"workspace_snapshot_v2",
|
|
877
|
+
"workspace_restore_v1",
|
|
878
|
+
...(nasWorkspaceEnabled() ? ["workspace_nas_v1"] : []),
|
|
822
879
|
...(workspaceCopyV1Supported() ? ["workspace_copy_v1"] : []),
|
|
823
880
|
],
|
|
881
|
+
workspace_snapshot_policy: workspaceSnapshotPolicyCapability(),
|
|
824
882
|
daemon_version: this.options.daemonVersion,
|
|
825
883
|
runtime_versions: {
|
|
826
884
|
course_daemon: this.options.daemonVersion,
|
|
@@ -952,16 +1010,19 @@ export class AgentServiceSandboxClient {
|
|
|
952
1010
|
});
|
|
953
1011
|
});
|
|
954
1012
|
return;
|
|
1013
|
+
case "workspace.checkpoint":
|
|
1014
|
+
this.scheduleLifecycle("workspace.checkpoint", () => this.handleWorkspaceCheckpoint(frame));
|
|
1015
|
+
return;
|
|
1016
|
+
case "workspace.thaw":
|
|
1017
|
+
await thawRuntimeWriters();
|
|
1018
|
+
await this.sendCommandAck(frame, "ok");
|
|
1019
|
+
return;
|
|
955
1020
|
case "sandbox.shutdown":
|
|
956
|
-
this.
|
|
1021
|
+
await this.stopGracefully();
|
|
957
1022
|
return;
|
|
958
1023
|
case "sandbox.drain":
|
|
959
1024
|
await this.sendCommandAck(frame, "ok");
|
|
960
|
-
this.scheduleLifecycle("sandbox.drain",
|
|
961
|
-
if (await this.dispatcher.drain(10_000) && this.isCurrentLifecycleFence(fence)) {
|
|
962
|
-
await this.sendControlFrame("sandbox.drained", {});
|
|
963
|
-
}
|
|
964
|
-
});
|
|
1025
|
+
this.scheduleLifecycle("sandbox.drain", (fence) => this.drainWorkspace(fence));
|
|
965
1026
|
return;
|
|
966
1027
|
case "protocol.error":
|
|
967
1028
|
throw new Error(`server protocol error: ${String(frame.payload.code ?? "unknown")}`);
|
|
@@ -979,16 +1040,15 @@ export class AgentServiceSandboxClient {
|
|
|
979
1040
|
this.scheduleLifecycle("sandbox.sync.close", (fence) => this.closeSession(sessionId, "sync_close", frame, fence));
|
|
980
1041
|
}
|
|
981
1042
|
const state = frame.payload.state;
|
|
1043
|
+
if (frame.payload.thaw_runtime_writers === true) {
|
|
1044
|
+
await thawRuntimeWriters();
|
|
1045
|
+
}
|
|
982
1046
|
if (state === "shutdown") {
|
|
983
|
-
this.
|
|
1047
|
+
await this.stopGracefully();
|
|
984
1048
|
return;
|
|
985
1049
|
}
|
|
986
1050
|
if (state === "drain") {
|
|
987
|
-
this.scheduleLifecycle("sandbox.sync.drain",
|
|
988
|
-
if (await this.dispatcher.drain(10_000) && this.isCurrentLifecycleFence(fence)) {
|
|
989
|
-
await this.sendControlFrame("sandbox.drained", {});
|
|
990
|
-
}
|
|
991
|
-
});
|
|
1051
|
+
this.scheduleLifecycle("sandbox.sync.drain", (fence) => this.drainWorkspace(fence));
|
|
992
1052
|
}
|
|
993
1053
|
// idle/run/cancel:实际工作全部由显式命令帧下发(合同 §1.3)。
|
|
994
1054
|
}
|
|
@@ -998,6 +1058,7 @@ export class AgentServiceSandboxClient {
|
|
|
998
1058
|
const courseRunId = frame.payload.course_run_id;
|
|
999
1059
|
const requestedWorkspaceRef = frame.payload.workspace_ref;
|
|
1000
1060
|
const requestedCopy = frame.payload.workspace_copy;
|
|
1061
|
+
const requestedRestore = frame.payload.workspace_restore;
|
|
1001
1062
|
const requestedCopyRecord = requestedCopy && typeof requestedCopy === "object"
|
|
1002
1063
|
? requestedCopy
|
|
1003
1064
|
: undefined;
|
|
@@ -1018,7 +1079,11 @@ export class AgentServiceSandboxClient {
|
|
|
1018
1079
|
typeof copyId !== "string" ||
|
|
1019
1080
|
copyId.length < 1 ||
|
|
1020
1081
|
typeof requestedCopySource !== "string" ||
|
|
1021
|
-
requestedCopySource.length < 1))
|
|
1082
|
+
requestedCopySource.length < 1))
|
|
1083
|
+
|| (requestedRestore !== null && requestedRestore !== undefined &&
|
|
1084
|
+
(typeof requestedRestore !== "object" || Array.isArray(requestedRestore)))
|
|
1085
|
+
|| (requestedCopy !== null && requestedCopy !== undefined &&
|
|
1086
|
+
requestedRestore !== null && requestedRestore !== undefined)) {
|
|
1022
1087
|
throw new Error("invalid session.open payload");
|
|
1023
1088
|
}
|
|
1024
1089
|
if (session.courseRunId !== null && session.courseRunId !== courseRunId) {
|
|
@@ -1032,6 +1097,165 @@ export class AgentServiceSandboxClient {
|
|
|
1032
1097
|
const isNewSession = session.courseRunId === null;
|
|
1033
1098
|
let markerPath = null;
|
|
1034
1099
|
try {
|
|
1100
|
+
if (requestedRestore && typeof requestedRestore === "object") {
|
|
1101
|
+
const raw = requestedRestore;
|
|
1102
|
+
const descriptorRaw = raw.descriptor;
|
|
1103
|
+
const controlFence = raw.control_fence;
|
|
1104
|
+
let restoreControlToken = raw.control_token;
|
|
1105
|
+
if (!controlFence || typeof controlFence !== "object" || Array.isArray(controlFence)) {
|
|
1106
|
+
throw new Error("invalid workspace restore control fence");
|
|
1107
|
+
}
|
|
1108
|
+
if (typeof restoreControlToken !== "string" || restoreControlToken.length < 32) {
|
|
1109
|
+
throw new Error("invalid workspace restore control token");
|
|
1110
|
+
}
|
|
1111
|
+
if (!descriptorRaw || !Array.isArray(raw.entries)) {
|
|
1112
|
+
throw new Error("invalid workspace restore payload");
|
|
1113
|
+
}
|
|
1114
|
+
const descriptor = {
|
|
1115
|
+
sandboxId: this.options.sandboxId,
|
|
1116
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1117
|
+
runtimeSessionId: sessionId,
|
|
1118
|
+
continuityState: descriptorRaw.continuity_state,
|
|
1119
|
+
continuityErrorCode: descriptorRaw.continuity_error_code,
|
|
1120
|
+
snapshotId: descriptorRaw.snapshot_id,
|
|
1121
|
+
revision: Number(descriptorRaw.revision),
|
|
1122
|
+
contentSha256: descriptorRaw.content_sha256,
|
|
1123
|
+
expectedFileCount: Number(descriptorRaw.expected_file_count),
|
|
1124
|
+
expectedEntryCount: Number(descriptorRaw.expected_entry_count),
|
|
1125
|
+
expectedTotalBytes: Number(descriptorRaw.expected_total_bytes),
|
|
1126
|
+
};
|
|
1127
|
+
const workspaceDirectory = runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration);
|
|
1128
|
+
const requiredFreeBytes = Math.max(WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes, descriptor.expectedTotalBytes + WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes);
|
|
1129
|
+
// Fail before issuing or consuming any object grant. The download staging is a
|
|
1130
|
+
// sibling on this same bounded backing filesystem and is renamed, not copied,
|
|
1131
|
+
// into the atomic restore staging tree.
|
|
1132
|
+
assertWorkspaceRestoreCapacity({
|
|
1133
|
+
workspaceDirectory,
|
|
1134
|
+
descriptor,
|
|
1135
|
+
requiredFreeBytes,
|
|
1136
|
+
entryCopies: 2,
|
|
1137
|
+
});
|
|
1138
|
+
const entries = [];
|
|
1139
|
+
const predownloadRoot = path.join(path.dirname(workspaceDirectory), `.botlearn-predownload-${sessionId}-${randomUUID()}`);
|
|
1140
|
+
mkdirSync(predownloadRoot, { recursive: true, mode: 0o700 });
|
|
1141
|
+
try {
|
|
1142
|
+
let nextCursor = null;
|
|
1143
|
+
do {
|
|
1144
|
+
if (nextCursor !== null && nextCursor.length > 2048) {
|
|
1145
|
+
throw new Error("invalid workspace restore cursor");
|
|
1146
|
+
}
|
|
1147
|
+
const pageSignal = AbortSignal.timeout(60_000);
|
|
1148
|
+
const response = await fetch(`${agentServiceHttpBase(this.options.wsUrl)}/runtime-sessions/${sessionId}/restore-grants`, {
|
|
1149
|
+
method: "POST",
|
|
1150
|
+
headers: {
|
|
1151
|
+
Authorization: `Bearer ${restoreControlToken}`,
|
|
1152
|
+
"Content-Type": "application/json",
|
|
1153
|
+
},
|
|
1154
|
+
body: JSON.stringify({
|
|
1155
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
1156
|
+
fence: controlFence,
|
|
1157
|
+
cursor: nextCursor,
|
|
1158
|
+
// Fetch one canonical entry at a time so a grant is consumed as soon as
|
|
1159
|
+
// it is signed, including deployments using the minimum 60-second TTL.
|
|
1160
|
+
max_entries: 1,
|
|
1161
|
+
}),
|
|
1162
|
+
signal: pageSignal,
|
|
1163
|
+
});
|
|
1164
|
+
if (!response.ok) {
|
|
1165
|
+
let code = "workspace_restore_unavailable";
|
|
1166
|
+
let retryable = response.status >= 500 || response.status === 408 ||
|
|
1167
|
+
response.status === 429;
|
|
1168
|
+
try {
|
|
1169
|
+
const rejected = await response.json();
|
|
1170
|
+
const detail = rejected.detail;
|
|
1171
|
+
if (detail && typeof detail === "object" && !Array.isArray(detail)) {
|
|
1172
|
+
const stableCode = detail.code;
|
|
1173
|
+
if (typeof stableCode === "string" && stableCode.length <= 120) {
|
|
1174
|
+
code = stableCode;
|
|
1175
|
+
}
|
|
1176
|
+
if (typeof detail.retryable === "boolean") {
|
|
1177
|
+
retryable = detail.retryable;
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
catch {
|
|
1182
|
+
// Content-free fallback retains the stable local failure code.
|
|
1183
|
+
}
|
|
1184
|
+
throw new WorkspaceRestoreError(code, retryable);
|
|
1185
|
+
}
|
|
1186
|
+
const page = await response.json();
|
|
1187
|
+
if (!Array.isArray(page.entries))
|
|
1188
|
+
throw new Error("invalid workspace restore page");
|
|
1189
|
+
if (typeof page.control_token !== "string" || page.control_token.length < 32) {
|
|
1190
|
+
throw new Error("invalid refreshed workspace restore control token");
|
|
1191
|
+
}
|
|
1192
|
+
restoreControlToken = page.control_token;
|
|
1193
|
+
const pageFiles = [];
|
|
1194
|
+
for (const value of page.entries) {
|
|
1195
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1196
|
+
throw new Error("invalid workspace restore entry");
|
|
1197
|
+
}
|
|
1198
|
+
const item = value;
|
|
1199
|
+
if (item.entry_type === "directory") {
|
|
1200
|
+
entries.push({ type: "directory", path: String(item.path), mode: "0700" });
|
|
1201
|
+
continue;
|
|
1202
|
+
}
|
|
1203
|
+
if (item.entry_type !== "file" || !item.grant || typeof item.grant !== "object") {
|
|
1204
|
+
throw new Error("invalid workspace restore file grant");
|
|
1205
|
+
}
|
|
1206
|
+
const entry = {
|
|
1207
|
+
type: "file",
|
|
1208
|
+
path: String(item.path),
|
|
1209
|
+
mode: item.mode,
|
|
1210
|
+
sizeBytes: Number(item.size_bytes),
|
|
1211
|
+
sha256: String(item.sha256),
|
|
1212
|
+
};
|
|
1213
|
+
entries.push(entry);
|
|
1214
|
+
pageFiles.push({ entry, grant: item.grant });
|
|
1215
|
+
}
|
|
1216
|
+
// Validate all paths and duplicates before using any page path locally.
|
|
1217
|
+
validateWorkspaceEntrySet({ schemaVersion: "agent-workspace-entry-set/1", entries }, WORKSPACE_SNAPSHOT_POLICY_V1.limits);
|
|
1218
|
+
for (const { entry, grant } of pageFiles) {
|
|
1219
|
+
if (typeof grant.url !== "string")
|
|
1220
|
+
throw new Error("workspace restore grant missing");
|
|
1221
|
+
const destination = path.join(predownloadRoot, ...entry.path.split("/"));
|
|
1222
|
+
mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
|
|
1223
|
+
const part = `${destination}.part`;
|
|
1224
|
+
const downloadSignal = AbortSignal.timeout(60_000);
|
|
1225
|
+
const download = await fetch(grant.url, {
|
|
1226
|
+
method: "GET",
|
|
1227
|
+
headers: grant.headers,
|
|
1228
|
+
signal: downloadSignal,
|
|
1229
|
+
});
|
|
1230
|
+
if (!download.ok)
|
|
1231
|
+
throw new Error(`workspace_restore_download_${download.status}`);
|
|
1232
|
+
if (!download.body)
|
|
1233
|
+
throw new Error("workspace_restore_download_body_missing");
|
|
1234
|
+
await pipeline(Readable.fromWeb(download.body), createWriteStream(part, { mode: 0o600 }), { signal: downloadSignal });
|
|
1235
|
+
renameSync(part, destination);
|
|
1236
|
+
}
|
|
1237
|
+
const rawNext = page.next_cursor;
|
|
1238
|
+
if (rawNext !== null && rawNext !== undefined && typeof rawNext !== "string") {
|
|
1239
|
+
throw new Error("invalid workspace restore cursor");
|
|
1240
|
+
}
|
|
1241
|
+
nextCursor = rawNext ?? null;
|
|
1242
|
+
} while (nextCursor !== null);
|
|
1243
|
+
await restoreWorkspaceSnapshot({
|
|
1244
|
+
workspaceDirectory,
|
|
1245
|
+
controlDirectory: path.join(ensureDaemonHome(), "agent-service-sandboxes", this.options.sandboxId, "workspace-control"),
|
|
1246
|
+
descriptor,
|
|
1247
|
+
entrySet: { schemaVersion: "agent-workspace-entry-set/1", entries },
|
|
1248
|
+
limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
|
|
1249
|
+
requiredFreeBytes,
|
|
1250
|
+
downloadFile: async (entry, partPath) => {
|
|
1251
|
+
renameSync(path.join(predownloadRoot, ...entry.path.split("/")), partPath);
|
|
1252
|
+
},
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
finally {
|
|
1256
|
+
rmSync(predownloadRoot, { recursive: true, force: true });
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1035
1259
|
if (typeof requestedCopySource === "string" && typeof copyId === "string") {
|
|
1036
1260
|
if (session.workspaceCopyReceipt !== null) {
|
|
1037
1261
|
const receipt = session.workspaceCopyReceipt;
|
|
@@ -1075,28 +1299,161 @@ export class AgentServiceSandboxClient {
|
|
|
1075
1299
|
});
|
|
1076
1300
|
return;
|
|
1077
1301
|
}
|
|
1302
|
+
if (requestedRestore) {
|
|
1303
|
+
const restoreError = error instanceof WorkspaceRestoreError ? error : null;
|
|
1304
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1305
|
+
error_code: restoreError?.code ?? "workspace_restore_unavailable",
|
|
1306
|
+
retryable: restoreError?.retryable ?? true,
|
|
1307
|
+
});
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
1078
1310
|
throw error;
|
|
1079
1311
|
}
|
|
1080
1312
|
session.courseRunId = courseRunId;
|
|
1081
1313
|
session.workspaceRef = session.workspaceRef ?? (typeof requestedWorkspaceRef === "string"
|
|
1082
1314
|
? requestedWorkspaceRef
|
|
1083
1315
|
: `ws_${sessionId.replaceAll("-", "")}_g${this.sandboxGeneration}`);
|
|
1084
|
-
|
|
1316
|
+
// Replaying session.open is part of reconnect reconciliation. An already-active
|
|
1317
|
+
// runtime may still be creating files in this workspace, so never transiently revoke
|
|
1318
|
+
// its group write/execute access before the following session.activate arrives.
|
|
1319
|
+
if (session.workspaceCopyReceipt === null && (isNewSession || session.activationId === null)) {
|
|
1085
1320
|
ensureRuntimeSessionDirectories(sessionId, this.sandboxGeneration);
|
|
1086
1321
|
}
|
|
1087
|
-
this.state.sessions[sessionId] = session;
|
|
1088
|
-
this.persist();
|
|
1089
1322
|
if (markerPath !== null) {
|
|
1090
1323
|
await finalizeRuntimeSessionWorkspaceCopy(markerPath);
|
|
1091
1324
|
}
|
|
1325
|
+
try {
|
|
1326
|
+
if (!nasWorkspaceEnabled()) {
|
|
1327
|
+
await applyRuntimeWorkspaceQuota(sessionId, this.sandboxGeneration);
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
catch (error) {
|
|
1331
|
+
if (requestedRestore) {
|
|
1332
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1333
|
+
error_code: "workspace_restore_unavailable",
|
|
1334
|
+
});
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
if (requestedCopy) {
|
|
1338
|
+
await this.sendSessionFrame("session.open_failed", sessionId, {
|
|
1339
|
+
error_code: "workspace_migration_io_failed",
|
|
1340
|
+
});
|
|
1341
|
+
return;
|
|
1342
|
+
}
|
|
1343
|
+
throw error;
|
|
1344
|
+
}
|
|
1345
|
+
this.state.sessions[sessionId] = session;
|
|
1346
|
+
this.persist();
|
|
1347
|
+
const materialization = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
|
|
1092
1348
|
await this.sendSessionFrame("session.opened", sessionId, {
|
|
1093
1349
|
workspace_ref: session.workspaceRef,
|
|
1094
1350
|
native_session_id: session.nativeSessionId,
|
|
1351
|
+
...(materialization
|
|
1352
|
+
? {
|
|
1353
|
+
workspace_marker_version: 1,
|
|
1354
|
+
workspace_continuity_state: materialization.workspaceContinuityState,
|
|
1355
|
+
workspace_snapshot_id: materialization.snapshotId,
|
|
1356
|
+
workspace_revision: materialization.revision,
|
|
1357
|
+
workspace_content_sha256: materialization.contentSha256,
|
|
1358
|
+
}
|
|
1359
|
+
: {}),
|
|
1095
1360
|
...(session.workspaceCopyReceipt !== null
|
|
1096
1361
|
? { workspace_copy_receipt: session.workspaceCopyReceipt }
|
|
1097
1362
|
: {}),
|
|
1098
1363
|
});
|
|
1099
1364
|
}
|
|
1365
|
+
async handleWorkspaceCheckpoint(frame) {
|
|
1366
|
+
const sessionId = frame.runtime_session_id;
|
|
1367
|
+
const session = this.state.sessions[sessionId];
|
|
1368
|
+
const checkpointId = frame.payload.checkpoint_id;
|
|
1369
|
+
const controlToken = frame.payload.control_token;
|
|
1370
|
+
const baseRevision = Number(frame.payload.base_revision);
|
|
1371
|
+
const sourceKind = frame.payload.source_kind;
|
|
1372
|
+
if (!session ||
|
|
1373
|
+
this.currentTurnSessionId === sessionId ||
|
|
1374
|
+
typeof checkpointId !== "string" ||
|
|
1375
|
+
typeof controlToken !== "string" ||
|
|
1376
|
+
controlToken.length < 32 ||
|
|
1377
|
+
frame.payload.policy_id !== "WorkspaceSnapshotPolicyV1" ||
|
|
1378
|
+
![
|
|
1379
|
+
"sandbox_pause",
|
|
1380
|
+
"sandbox_cleanup",
|
|
1381
|
+
"generation_rotation",
|
|
1382
|
+
"initial_migration",
|
|
1383
|
+
"live_workspace_copy",
|
|
1384
|
+
"maintenance_repair",
|
|
1385
|
+
].includes(String(sourceKind)) ||
|
|
1386
|
+
!Number.isSafeInteger(baseRevision) ||
|
|
1387
|
+
baseRevision < 0 ||
|
|
1388
|
+
frame.payload.agent_run_id != null ||
|
|
1389
|
+
frame.payload.worker_attempt != null ||
|
|
1390
|
+
frame.payload.activation_id != null) {
|
|
1391
|
+
await this.sendCommandAck(frame, "rejected", "invalid_workspace_checkpoint");
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
const lifecycleCheckpoint = [
|
|
1395
|
+
"sandbox_pause",
|
|
1396
|
+
"sandbox_cleanup",
|
|
1397
|
+
"generation_rotation",
|
|
1398
|
+
].includes(String(sourceKind));
|
|
1399
|
+
let checkpointCommitted = false;
|
|
1400
|
+
try {
|
|
1401
|
+
await freezeRuntimeWriters();
|
|
1402
|
+
const freezeProof = issueWorkspaceWriterFreezeProof({
|
|
1403
|
+
sandboxId: this.options.sandboxId,
|
|
1404
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1405
|
+
runtimeSessionId: sessionId,
|
|
1406
|
+
checkpointId,
|
|
1407
|
+
});
|
|
1408
|
+
await checkpointWorkspace({
|
|
1409
|
+
wsUrl: this.options.wsUrl,
|
|
1410
|
+
reconnectToken: controlToken,
|
|
1411
|
+
workspaceDirectory: runtimeSessionWorkspaceDir(sessionId, this.sandboxGeneration),
|
|
1412
|
+
scope: {
|
|
1413
|
+
checkpointId,
|
|
1414
|
+
baseRevision,
|
|
1415
|
+
sandboxId: this.options.sandboxId,
|
|
1416
|
+
sandboxGeneration: this.sandboxGeneration,
|
|
1417
|
+
connectionEpoch: this.connectionEpoch,
|
|
1418
|
+
runtimeSessionId: sessionId,
|
|
1419
|
+
},
|
|
1420
|
+
freezeProof,
|
|
1421
|
+
});
|
|
1422
|
+
checkpointCommitted = true;
|
|
1423
|
+
const marker = readWorkspaceMaterializationMarker(this.options.sandboxId, sessionId);
|
|
1424
|
+
if (!marker)
|
|
1425
|
+
throw new Error("workspace checkpoint marker missing");
|
|
1426
|
+
await this.sendSessionFrame("session.opened", sessionId, {
|
|
1427
|
+
workspace_ref: session.workspaceRef,
|
|
1428
|
+
native_session_id: session.nativeSessionId,
|
|
1429
|
+
workspace_marker_version: 1,
|
|
1430
|
+
workspace_continuity_state: marker.workspaceContinuityState,
|
|
1431
|
+
workspace_snapshot_id: marker.snapshotId,
|
|
1432
|
+
workspace_revision: marker.revision,
|
|
1433
|
+
workspace_content_sha256: marker.contentSha256,
|
|
1434
|
+
});
|
|
1435
|
+
await this.sendCommandAck(frame, "ok");
|
|
1436
|
+
if (!lifecycleCheckpoint)
|
|
1437
|
+
await thawRuntimeWriters();
|
|
1438
|
+
}
|
|
1439
|
+
catch (error) {
|
|
1440
|
+
if (!lifecycleCheckpoint || !checkpointCommitted) {
|
|
1441
|
+
await thawRuntimeWriters().catch(() => undefined);
|
|
1442
|
+
}
|
|
1443
|
+
this.log.error("workspace baseline checkpoint failed", {
|
|
1444
|
+
sandboxId: this.options.sandboxId,
|
|
1445
|
+
runtimeSessionId: sessionId,
|
|
1446
|
+
error: error instanceof Error ? redactSecretString(error.message) : "unexpected",
|
|
1447
|
+
});
|
|
1448
|
+
const deterministic = (error instanceof WorkspaceSnapshotStagingError && !error.retryable) ||
|
|
1449
|
+
error instanceof WorkspaceEntrySetError ||
|
|
1450
|
+
(error instanceof WorkspaceSnapshotControlError && !error.retryable);
|
|
1451
|
+
const errorCode = deterministic && "code" in error
|
|
1452
|
+
? String(error.code)
|
|
1453
|
+
: "workspace_checkpoint_retryable";
|
|
1454
|
+
await this.sendCommandAck(frame, "rejected", errorCode, !deterministic);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1100
1457
|
async handleSessionActivate(frame, fence) {
|
|
1101
1458
|
const sessionId = frame.runtime_session_id;
|
|
1102
1459
|
const activationId = frame.activation_id;
|
|
@@ -1233,7 +1590,7 @@ export class AgentServiceSandboxClient {
|
|
|
1233
1590
|
const previousSessionId = this.activeSessionId;
|
|
1234
1591
|
const changesActivation = previousSessionId !== null && (previousSessionId !== sessionId || session.activationId !== activationId);
|
|
1235
1592
|
if (changesActivation) {
|
|
1236
|
-
const stopped = await this.deactivateSession(previousSessionId);
|
|
1593
|
+
const stopped = await this.deactivateSession(previousSessionId, previousSessionId !== sessionId);
|
|
1237
1594
|
if (fence && !this.isCurrentLifecycleFence(fence))
|
|
1238
1595
|
return;
|
|
1239
1596
|
if (!stopped) {
|
|
@@ -1419,7 +1776,7 @@ export class AgentServiceSandboxClient {
|
|
|
1419
1776
|
}
|
|
1420
1777
|
delete session.acceptedCommands[commandId];
|
|
1421
1778
|
this.persist();
|
|
1422
|
-
this.finishTurn({ agent_run_id: runId });
|
|
1779
|
+
await this.finishTurn({ agent_run_id: runId });
|
|
1423
1780
|
}
|
|
1424
1781
|
/**
|
|
1425
1782
|
* 幂等关闭一个 runtime session:终止其 runtime 子进程、删除 workspace/native state、
|
|
@@ -1429,15 +1786,16 @@ export class AgentServiceSandboxClient {
|
|
|
1429
1786
|
const session = this.state.sessions[sessionId];
|
|
1430
1787
|
if (session) {
|
|
1431
1788
|
const runIds = this.runIdsForSession(sessionId);
|
|
1432
|
-
session.activationId = null;
|
|
1433
1789
|
this.activationContexts.delete(sessionId);
|
|
1434
1790
|
this.sessionProfiles.delete(sessionId);
|
|
1435
|
-
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1436
1791
|
for (const runId of runIds)
|
|
1437
1792
|
this.dispatcher.cancel(runId);
|
|
1438
1793
|
if (!await this.dispatcher.waitForRuns(runIds, 10_000)) {
|
|
1439
1794
|
throw new Error(`runtime session ${sessionId} did not stop before close`);
|
|
1440
1795
|
}
|
|
1796
|
+
await this.terminateSessionWriters(session);
|
|
1797
|
+
session.activationId = null;
|
|
1798
|
+
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1441
1799
|
if (fence && !this.isCurrentLifecycleFence(fence))
|
|
1442
1800
|
return;
|
|
1443
1801
|
if (this.activeSessionId === sessionId)
|
|
@@ -1478,7 +1836,7 @@ export class AgentServiceSandboxClient {
|
|
|
1478
1836
|
delete session.acceptedCommands[commandId];
|
|
1479
1837
|
this.persist();
|
|
1480
1838
|
}
|
|
1481
|
-
this.finishTurn({ agent_run_id: accepted.agentRunId });
|
|
1839
|
+
await this.finishTurn({ agent_run_id: accepted.agentRunId });
|
|
1482
1840
|
}
|
|
1483
1841
|
catch (error) {
|
|
1484
1842
|
this.log.error("failed to reconcile an interrupted accepted turn", {
|
|
@@ -1580,21 +1938,40 @@ export class AgentServiceSandboxClient {
|
|
|
1580
1938
|
.filter(([, scope]) => scope.sessionId === sessionId)
|
|
1581
1939
|
.map(([runId]) => runId);
|
|
1582
1940
|
}
|
|
1583
|
-
async deactivateSession(sessionId) {
|
|
1941
|
+
async deactivateSession(sessionId, terminateWriters = false) {
|
|
1584
1942
|
const session = this.state.sessions[sessionId];
|
|
1585
1943
|
if (!session)
|
|
1586
1944
|
return true;
|
|
1587
1945
|
const runIds = this.runIdsForSession(sessionId);
|
|
1588
|
-
session.activationId = null;
|
|
1589
1946
|
this.activationContexts.delete(sessionId);
|
|
1590
1947
|
this.sessionProfiles.delete(sessionId);
|
|
1591
|
-
|
|
1948
|
+
if (terminateWriters) {
|
|
1949
|
+
revokeRuntimeSessionWorkspace(sessionId, this.sandboxGeneration);
|
|
1950
|
+
}
|
|
1592
1951
|
if (this.activeSessionId === sessionId)
|
|
1593
1952
|
this.activeSessionId = null;
|
|
1594
1953
|
for (const runId of runIds)
|
|
1595
1954
|
this.dispatcher.cancel(runId);
|
|
1596
1955
|
this.persist();
|
|
1597
|
-
|
|
1956
|
+
const stopped = await this.dispatcher.waitForRuns(runIds, 10_000);
|
|
1957
|
+
if (stopped && terminateWriters) {
|
|
1958
|
+
await this.terminateSessionWriters(session);
|
|
1959
|
+
}
|
|
1960
|
+
session.activationId = null;
|
|
1961
|
+
this.persist();
|
|
1962
|
+
return stopped;
|
|
1963
|
+
}
|
|
1964
|
+
async terminateSessionWriters(session) {
|
|
1965
|
+
const activationIds = new Set([
|
|
1966
|
+
...session.completedActivations,
|
|
1967
|
+
...(session.activationId ? [session.activationId] : []),
|
|
1968
|
+
...(session.pendingActivationCleanup?.activationId
|
|
1969
|
+
? [session.pendingActivationCleanup.activationId]
|
|
1970
|
+
: []),
|
|
1971
|
+
]);
|
|
1972
|
+
for (const activationId of activationIds) {
|
|
1973
|
+
await quiesceRuntimeWriters(activationId);
|
|
1974
|
+
}
|
|
1598
1975
|
}
|
|
1599
1976
|
sameTurnScope(left, right) {
|
|
1600
1977
|
return left.sessionId === right.sessionId &&
|
|
@@ -1687,7 +2064,7 @@ export class AgentServiceSandboxClient {
|
|
|
1687
2064
|
spool_frames: this.spoolFrameCount(),
|
|
1688
2065
|
});
|
|
1689
2066
|
}
|
|
1690
|
-
async sendCommandAck(frame, status, error) {
|
|
2067
|
+
async sendCommandAck(frame, status, error, retryable) {
|
|
1691
2068
|
if (frame.sandbox_generation !== this.sandboxGeneration ||
|
|
1692
2069
|
frame.connection_epoch !== this.connectionEpoch)
|
|
1693
2070
|
return;
|
|
@@ -1695,6 +2072,7 @@ export class AgentServiceSandboxClient {
|
|
|
1695
2072
|
command_frame_id: frame.frame_id,
|
|
1696
2073
|
status,
|
|
1697
2074
|
...(error !== undefined ? { error } : {}),
|
|
2075
|
+
...(retryable !== undefined ? { retryable } : {}),
|
|
1698
2076
|
});
|
|
1699
2077
|
}
|
|
1700
2078
|
async handleWorkspaceFileRead(frame) {
|