@botlearn-course/daemon 0.0.19 → 0.0.20-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-service-sandbox.d.ts +9 -1
- package/dist/agent-service-sandbox.js +490 -16
- package/dist/agent-service-ws-protocol.d.ts +3 -3
- package/dist/agent-service-ws-protocol.js +6 -2
- package/dist/cli.js +19 -1
- package/dist/file-candidates.d.ts +28 -1
- package/dist/file-candidates.js +57 -11
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/run-dispatcher.d.ts +3 -2
- package/dist/run-dispatcher.js +62 -5
- package/dist/runtime-env.js +4 -4
- package/dist/runtime-quiescence.d.ts +16 -0
- package/dist/runtime-quiescence.js +42 -0
- package/dist/runtimes/engine.js +1 -1
- package/dist/tool-observation.d.ts +7 -4
- package/dist/tool-observation.js +40 -18
- package/dist/trace-projection.d.ts +21 -0
- package/dist/trace-projection.js +56 -0
- package/dist/types.d.ts +1 -1
- package/dist/workspace-entry-set.d.ts +31 -0
- package/dist/workspace-entry-set.js +164 -0
- package/dist/workspace-materialization.d.ts +16 -0
- package/dist/workspace-materialization.js +136 -0
- package/dist/workspace-quota.d.ts +4 -0
- package/dist/workspace-quota.js +42 -0
- package/dist/workspace-restore.d.ts +42 -0
- package/dist/workspace-restore.js +347 -0
- package/dist/workspace-snapshot-control.d.ts +29 -0
- package/dist/workspace-snapshot-control.js +169 -0
- package/dist/workspace-snapshot-policy.d.ts +24 -0
- package/dist/workspace-snapshot-policy.js +45 -0
- package/dist/workspace-snapshot-staging.d.ts +27 -0
- package/dist/workspace-snapshot-staging.js +275 -0
- package/dist/workspace.d.ts +56 -0
- package/dist/workspace.js +553 -1
- package/package.json +1 -1
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statfsSync, unlinkSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { validateWorkspaceEntrySet, } from "./workspace-entry-set.js";
|
|
5
|
+
import { WORKSPACE_MATERIALIZATION_SCHEMA, writeWorkspaceMaterializationMarker, } from "./workspace-materialization.js";
|
|
6
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
7
|
+
const SHA256 = /^[0-9a-f]{64}$/;
|
|
8
|
+
const RESTORE_JOURNAL_SCHEMA = "agent-workspace-restore-journal/1";
|
|
9
|
+
const COPY_CHUNK_BYTES = 64 * 1024;
|
|
10
|
+
export class WorkspaceRestoreError extends Error {
|
|
11
|
+
code;
|
|
12
|
+
retryable;
|
|
13
|
+
constructor(code, retryable = false) {
|
|
14
|
+
super(code);
|
|
15
|
+
this.code = code;
|
|
16
|
+
this.retryable = retryable;
|
|
17
|
+
this.name = "WorkspaceRestoreError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function fail(code) {
|
|
21
|
+
throw new WorkspaceRestoreError(code, code === "workspace_restore_unavailable" ||
|
|
22
|
+
code === "workspace_restore_disk_pressure" ||
|
|
23
|
+
code === "workspace_restore_inode_pressure");
|
|
24
|
+
}
|
|
25
|
+
function assertUuid(value, label) {
|
|
26
|
+
if (!UUID.test(value))
|
|
27
|
+
fail(`workspace_restore_${label}_invalid`);
|
|
28
|
+
}
|
|
29
|
+
function hasExactKeys(value, expected) {
|
|
30
|
+
const actual = Object.keys(value).sort();
|
|
31
|
+
const sortedExpected = [...expected].sort();
|
|
32
|
+
return actual.length === sortedExpected.length &&
|
|
33
|
+
actual.every((key, index) => key === sortedExpected[index]);
|
|
34
|
+
}
|
|
35
|
+
function containsPath(parent, candidate) {
|
|
36
|
+
const relative = path.relative(parent, candidate);
|
|
37
|
+
return relative === "" || (!relative.startsWith(".." + path.sep) && relative !== "..");
|
|
38
|
+
}
|
|
39
|
+
function validJournalSwapName(value, prefix) {
|
|
40
|
+
return typeof value === "string" && value.startsWith(prefix) &&
|
|
41
|
+
UUID.test(value.slice(prefix.length));
|
|
42
|
+
}
|
|
43
|
+
function assertControlPath(workspaceDirectory, controlDirectory) {
|
|
44
|
+
if (containsPath(path.resolve(workspaceDirectory), path.resolve(controlDirectory))) {
|
|
45
|
+
fail("workspace_restore_control_path_invalid");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
function assertDescriptor(descriptor) {
|
|
49
|
+
assertUuid(descriptor.sandboxId, "sandbox_id");
|
|
50
|
+
assertUuid(descriptor.runtimeSessionId, "session_id");
|
|
51
|
+
if (!Number.isSafeInteger(descriptor.sandboxGeneration) ||
|
|
52
|
+
descriptor.sandboxGeneration < 1 ||
|
|
53
|
+
!Number.isSafeInteger(descriptor.revision) ||
|
|
54
|
+
descriptor.revision < 0 ||
|
|
55
|
+
!Number.isSafeInteger(descriptor.expectedFileCount) ||
|
|
56
|
+
descriptor.expectedFileCount < 0 ||
|
|
57
|
+
!Number.isSafeInteger(descriptor.expectedEntryCount) ||
|
|
58
|
+
descriptor.expectedEntryCount < 0 ||
|
|
59
|
+
!Number.isSafeInteger(descriptor.expectedTotalBytes) ||
|
|
60
|
+
descriptor.expectedTotalBytes < 0 ||
|
|
61
|
+
(descriptor.continuityState !== "healthy" && descriptor.continuityState !== "degraded") ||
|
|
62
|
+
(descriptor.continuityState === "degraded" && !descriptor.continuityErrorCode) ||
|
|
63
|
+
(descriptor.continuityState === "healthy" && descriptor.continuityErrorCode !== null))
|
|
64
|
+
fail("workspace_restore_descriptor_invalid");
|
|
65
|
+
const revisionZero = descriptor.revision === 0;
|
|
66
|
+
if (revisionZero !== (descriptor.snapshotId === null) ||
|
|
67
|
+
revisionZero !== (descriptor.contentSha256 === null) ||
|
|
68
|
+
(!revisionZero &&
|
|
69
|
+
(!UUID.test(descriptor.snapshotId) || !SHA256.test(descriptor.contentSha256))) ||
|
|
70
|
+
(revisionZero &&
|
|
71
|
+
(descriptor.expectedFileCount !== 0 || descriptor.expectedEntryCount !== 0 ||
|
|
72
|
+
descriptor.expectedTotalBytes !== 0)))
|
|
73
|
+
fail("workspace_restore_descriptor_invalid");
|
|
74
|
+
}
|
|
75
|
+
export function workspaceRestoreJournalPath(controlDirectory, runtimeSessionId) {
|
|
76
|
+
assertUuid(runtimeSessionId, "session_id");
|
|
77
|
+
return path.join(controlDirectory, "workspace-restore-journals", `${runtimeSessionId}.json`);
|
|
78
|
+
}
|
|
79
|
+
function writeJournal(file, journal) {
|
|
80
|
+
const directory = path.dirname(file);
|
|
81
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
82
|
+
chmodSync(directory, 0o700);
|
|
83
|
+
const staging = `${file}.tmp-${process.pid}-${randomUUID()}`;
|
|
84
|
+
try {
|
|
85
|
+
writeFileSync(staging, JSON.stringify(journal), { encoding: "utf8", mode: 0o600 });
|
|
86
|
+
const handle = openSync(staging, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
87
|
+
try {
|
|
88
|
+
fsyncSync(handle);
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
closeSync(handle);
|
|
92
|
+
}
|
|
93
|
+
renameSync(staging, file);
|
|
94
|
+
fsyncDirectory(directory);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
try {
|
|
98
|
+
unlinkSync(staging);
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
// The atomic staging file may not have been created.
|
|
102
|
+
}
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function parseJournal(raw, descriptor) {
|
|
107
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
108
|
+
fail("workspace_restore_journal_invalid");
|
|
109
|
+
}
|
|
110
|
+
const value = raw;
|
|
111
|
+
if (!hasExactKeys(value, [
|
|
112
|
+
"schemaVersion",
|
|
113
|
+
"sandboxId",
|
|
114
|
+
"sandboxGeneration",
|
|
115
|
+
"runtimeSessionId",
|
|
116
|
+
"stagingName",
|
|
117
|
+
"backupName",
|
|
118
|
+
"hadWorkspace",
|
|
119
|
+
"phase",
|
|
120
|
+
"marker",
|
|
121
|
+
]))
|
|
122
|
+
fail("workspace_restore_journal_invalid");
|
|
123
|
+
const namePrefix = `.botlearn-restore-${descriptor.runtimeSessionId}-`;
|
|
124
|
+
const backupPrefix = `.botlearn-backup-${descriptor.runtimeSessionId}-`;
|
|
125
|
+
if (value.schemaVersion !== RESTORE_JOURNAL_SCHEMA ||
|
|
126
|
+
value.sandboxId !== descriptor.sandboxId ||
|
|
127
|
+
value.sandboxGeneration !== descriptor.sandboxGeneration ||
|
|
128
|
+
value.runtimeSessionId !== descriptor.runtimeSessionId ||
|
|
129
|
+
!validJournalSwapName(value.stagingName, namePrefix) ||
|
|
130
|
+
!validJournalSwapName(value.backupName, backupPrefix) ||
|
|
131
|
+
typeof value.hadWorkspace !== "boolean" ||
|
|
132
|
+
!["prepared", "old_moved", "new_moved"].includes(String(value.phase)))
|
|
133
|
+
fail("workspace_restore_journal_invalid");
|
|
134
|
+
const marker = value.marker;
|
|
135
|
+
if (!marker || marker.schemaVersion !== WORKSPACE_MATERIALIZATION_SCHEMA ||
|
|
136
|
+
marker.sandboxId !== descriptor.sandboxId ||
|
|
137
|
+
marker.sandboxGeneration !== descriptor.sandboxGeneration ||
|
|
138
|
+
marker.runtimeSessionId !== descriptor.runtimeSessionId ||
|
|
139
|
+
marker.workspaceContinuityState !== descriptor.continuityState ||
|
|
140
|
+
marker.snapshotId !== descriptor.snapshotId || marker.revision !== descriptor.revision ||
|
|
141
|
+
marker.contentSha256 !== descriptor.contentSha256)
|
|
142
|
+
fail("workspace_restore_journal_invalid");
|
|
143
|
+
return value;
|
|
144
|
+
}
|
|
145
|
+
function fsyncDirectory(directory) {
|
|
146
|
+
const handle = openSync(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
147
|
+
try {
|
|
148
|
+
fsyncSync(handle);
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
closeSync(handle);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function verifyDownloadedFile(file, expected) {
|
|
155
|
+
const handle = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
156
|
+
try {
|
|
157
|
+
const stats = fstatSync(handle, { bigint: true });
|
|
158
|
+
if (!stats.isFile() || stats.nlink !== 1n || stats.size !== BigInt(expected.expectedSize)) {
|
|
159
|
+
fail("workspace_restore_integrity_failed");
|
|
160
|
+
}
|
|
161
|
+
const hash = createHash("sha256");
|
|
162
|
+
const buffer = Buffer.allocUnsafe(COPY_CHUNK_BYTES);
|
|
163
|
+
let total = 0;
|
|
164
|
+
while (true) {
|
|
165
|
+
const bytesRead = readSync(handle, buffer, 0, buffer.length, null);
|
|
166
|
+
if (bytesRead === 0)
|
|
167
|
+
break;
|
|
168
|
+
total += bytesRead;
|
|
169
|
+
hash.update(buffer.subarray(0, bytesRead));
|
|
170
|
+
}
|
|
171
|
+
if (total !== expected.expectedSize || hash.digest("hex") !== expected.expectedSha256) {
|
|
172
|
+
fail("workspace_restore_integrity_failed");
|
|
173
|
+
}
|
|
174
|
+
fsyncSync(handle);
|
|
175
|
+
}
|
|
176
|
+
finally {
|
|
177
|
+
closeSync(handle);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
function markerFromDescriptor(descriptor) {
|
|
181
|
+
return {
|
|
182
|
+
schemaVersion: WORKSPACE_MATERIALIZATION_SCHEMA,
|
|
183
|
+
sandboxId: descriptor.sandboxId,
|
|
184
|
+
sandboxGeneration: descriptor.sandboxGeneration,
|
|
185
|
+
runtimeSessionId: descriptor.runtimeSessionId,
|
|
186
|
+
workspaceContinuityState: descriptor.continuityState,
|
|
187
|
+
snapshotId: descriptor.snapshotId,
|
|
188
|
+
revision: descriptor.revision,
|
|
189
|
+
contentSha256: descriptor.contentSha256,
|
|
190
|
+
writtenAt: new Date().toISOString(),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
export function recoverWorkspaceRestoreSwap(options) {
|
|
194
|
+
assertDescriptor(options.descriptor);
|
|
195
|
+
assertControlPath(options.workspaceDirectory, options.controlDirectory);
|
|
196
|
+
const journalFile = workspaceRestoreJournalPath(options.controlDirectory, options.descriptor.runtimeSessionId);
|
|
197
|
+
if (!existsSync(journalFile))
|
|
198
|
+
return false;
|
|
199
|
+
const journal = parseJournal(JSON.parse(readFileSync(journalFile, "utf8")), options.descriptor);
|
|
200
|
+
const parent = path.dirname(options.workspaceDirectory);
|
|
201
|
+
const staging = path.join(parent, journal.stagingName);
|
|
202
|
+
const backup = path.join(parent, journal.backupName);
|
|
203
|
+
const workspaceExists = existsSync(options.workspaceDirectory);
|
|
204
|
+
const stagingExists = existsSync(staging);
|
|
205
|
+
const backupExists = existsSync(backup);
|
|
206
|
+
if (journal.phase === "new_moved" ||
|
|
207
|
+
(journal.phase === "old_moved" && workspaceExists && !stagingExists)) {
|
|
208
|
+
if (!workspaceExists)
|
|
209
|
+
fail("workspace_restore_journal_invalid");
|
|
210
|
+
writeWorkspaceMaterializationMarker(journal.marker);
|
|
211
|
+
if (backupExists)
|
|
212
|
+
rmSync(backup, { recursive: true, force: true });
|
|
213
|
+
unlinkSync(journalFile);
|
|
214
|
+
fsyncDirectory(path.dirname(journalFile));
|
|
215
|
+
fsyncDirectory(parent);
|
|
216
|
+
return true;
|
|
217
|
+
}
|
|
218
|
+
if (!stagingExists)
|
|
219
|
+
fail("workspace_restore_journal_invalid");
|
|
220
|
+
if (!workspaceExists && backupExists)
|
|
221
|
+
renameSync(backup, options.workspaceDirectory);
|
|
222
|
+
else if (!workspaceExists && journal.hadWorkspace)
|
|
223
|
+
fail("workspace_restore_journal_invalid");
|
|
224
|
+
else if (workspaceExists && backupExists)
|
|
225
|
+
fail("workspace_restore_journal_invalid");
|
|
226
|
+
rmSync(staging, { recursive: true, force: true });
|
|
227
|
+
unlinkSync(journalFile);
|
|
228
|
+
fsyncDirectory(path.dirname(journalFile));
|
|
229
|
+
fsyncDirectory(parent);
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
export async function restoreWorkspaceSnapshot(options) {
|
|
233
|
+
const descriptor = options.descriptor;
|
|
234
|
+
assertDescriptor(descriptor);
|
|
235
|
+
assertControlPath(options.workspaceDirectory, options.controlDirectory);
|
|
236
|
+
assertWorkspaceRestoreCapacity(options);
|
|
237
|
+
recoverWorkspaceRestoreSwap({
|
|
238
|
+
workspaceDirectory: options.workspaceDirectory,
|
|
239
|
+
controlDirectory: options.controlDirectory,
|
|
240
|
+
descriptor,
|
|
241
|
+
});
|
|
242
|
+
const canonical = validateWorkspaceEntrySet(options.entrySet, options.limits);
|
|
243
|
+
if (canonical.entries.length !== descriptor.expectedEntryCount ||
|
|
244
|
+
canonical.fileCount !== descriptor.expectedFileCount ||
|
|
245
|
+
canonical.totalBytes !== descriptor.expectedTotalBytes ||
|
|
246
|
+
(descriptor.revision > 0 && canonical.contentSha256 !== descriptor.contentSha256))
|
|
247
|
+
fail("workspace_restore_integrity_failed");
|
|
248
|
+
const parent = path.dirname(options.workspaceDirectory);
|
|
249
|
+
const nonce = randomUUID();
|
|
250
|
+
const stagingName = `.botlearn-restore-${descriptor.runtimeSessionId}-${nonce}`;
|
|
251
|
+
const backupName = `.botlearn-backup-${descriptor.runtimeSessionId}-${nonce}`;
|
|
252
|
+
const staging = path.join(parent, stagingName);
|
|
253
|
+
const backup = path.join(parent, backupName);
|
|
254
|
+
mkdirSync(staging, { mode: 0o700 });
|
|
255
|
+
chmodSync(staging, 0o700);
|
|
256
|
+
let journalWritten = false;
|
|
257
|
+
try {
|
|
258
|
+
for (const entry of canonical.entries) {
|
|
259
|
+
const destination = path.join(staging, ...entry.path.split("/"));
|
|
260
|
+
if (entry.type === "directory") {
|
|
261
|
+
mkdirSync(destination, { recursive: true, mode: 0o700 });
|
|
262
|
+
chmodSync(destination, 0o700);
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
|
|
266
|
+
const part = `${destination}.part`;
|
|
267
|
+
await options.downloadFile(entry, part);
|
|
268
|
+
verifyDownloadedFile(part, {
|
|
269
|
+
expectedSize: entry.sizeBytes,
|
|
270
|
+
expectedSha256: entry.sha256,
|
|
271
|
+
});
|
|
272
|
+
chmodSync(part, entry.mode === "0700" ? 0o700 : 0o600);
|
|
273
|
+
renameSync(part, destination);
|
|
274
|
+
}
|
|
275
|
+
const directories = new Set([staging]);
|
|
276
|
+
for (const entry of canonical.entries) {
|
|
277
|
+
let directory = entry.type === "directory"
|
|
278
|
+
? path.join(staging, ...entry.path.split("/"))
|
|
279
|
+
: path.dirname(path.join(staging, ...entry.path.split("/")));
|
|
280
|
+
while (directory.startsWith(staging)) {
|
|
281
|
+
directories.add(directory);
|
|
282
|
+
if (directory === staging)
|
|
283
|
+
break;
|
|
284
|
+
directory = path.dirname(directory);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
for (const directory of [...directories].sort((a, b) => b.length - a.length)) {
|
|
288
|
+
fsyncDirectory(directory);
|
|
289
|
+
}
|
|
290
|
+
const journalFile = workspaceRestoreJournalPath(options.controlDirectory, descriptor.runtimeSessionId);
|
|
291
|
+
const journal = {
|
|
292
|
+
schemaVersion: RESTORE_JOURNAL_SCHEMA,
|
|
293
|
+
sandboxId: descriptor.sandboxId,
|
|
294
|
+
sandboxGeneration: descriptor.sandboxGeneration,
|
|
295
|
+
runtimeSessionId: descriptor.runtimeSessionId,
|
|
296
|
+
stagingName,
|
|
297
|
+
backupName,
|
|
298
|
+
hadWorkspace: existsSync(options.workspaceDirectory),
|
|
299
|
+
phase: "prepared",
|
|
300
|
+
marker: markerFromDescriptor(descriptor),
|
|
301
|
+
};
|
|
302
|
+
writeJournal(journalFile, journal);
|
|
303
|
+
journalWritten = true;
|
|
304
|
+
if (journal.hadWorkspace)
|
|
305
|
+
renameSync(options.workspaceDirectory, backup);
|
|
306
|
+
journal.phase = "old_moved";
|
|
307
|
+
writeJournal(journalFile, journal);
|
|
308
|
+
renameSync(staging, options.workspaceDirectory);
|
|
309
|
+
fsyncDirectory(parent);
|
|
310
|
+
journal.phase = "new_moved";
|
|
311
|
+
writeJournal(journalFile, journal);
|
|
312
|
+
writeWorkspaceMaterializationMarker(journal.marker);
|
|
313
|
+
if (journal.hadWorkspace)
|
|
314
|
+
rmSync(backup, { recursive: true, force: true });
|
|
315
|
+
unlinkSync(journalFile);
|
|
316
|
+
fsyncDirectory(path.dirname(journalFile));
|
|
317
|
+
fsyncDirectory(parent);
|
|
318
|
+
}
|
|
319
|
+
catch (error) {
|
|
320
|
+
if (!journalWritten)
|
|
321
|
+
rmSync(staging, { recursive: true, force: true });
|
|
322
|
+
throw error;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
export function assertWorkspaceRestoreCapacity(options) {
|
|
326
|
+
const { descriptor } = options;
|
|
327
|
+
assertDescriptor(descriptor);
|
|
328
|
+
if (!Number.isSafeInteger(options.requiredFreeBytes) ||
|
|
329
|
+
options.requiredFreeBytes < descriptor.expectedTotalBytes)
|
|
330
|
+
fail("workspace_restore_reservation_invalid");
|
|
331
|
+
const parent = path.dirname(options.workspaceDirectory);
|
|
332
|
+
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
333
|
+
if (existsSync(options.workspaceDirectory)) {
|
|
334
|
+
const current = lstatSync(options.workspaceDirectory, { bigint: true });
|
|
335
|
+
if (!current.isDirectory() || current.isSymbolicLink()) {
|
|
336
|
+
fail("workspace_restore_existing_workspace_invalid");
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
const filesystem = statfsSync(parent, { bigint: true });
|
|
340
|
+
if (filesystem.bavail * filesystem.bsize < BigInt(options.requiredFreeBytes)) {
|
|
341
|
+
fail("workspace_restore_disk_pressure");
|
|
342
|
+
}
|
|
343
|
+
const entryCopies = options.entryCopies ?? 1;
|
|
344
|
+
if (filesystem.ffree < BigInt(descriptor.expectedEntryCount * entryCopies + 16)) {
|
|
345
|
+
fail("workspace_restore_inode_pressure");
|
|
346
|
+
}
|
|
347
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { WorkspaceQuiescenceProof } from "./runtime-quiescence.js";
|
|
2
|
+
interface WorkspaceCheckpointScope {
|
|
3
|
+
checkpointId: string;
|
|
4
|
+
baseRevision: number;
|
|
5
|
+
sandboxId: string;
|
|
6
|
+
sandboxGeneration: number;
|
|
7
|
+
connectionEpoch: number;
|
|
8
|
+
runtimeSessionId: string;
|
|
9
|
+
agentRunId?: string;
|
|
10
|
+
workerAttempt?: number;
|
|
11
|
+
activationId?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare class WorkspaceSnapshotControlError extends Error {
|
|
14
|
+
readonly code: string;
|
|
15
|
+
readonly retryable: boolean;
|
|
16
|
+
constructor(code: string, retryable: boolean);
|
|
17
|
+
}
|
|
18
|
+
export declare function checkpointWorkspace(options: {
|
|
19
|
+
wsUrl: string;
|
|
20
|
+
reconnectToken: string;
|
|
21
|
+
workspaceDirectory: string;
|
|
22
|
+
scope: WorkspaceCheckpointScope;
|
|
23
|
+
quiescenceProof: WorkspaceQuiescenceProof;
|
|
24
|
+
}): Promise<{
|
|
25
|
+
revision: number;
|
|
26
|
+
snapshotId: string | null;
|
|
27
|
+
contentSha256: string;
|
|
28
|
+
}>;
|
|
29
|
+
export {};
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { createReadStream, rmSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { ensureDaemonHome } from "./auth-store.js";
|
|
4
|
+
import { writeWorkspaceMaterializationMarker } from "./workspace-materialization.js";
|
|
5
|
+
import { WORKSPACE_SNAPSHOT_POLICY_V1 } from "./workspace-snapshot-policy.js";
|
|
6
|
+
import { stageWorkspaceSnapshot } from "./workspace-snapshot-staging.js";
|
|
7
|
+
export class WorkspaceSnapshotControlError extends Error {
|
|
8
|
+
code;
|
|
9
|
+
retryable;
|
|
10
|
+
constructor(code, retryable) {
|
|
11
|
+
super(code);
|
|
12
|
+
this.code = code;
|
|
13
|
+
this.retryable = retryable;
|
|
14
|
+
this.name = "WorkspaceSnapshotControlError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function controlBase(wsUrl) {
|
|
18
|
+
const url = new URL(wsUrl);
|
|
19
|
+
url.protocol = url.protocol === "wss:" ? "https:" : "http:";
|
|
20
|
+
url.pathname = "/internal/v1/workspace-checkpoints";
|
|
21
|
+
url.search = "";
|
|
22
|
+
url.hash = "";
|
|
23
|
+
return url.toString().replace(/\/$/, "");
|
|
24
|
+
}
|
|
25
|
+
async function requestJson(url, token, body, signal) {
|
|
26
|
+
const response = await fetch(url, {
|
|
27
|
+
method: "POST",
|
|
28
|
+
headers: {
|
|
29
|
+
Authorization: `Bearer ${token}`,
|
|
30
|
+
"Content-Type": "application/json",
|
|
31
|
+
},
|
|
32
|
+
body: JSON.stringify(body),
|
|
33
|
+
signal,
|
|
34
|
+
});
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
let code = `workspace_snapshot_control_${response.status}`;
|
|
37
|
+
let explicitRetryable;
|
|
38
|
+
try {
|
|
39
|
+
const payload = await response.json();
|
|
40
|
+
const detail = payload.detail;
|
|
41
|
+
if (detail && typeof detail === "object" && !Array.isArray(detail)) {
|
|
42
|
+
const stable = detail.code;
|
|
43
|
+
if (typeof stable === "string" && /^workspace_[a-z0-9_]+$/u.test(stable))
|
|
44
|
+
code = stable;
|
|
45
|
+
const retryable = detail.retryable;
|
|
46
|
+
if (typeof retryable === "boolean")
|
|
47
|
+
explicitRetryable = retryable;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
// Content-free HTTP fallback remains stable and retry classification stays status-based.
|
|
52
|
+
}
|
|
53
|
+
throw new WorkspaceSnapshotControlError(code, explicitRetryable ?? (response.status >= 500 || response.status === 408 || response.status === 429 ||
|
|
54
|
+
response.status === 409));
|
|
55
|
+
}
|
|
56
|
+
return await response.json();
|
|
57
|
+
}
|
|
58
|
+
function fence(scope) {
|
|
59
|
+
return {
|
|
60
|
+
sandbox_id: scope.sandboxId,
|
|
61
|
+
sandbox_generation: scope.sandboxGeneration,
|
|
62
|
+
connection_epoch: scope.connectionEpoch,
|
|
63
|
+
runtime_session_id: scope.runtimeSessionId,
|
|
64
|
+
...(scope.agentRunId ? { agent_run_id: scope.agentRunId } : {}),
|
|
65
|
+
...(scope.workerAttempt ? { worker_attempt: scope.workerAttempt } : {}),
|
|
66
|
+
...(scope.activationId ? { activation_id: scope.activationId } : {}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export async function checkpointWorkspace(options) {
|
|
70
|
+
const { scope } = options;
|
|
71
|
+
const signal = AbortSignal.timeout(WORKSPACE_SNAPSHOT_POLICY_V1.checkpointAttemptDeadlineMs);
|
|
72
|
+
const checkpointUrl = `${controlBase(options.wsUrl)}/${scope.checkpointId}`;
|
|
73
|
+
const controlDirectory = path.join(ensureDaemonHome(), "agent-service-sandboxes", scope.sandboxId, "workspace-control");
|
|
74
|
+
const staged = stageWorkspaceSnapshot({
|
|
75
|
+
workspaceDirectory: options.workspaceDirectory,
|
|
76
|
+
controlDirectory,
|
|
77
|
+
checkpointId: scope.checkpointId,
|
|
78
|
+
limits: WORKSPACE_SNAPSHOT_POLICY_V1.limits,
|
|
79
|
+
maxStabilityRetries: WORKSPACE_SNAPSHOT_POLICY_V1.maxStabilityRetries,
|
|
80
|
+
quiescenceProof: options.quiescenceProof,
|
|
81
|
+
});
|
|
82
|
+
try {
|
|
83
|
+
const entrySet = {
|
|
84
|
+
schemaVersion: "agent-workspace-entry-set/1",
|
|
85
|
+
entries: staged.entrySet.entries,
|
|
86
|
+
};
|
|
87
|
+
const proposal = await requestJson(`${checkpointUrl}/proposal`, options.reconnectToken, {
|
|
88
|
+
fence: fence(scope),
|
|
89
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
90
|
+
base_revision: scope.baseRevision,
|
|
91
|
+
entry_count: staged.entrySet.entries.length,
|
|
92
|
+
file_count: staged.entrySet.fileCount,
|
|
93
|
+
total_bytes: staged.entrySet.totalBytes,
|
|
94
|
+
content_sha256: staged.entrySet.contentSha256,
|
|
95
|
+
entry_set: entrySet,
|
|
96
|
+
}, signal);
|
|
97
|
+
if (proposal.unchanged) {
|
|
98
|
+
if (proposal.revision === 0 && proposal.snapshot_id === null) {
|
|
99
|
+
writeWorkspaceMaterializationMarker({
|
|
100
|
+
schemaVersion: "agent-workspace-materialization/1",
|
|
101
|
+
sandboxId: scope.sandboxId,
|
|
102
|
+
sandboxGeneration: scope.sandboxGeneration,
|
|
103
|
+
runtimeSessionId: scope.runtimeSessionId,
|
|
104
|
+
workspaceContinuityState: "healthy",
|
|
105
|
+
snapshotId: null,
|
|
106
|
+
revision: 0,
|
|
107
|
+
contentSha256: null,
|
|
108
|
+
writtenAt: new Date().toISOString(),
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
revision: proposal.revision,
|
|
113
|
+
snapshotId: proposal.snapshot_id,
|
|
114
|
+
contentSha256: proposal.content_sha256,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
for (let offset = 0; offset < proposal.missing_blob_ids.length; offset += 25) {
|
|
118
|
+
const requestedBlobIds = proposal.missing_blob_ids.slice(offset, offset + 25);
|
|
119
|
+
const grants = await requestJson(`${checkpointUrl}/upload-grants`, options.reconnectToken, { fence: fence(scope), blob_ids: requestedBlobIds }, signal);
|
|
120
|
+
if (grants.length !== requestedBlobIds.length) {
|
|
121
|
+
throw new Error("workspace_snapshot_upload_grant_count_mismatch");
|
|
122
|
+
}
|
|
123
|
+
for (const item of grants) {
|
|
124
|
+
const source = staged.files.find((file) => file.sha256 === item.sha256 && file.sizeBytes === item.size_bytes);
|
|
125
|
+
if (!source)
|
|
126
|
+
throw new Error("workspace_snapshot_upload_source_missing");
|
|
127
|
+
const response = await fetch(item.grant.url, {
|
|
128
|
+
method: item.grant.method,
|
|
129
|
+
headers: item.grant.headers,
|
|
130
|
+
body: createReadStream(source.stagingPath),
|
|
131
|
+
duplex: "half",
|
|
132
|
+
signal,
|
|
133
|
+
});
|
|
134
|
+
if (!response.ok)
|
|
135
|
+
throw new Error(`workspace_snapshot_upload_${response.status}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
let committed = await requestJson(`${checkpointUrl}/complete`, options.reconnectToken, { fence: fence(scope), policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId }, signal);
|
|
139
|
+
while (committed.status === "pending") {
|
|
140
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
141
|
+
committed = await requestJson(`${checkpointUrl}/status`, options.reconnectToken, { fence: fence(scope) }, signal);
|
|
142
|
+
}
|
|
143
|
+
if (committed.status !== "committed" ||
|
|
144
|
+
committed.snapshot_id === null ||
|
|
145
|
+
committed.revision === null ||
|
|
146
|
+
committed.content_sha256 === null) {
|
|
147
|
+
throw new WorkspaceSnapshotControlError(committed.error_code || "workspace_snapshot_verification_failed", false);
|
|
148
|
+
}
|
|
149
|
+
writeWorkspaceMaterializationMarker({
|
|
150
|
+
schemaVersion: "agent-workspace-materialization/1",
|
|
151
|
+
sandboxId: scope.sandboxId,
|
|
152
|
+
sandboxGeneration: scope.sandboxGeneration,
|
|
153
|
+
runtimeSessionId: scope.runtimeSessionId,
|
|
154
|
+
workspaceContinuityState: "healthy",
|
|
155
|
+
snapshotId: committed.snapshot_id,
|
|
156
|
+
revision: committed.revision,
|
|
157
|
+
contentSha256: committed.content_sha256,
|
|
158
|
+
writtenAt: new Date().toISOString(),
|
|
159
|
+
});
|
|
160
|
+
return {
|
|
161
|
+
revision: committed.revision,
|
|
162
|
+
snapshotId: committed.snapshot_id,
|
|
163
|
+
contentSha256: committed.content_sha256,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
finally {
|
|
167
|
+
rmSync(staged.stagingDirectory, { recursive: true, force: true });
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Protocol limits frozen with Course Service's WorkspaceSnapshotPolicyV1.
|
|
3
|
+
* Deployment may enable the capability, but must never override these values.
|
|
4
|
+
*/
|
|
5
|
+
export declare const WORKSPACE_SNAPSHOT_POLICY_V1: {
|
|
6
|
+
readonly policyId: "WorkspaceSnapshotPolicyV1";
|
|
7
|
+
readonly limits: {
|
|
8
|
+
maxPathSegmentBytes: number;
|
|
9
|
+
maxPathBytes: number;
|
|
10
|
+
maxDepth: number;
|
|
11
|
+
maxFileBytes: number;
|
|
12
|
+
maxFileCount: number;
|
|
13
|
+
maxTotalBytes: number;
|
|
14
|
+
maxEntrySetBytes: number;
|
|
15
|
+
};
|
|
16
|
+
readonly maxStabilityRetries: 2;
|
|
17
|
+
readonly checkpointAttemptDeadlineMs: 30000;
|
|
18
|
+
readonly requiredRestoreFreeBytes: number;
|
|
19
|
+
readonly sessionRetainedHardLimitBytes: number;
|
|
20
|
+
readonly checkpointReservationBytes: number;
|
|
21
|
+
readonly restorePageEntries: 25;
|
|
22
|
+
};
|
|
23
|
+
/** Wire representation validated exactly by Agent Service during the v0.3 handshake. */
|
|
24
|
+
export declare function workspaceSnapshotPolicyCapability(): Record<string, unknown>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { RUNTIME_WORKSPACE_INODE_QUOTA, RUNTIME_WORKSPACE_QUOTA_BYTES, } from "./workspace-quota.js";
|
|
2
|
+
/**
|
|
3
|
+
* Protocol limits frozen with Course Service's WorkspaceSnapshotPolicyV1.
|
|
4
|
+
* Deployment may enable the capability, but must never override these values.
|
|
5
|
+
*/
|
|
6
|
+
export const WORKSPACE_SNAPSHOT_POLICY_V1 = {
|
|
7
|
+
policyId: "WorkspaceSnapshotPolicyV1",
|
|
8
|
+
limits: {
|
|
9
|
+
maxPathSegmentBytes: 255,
|
|
10
|
+
maxPathBytes: 1024,
|
|
11
|
+
maxDepth: 32,
|
|
12
|
+
maxFileBytes: 50 * 1024 * 1024,
|
|
13
|
+
maxFileCount: 2_000,
|
|
14
|
+
maxTotalBytes: 512 * 1024 * 1024,
|
|
15
|
+
maxEntrySetBytes: 1024 * 1024,
|
|
16
|
+
},
|
|
17
|
+
maxStabilityRetries: 2,
|
|
18
|
+
checkpointAttemptDeadlineMs: 30_000,
|
|
19
|
+
requiredRestoreFreeBytes: 1024 * 1024,
|
|
20
|
+
sessionRetainedHardLimitBytes: 2 * 1024 * 1024 * 1024,
|
|
21
|
+
checkpointReservationBytes: 512 * 1024 * 1024,
|
|
22
|
+
restorePageEntries: 25,
|
|
23
|
+
};
|
|
24
|
+
/** Wire representation validated exactly by Agent Service during the v0.3 handshake. */
|
|
25
|
+
export function workspaceSnapshotPolicyCapability() {
|
|
26
|
+
return {
|
|
27
|
+
policy_id: WORKSPACE_SNAPSHOT_POLICY_V1.policyId,
|
|
28
|
+
max_path_segment_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxPathSegmentBytes,
|
|
29
|
+
max_path_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxPathBytes,
|
|
30
|
+
max_depth: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxDepth,
|
|
31
|
+
max_file_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxFileBytes,
|
|
32
|
+
max_file_count: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxFileCount,
|
|
33
|
+
max_total_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxTotalBytes,
|
|
34
|
+
max_entry_set_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.limits.maxEntrySetBytes,
|
|
35
|
+
required_restore_free_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.requiredRestoreFreeBytes,
|
|
36
|
+
session_retained_hard_limit_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.sessionRetainedHardLimitBytes,
|
|
37
|
+
checkpoint_reservation_bytes: WORKSPACE_SNAPSHOT_POLICY_V1.checkpointReservationBytes,
|
|
38
|
+
restore_page_entries: WORKSPACE_SNAPSHOT_POLICY_V1.restorePageEntries,
|
|
39
|
+
max_stability_retries: WORKSPACE_SNAPSHOT_POLICY_V1.maxStabilityRetries,
|
|
40
|
+
checkpoint_attempt_deadline_ms: WORKSPACE_SNAPSHOT_POLICY_V1.checkpointAttemptDeadlineMs,
|
|
41
|
+
prepare_deadline_seconds: 15 * 60,
|
|
42
|
+
runtime_filesystem_quota_bytes: RUNTIME_WORKSPACE_QUOTA_BYTES,
|
|
43
|
+
runtime_inode_quota: RUNTIME_WORKSPACE_INODE_QUOTA,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type WorkspaceQuiescenceProof } from "./runtime-quiescence.js";
|
|
2
|
+
import { type CanonicalWorkspaceEntrySet, type WorkspaceEntrySetLimits } from "./workspace-entry-set.js";
|
|
3
|
+
export declare class WorkspaceSnapshotStagingError extends Error {
|
|
4
|
+
readonly code: string;
|
|
5
|
+
readonly retryable: boolean;
|
|
6
|
+
constructor(code: string, retryable?: boolean);
|
|
7
|
+
}
|
|
8
|
+
export interface StagedWorkspaceFile {
|
|
9
|
+
path: string;
|
|
10
|
+
stagingPath: string;
|
|
11
|
+
sizeBytes: number;
|
|
12
|
+
sha256: string;
|
|
13
|
+
}
|
|
14
|
+
export interface StagedWorkspaceSnapshot {
|
|
15
|
+
stagingDirectory: string;
|
|
16
|
+
entrySet: CanonicalWorkspaceEntrySet;
|
|
17
|
+
files: StagedWorkspaceFile[];
|
|
18
|
+
}
|
|
19
|
+
export interface WorkspaceSnapshotStagingOptions {
|
|
20
|
+
workspaceDirectory: string;
|
|
21
|
+
controlDirectory: string;
|
|
22
|
+
checkpointId: string;
|
|
23
|
+
limits: WorkspaceEntrySetLimits;
|
|
24
|
+
maxStabilityRetries: number;
|
|
25
|
+
quiescenceProof: WorkspaceQuiescenceProof;
|
|
26
|
+
}
|
|
27
|
+
export declare function stageWorkspaceSnapshot(options: WorkspaceSnapshotStagingOptions): StagedWorkspaceSnapshot;
|