@opengeni/worker-bundle 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7113 -1652
  3. package/dist/index.js.map +1 -1
  4. package/package.json +18 -18
  5. package/src/activities/agent-turn.ts +4337 -1052
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
@@ -0,0 +1,964 @@
1
+ // Workbench v2 — turn-end workspace capture (dossier §10.1).
2
+ //
3
+ // The universal spine that makes the session workspace paint instantly on cold /
4
+ // offline reads: at TURN END, while the box is still guaranteed live and the
5
+ // turn's lease pins the refcount (the reaper cannot drain), we probe the box's
6
+ // CHANGED files directly off the un-proxied Channel-A session — per-repo git
7
+ // status + diff, after-images of the touched files, and a pruned tree index —
8
+ // serialize a manifest, PUT it + the after-image blobs to object storage, and
9
+ // insert an epoch-fenced `workspace_captures` row. The read path (M2/M3) serves
10
+ // this when the machine is cold/offline; a warm box always wins (capture is a
11
+ // labelled cache, never a replacement).
12
+ //
13
+ // SAFETY INVARIANTS (non-negotiable — see dossier §7.3 / §19):
14
+ // • This module NEVER calls close()/terminate()/kill() on the session handle.
15
+ // session.close() == terminate() on Modal and would kill the user's box.
16
+ // Only the reaper terminates. We drop references, nothing more.
17
+ // • The whole capture is time-capped (60s) and best-effort: ANY failure logs
18
+ // "workspace capture failed — turn outcome unaffected" and returns; nothing
19
+ // throws past captureWorkspaceRevision.
20
+ // • The DB write is fenced on the lease epoch — a superseded lease writes zero
21
+ // rows (insertWorkspaceCapture).
22
+ // • F9 storage ordering: blobs + manifest are PUT and the row is committed
23
+ // BEFORE any GC delete runs.
24
+ //
25
+ // It is deliberately an EXTERNAL module with a SINGLE call-site line in
26
+ // agent-turn.ts's finally (dossier "Accepted warts" #2 — do not grow the turn
27
+ // activity). The emitted `workspace.revision.captured` event is ANNOUNCE-ONLY
28
+ // (metadata, never content) and must never gain a rendered timeline item.
29
+
30
+ import { createHash, randomUUID } from "node:crypto";
31
+ import type { Settings } from "@opengeni/config";
32
+ import type { Database } from "@opengeni/db";
33
+ import {
34
+ deleteWorkspaceCaptureRows,
35
+ insertFailedWorkspaceCapture,
36
+ insertWorkspaceCapture,
37
+ latestWorkspaceCapture,
38
+ planWorkspaceCaptureGc,
39
+ } from "@opengeni/db";
40
+ import type { ObjectStorage } from "@opengeni/storage";
41
+ import type { Observability } from "@opengeni/observability";
42
+ // The un-agent-loop leaf. Constructing the Channel-A service over the un-proxied
43
+ // setupBoxSession here (NOT the routing veneer) guarantees a mid-turn
44
+ // sandbox_swap can never re-route capture execs onto a user's machine — the same
45
+ // reason the snapshot uses setupBoxSession (dossier §7.3).
46
+ import {
47
+ SandboxChannelAService,
48
+ type ChannelASession,
49
+ type RepositoryDiscoveryDegradedReason,
50
+ } from "@opengeni/runtime/sandbox";
51
+ import type {
52
+ FsTreeNode,
53
+ GitFileStatus,
54
+ GitFileStatusCode,
55
+ SessionEvent,
56
+ WorkspaceCaptureFile,
57
+ WorkspaceCaptureDegradedReason,
58
+ WorkspaceCaptureManifest,
59
+ WorkspaceCaptureRepo,
60
+ WorkspaceCaptureStats,
61
+ } from "@opengeni/contracts";
62
+
63
+ // ─── Guards & constants (dossier §10.1 — pathological-only; configurable) ─────
64
+ export const CAPTURE_TIMEOUT_MS = 60_000;
65
+ export const PER_FILE_CONTENT_GUARD_BYTES = 5 * 1024 * 1024; // after-image; over → tooLarge marker
66
+ export const PER_FILE_DIFF_GUARD_BYTES = 10 * 1024 * 1024; // per-file diff; over → truncated marker
67
+ export const WHOLE_CAPTURE_GUARD_BYTES = 200 * 1024 * 1024; // total → skip, fall back to live
68
+ export const KEEP_LATEST_REVISIONS = 10;
69
+ // Directories listed as collapsed nodes in the tree index but NEVER descended
70
+ // (their contents live on the machine — the Files tab wakes to expand them).
71
+ //
72
+ // Two classes:
73
+ // • BUILD/DEP residue — huge, machine-resident, never review content.
74
+ // • DESKTOP/SYSTEM residue (the Modal desktop-box fix): on a desktop image the
75
+ // workspace root IS $HOME, and XFCE/dbus/etc. CONTINUOUSLY rewrite dotfiles
76
+ // under ~/.config/xfce4, ~/.cache, ~/.dbus, … A tree walk that descends into
77
+ // them (a) wastes the round-trip budget on churn no user is reviewing and
78
+ // (b) races files that VANISH mid-walk. These are never workspace content, so
79
+ // collapse them at the source. Legit hidden entries a user DOES author
80
+ // (.gitignore, .env, .github, .vscode, .devcontainer) are deliberately NOT
81
+ // here — they stay fully visible.
82
+ export const RESIDUE_DIRS: readonly string[] = [
83
+ // build/dep residue
84
+ "node_modules",
85
+ ".git",
86
+ // Platform credential/helper state. The workspace root can be $HOME, so this
87
+ // directory may sit inside a root repository; it must never enter a revision.
88
+ ".opengeni",
89
+ "dist",
90
+ "build",
91
+ "target",
92
+ ".venv",
93
+ "__pycache__",
94
+ ".next",
95
+ // desktop/system residue (never workspace content; churned by the desktop stack)
96
+ ".config",
97
+ ".cache",
98
+ ".local",
99
+ ".dbus",
100
+ ".gnupg",
101
+ ".ssh",
102
+ ".mozilla",
103
+ ".xfce4",
104
+ ".pki",
105
+ ".gvfs",
106
+ ".dbus-keyrings",
107
+ ".Xauthority",
108
+ ".ICEauthority",
109
+ ];
110
+ const RESIDUE_DIR_SET: ReadonlySet<string> = new Set(RESIDUE_DIRS);
111
+
112
+ /**
113
+ * True when a workspace-relative FILE path lives INSIDE a residue dir — i.e. a
114
+ * residue dir is one of its ANCESTOR segments (not the leaf), so the file is
115
+ * churn/machine content never worth an after-image. `.config/xfce4/xfconf/…` and
116
+ * `.config/mimeapps.list` → true; a root FILE literally named `.config` (a user
117
+ * config the seed edits) → false (single segment, no residue ancestor); and
118
+ * `.github/x.yml`, `.gitignore` → false (`.github`/`.gitignore` are not residue).
119
+ * Matches the tree-BFS collapse (which collapses residue DIR nodes) so the
120
+ * Changes tab and the tree agree on what is workspace content.
121
+ */
122
+ export function isUnderResidueDir(wsPath: string): boolean {
123
+ const segments = wsPath.split("/");
124
+ // Check ancestors only (every segment except the leaf): the leaf is the file
125
+ // itself; a residue-named file at the root is still legitimate content.
126
+ for (let i = 0; i < segments.length - 1; i++) {
127
+ const seg = segments[i];
128
+ if (seg && RESIDUE_DIR_SET.has(seg)) return true;
129
+ }
130
+ return false;
131
+ }
132
+
133
+ /**
134
+ * The box is being torn down under us (Modal reclaim / reaper drain / terminate)
135
+ * — NOT a single file vanishing. Every subsequent op will fail too, so the
136
+ * capture must ABORT (throw) rather than skip-and-continue, so it never commits a
137
+ * bogus empty/partial revision for a dead box. Matched loosely so a provider
138
+ * wording tweak still classifies. (The real fix for this is keeping the box
139
+ * pinned through the capture window — sandbox-resume / agent-turn lease
140
+ * heartbeat; this classifier only guarantees we fail HONESTLY if it still races.)
141
+ */
142
+ export function isBoxExitingError(error: unknown): boolean {
143
+ const msg = (error instanceof Error ? error.message : String(error ?? "")).toLowerCase();
144
+ return (
145
+ msg.includes("container exiting") ||
146
+ msg.includes("container is exiting") ||
147
+ msg.includes("container exited") ||
148
+ msg.includes("sandbox has been terminated") ||
149
+ msg.includes("sandbox is not running") ||
150
+ msg.includes("sandbox has terminated") ||
151
+ msg.includes("task has exited")
152
+ );
153
+ }
154
+
155
+ /** Marker thrown when the box died mid-capture — aborts cleanly (no partial row). */
156
+ export class BoxExitingError extends Error {
157
+ constructor(message: string) {
158
+ super(message);
159
+ this.name = "BoxExitingError";
160
+ }
161
+ }
162
+
163
+ /** Re-throw as a BoxExitingError when the box is gone; otherwise return null so
164
+ * the caller SKIPS this single entry (a vanished/unreadable file) and continues.
165
+ * One churning file must never kill the capture; a dead box must never yield a
166
+ * bogus revision. */
167
+ function classifyCaptureEntryError(error: unknown): BoxExitingError | null {
168
+ if (isBoxExitingError(error)) {
169
+ return new BoxExitingError(error instanceof Error ? error.message : String(error));
170
+ }
171
+ return null;
172
+ }
173
+ const TREE_MAX_ENTRIES = 20_000; // whole-tree node cap (truncate beyond)
174
+ const TREE_MAX_DIRS = 600; // BFS round-trip cap (bounds turn-end latency)
175
+
176
+ // The capture row and announcement are committed together by @opengeni/db. This
177
+ // callback performs only best-effort live fanout of those already-durable events.
178
+ export type CaptureEventPublisher = (events: SessionEvent[]) => Promise<void>;
179
+
180
+ export type CaptureWorkspaceRevisionInput = {
181
+ db: Database;
182
+ objectStorage: ObjectStorage | null;
183
+ settings: Settings;
184
+ publish: CaptureEventPublisher | null;
185
+ /** The un-proxied setupBoxSession (NOT the routing veneer). */
186
+ session: ChannelASession;
187
+ leaseEpoch: number;
188
+ sandboxGroupId: string;
189
+ accountId: string;
190
+ workspaceId: string;
191
+ sessionId: string;
192
+ turnId: string | null;
193
+ observability: Observability;
194
+ /** Test-only: override keep-latest-N GC threshold (default 10). */
195
+ keepLatest?: number;
196
+ };
197
+
198
+ let loggedStorageNullOnce = false;
199
+
200
+ /**
201
+ * Turn-end entrypoint. Gates on the flag + configured object storage, races the
202
+ * whole capture against a 60s cap, and swallows every failure ("turn outcome
203
+ * unaffected"). Returns void — the caller awaits it (it self-caps) and moves on
204
+ * to release() regardless.
205
+ */
206
+ export async function captureWorkspaceRevision(
207
+ input: CaptureWorkspaceRevisionInput,
208
+ ): Promise<void> {
209
+ const { observability } = input;
210
+ if (!input.settings.workspaceCaptureEnabled) {
211
+ return; // flag off → capture skipped; reads fall back to live/wake (status quo)
212
+ }
213
+ if (!input.objectStorage) {
214
+ if (!loggedStorageNullOnce) {
215
+ loggedStorageNullOnce = true;
216
+ observability.info("workspace capture skipped — object storage not configured");
217
+ }
218
+ observability.incrementCounter({
219
+ name: "opengeni_workspace_capture_total",
220
+ labels: { result: "skipped_no_storage" },
221
+ });
222
+ return;
223
+ }
224
+
225
+ const startedAt = Date.now();
226
+ const controller = new AbortController();
227
+ observability.incrementGauge({
228
+ name: "opengeni_workspace_captures_inflight",
229
+ help: "Current workspace-capture operations still resident in this worker process.",
230
+ });
231
+ const capture = runCapture(
232
+ input,
233
+ { ...input, objectStorage: input.objectStorage },
234
+ startedAt,
235
+ controller.signal,
236
+ ).finally(() => {
237
+ observability.incrementGauge({
238
+ name: "opengeni_workspace_captures_inflight",
239
+ help: "Current workspace-capture operations still resident in this worker process.",
240
+ amount: -1,
241
+ });
242
+ });
243
+ let timer: ReturnType<typeof setTimeout> | undefined;
244
+ try {
245
+ await Promise.race([
246
+ capture,
247
+ new Promise<never>((_, reject) => {
248
+ timer = setTimeout(() => {
249
+ controller.abort();
250
+ reject(new Error(`workspace capture exceeded ${CAPTURE_TIMEOUT_MS}ms`));
251
+ }, CAPTURE_TIMEOUT_MS);
252
+ }),
253
+ ]);
254
+ } catch (error) {
255
+ // The one and only place a capture failure surfaces — as a log line, never
256
+ // an exception past this boundary (the turn already completed).
257
+ observability.warn("workspace capture failed — turn outcome unaffected", {
258
+ "opengeni.session_id": input.sessionId,
259
+ "opengeni.turn_id": input.turnId ?? "",
260
+ "error.message": error instanceof Error ? error.message : String(error),
261
+ "workspace_capture.duration_ms": Date.now() - startedAt,
262
+ });
263
+ observability.incrementCounter({
264
+ name: "opengeni_workspace_capture_total",
265
+ labels: { result: "failed" },
266
+ });
267
+ } finally {
268
+ if (timer) clearTimeout(timer);
269
+ controller.abort();
270
+ }
271
+ }
272
+
273
+ // The strongly-typed inner run (objectStorage proven non-null by the caller).
274
+ async function runCapture(
275
+ input: CaptureWorkspaceRevisionInput,
276
+ ctx: CaptureWorkspaceRevisionInput & { objectStorage: ObjectStorage },
277
+ startedAt: number,
278
+ signal: AbortSignal,
279
+ ): Promise<void> {
280
+ const { observability } = input;
281
+ const storage = ctx.objectStorage;
282
+ const keepN = input.keepLatest ?? KEEP_LATEST_REVISIONS;
283
+ // Reads only — no emitter (we publish the announce ourselves after commit).
284
+ const svc = new SandboxChannelAService({ session: input.session, leaseEpoch: input.leaseEpoch });
285
+
286
+ // Resolve the previous revision before discovery so even a failed discovery
287
+ // can commit a monotonic, explicit degraded marker. That marker becomes the
288
+ // newest read result and prevents an older successful capture from being
289
+ // mistaken for the current turn's authoritative workspace state.
290
+ const prev = await latestWorkspaceCapture(input.db, input.workspaceId, input.sessionId);
291
+ throwIfCaptureAborted(signal);
292
+ const revision = (prev?.revision ?? -1) + 1;
293
+
294
+ // ── 1. per-repo status + diff, union the touched set ──────────────────────
295
+ const discovery = await svc.detectReposDetailed();
296
+ throwIfCaptureAborted(signal);
297
+ if (!discovery.complete) {
298
+ const capturedAt = new Date();
299
+ const reason = captureDegradedReason(discovery.degradedReason);
300
+ const stats = {
301
+ degradedReason: reason,
302
+ discoveredRepoCount: discovery.repos.length,
303
+ durationMs: Date.now() - startedAt,
304
+ };
305
+ const inserted = await insertFailedWorkspaceCapture(input.db, {
306
+ accountId: input.accountId,
307
+ workspaceId: input.workspaceId,
308
+ sessionId: input.sessionId,
309
+ turnId: input.turnId,
310
+ sandboxGroupId: input.sandboxGroupId,
311
+ expectedEpoch: input.leaseEpoch,
312
+ revision,
313
+ stats,
314
+ capturedAt,
315
+ });
316
+ throwIfCaptureAborted(signal);
317
+ if (!inserted) {
318
+ observability.incrementCounter({
319
+ name: "opengeni_workspace_capture_total",
320
+ labels: { result: "superseded" },
321
+ });
322
+ return;
323
+ }
324
+ observability.warn("workspace capture degraded — repository discovery incomplete", {
325
+ "opengeni.session_id": input.sessionId,
326
+ "opengeni.turn_id": input.turnId ?? "",
327
+ "workspace_capture.degraded_reason": reason,
328
+ "workspace_capture.discovered_repo_count": discovery.repos.length,
329
+ });
330
+ observability.incrementCounter({
331
+ name: "opengeni_workspace_capture_total",
332
+ labels: { result: "degraded_repository_discovery" },
333
+ });
334
+ if (input.publish) {
335
+ await input.publish(inserted.events).catch(() => undefined);
336
+ }
337
+ return;
338
+ }
339
+ const repoRoots = discovery.repos;
340
+ const repos: WorkspaceCaptureRepo[] = [];
341
+ // workspace-relative path → touched-file descriptor
342
+ const touched = new Map<string, { status: GitFileStatusCode; deleted: boolean }>();
343
+ let additions = 0;
344
+ let deletions = 0;
345
+
346
+ for (const root of repoRoots) {
347
+ throwIfCaptureAborted(signal);
348
+ // Per-repo resilience: a box death aborts the whole capture (BoxExitingError
349
+ // propagates); any other single-repo failure (a repo that vanished, a
350
+ // transient git error) SKIPS that repo and the capture continues with the
351
+ // rest. One flaky repo must never kill the capture.
352
+ let status: Awaited<ReturnType<typeof svc.gitStatus>>;
353
+ try {
354
+ status = await svc.gitStatus({ path: root });
355
+ throwIfCaptureAborted(signal);
356
+ } catch (error) {
357
+ const boxExiting = classifyCaptureEntryError(error);
358
+ if (boxExiting) throw boxExiting;
359
+ continue;
360
+ }
361
+ if (!status.isRepo) continue;
362
+ // `git diff HEAD`: combined staged+unstaged tracked changes vs HEAD (the
363
+ // review diff). Untracked files are absent here but present in status.files
364
+ // and captured as after-images. A repo with no commits yields an empty diff
365
+ // (git diff HEAD errors → empty numstat) and everything reads as untracked.
366
+ let diff: Awaited<ReturnType<typeof svc.gitDiff>>;
367
+ try {
368
+ diff = await svc.gitDiff({
369
+ path: root,
370
+ staged: false,
371
+ fromRef: "HEAD",
372
+ pathspec: [],
373
+ contextLines: 3,
374
+ maxBytesPerFile: PER_FILE_DIFF_GUARD_BYTES,
375
+ });
376
+ throwIfCaptureAborted(signal);
377
+ } catch (error) {
378
+ const boxExiting = classifyCaptureEntryError(error);
379
+ if (boxExiting) throw boxExiting;
380
+ diff = { files: [], revision: 0 };
381
+ }
382
+ // Drop residue churn from the review diff too. On a desktop box the seed's
383
+ // `git add -A` in $HOME commits ~/.config/xfce4/* into HEAD, so `git diff HEAD`
384
+ // lists the continuously-rewritten desktop dotfiles as changed — noise the
385
+ // Changes tab must not show. The status/after-image loop already excludes them
386
+ // (isUnderResidueDir); exclude them here so the diff surface agrees.
387
+ const diffFiles = diff.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
388
+ // Also drop residue-only status entries below; status.files is filtered in the
389
+ // touched loop (isUnderResidueDir), so `status: status.files` would still carry
390
+ // residue rows — filter for a consistent captured status surface.
391
+ const statusFiles = status.files.filter((f) => !isUnderResidueDir(joinRepoPath(root, f.path)));
392
+ repos.push({
393
+ root,
394
+ head: status.head,
395
+ detached: status.detached,
396
+ upstream: status.upstream,
397
+ ahead: status.ahead,
398
+ behind: status.behind,
399
+ status: statusFiles,
400
+ diff: diffFiles,
401
+ });
402
+ for (const f of diffFiles) {
403
+ additions += f.additions;
404
+ deletions += f.deletions;
405
+ }
406
+ for (const f of status.files) {
407
+ const wsPath = joinRepoPath(root, f.path);
408
+ // Skip desktop/system residue churn (~/.config/xfce4/…): on a desktop box
409
+ // the workspace root is $HOME, so git reports the continuously-rewritten
410
+ // XFCE/cache dotfiles as untracked. They are never review content — dropping
411
+ // them here keeps the after-image loop off churn AND matches the tree BFS's
412
+ // residue collapse (Changes tab and tree agree on what is workspace content).
413
+ if (isUnderResidueDir(wsPath)) continue;
414
+ touched.set(wsPath, {
415
+ status: statusCodeOf(f),
416
+ deleted: f.worktree === "deleted" || f.index === "deleted",
417
+ });
418
+ }
419
+ }
420
+
421
+ // Previous revision (for the empty-turn gate + revision assignment). The gate
422
+ // itself fires AFTER the after-image loop, once the change fingerprint is known
423
+ // (§10.1 / B3): uncommitted changes persist in the box across turns, so a
424
+ // literal "skip when git status is clean" gate would re-capture an identical
425
+ // dirty tree on every read-only turn. Instead we skip when the change surface
426
+ // is byte-identical to the previous revision — "no new revision when nothing
427
+ // changed" holds even with a persistently dirty tree, and content-addressed
428
+ // blobs already dedupe storage. (Deviation from the dossier's literal "clean"
429
+ // wording — same intent, strictly better; recorded in PROGRESS.)
430
+ // ── 3. after-images of touched files (size-gated), content-addressed ───────
431
+ const files: WorkspaceCaptureFile[] = [];
432
+ const blobKeys = new Set<string>();
433
+ let totalBytes = 0;
434
+ let tooLargeCount = 0;
435
+ let binaryCount = 0;
436
+
437
+ for (const [wsPath, info] of touched) {
438
+ throwIfCaptureAborted(signal);
439
+ if (info.deleted) {
440
+ files.push({
441
+ path: wsPath,
442
+ status: "deleted",
443
+ hash: null,
444
+ baseHash: null,
445
+ contentRef: null,
446
+ sizeBytes: 0,
447
+ isBinary: false,
448
+ tooLarge: false,
449
+ deleted: true,
450
+ });
451
+ continue;
452
+ }
453
+ // Always request base64 so we get raw bytes uniformly (text or binary) for
454
+ // content-addressing. maxBytes caps the read at the 5MB guard; `truncated`
455
+ // means the file is ≥5MB → we record a tooLarge marker (no content blob).
456
+ //
457
+ // Per-file resilience: a file the box reported as changed can VANISH or turn
458
+ // unreadable between git-status and this read (fs churn, a follow-up mutation,
459
+ // a race). That must NEVER abort the whole capture — skip the one entry and
460
+ // keep the rest. A box-death error is different: it aborts (BoxExitingError),
461
+ // because every remaining read would fail too and we must not commit a partial
462
+ // revision for a dead box.
463
+ let read: Awaited<ReturnType<typeof svc.fsRead>>;
464
+ try {
465
+ read = await svc.fsRead({
466
+ path: wsPath,
467
+ encoding: "base64",
468
+ maxBytes: PER_FILE_CONTENT_GUARD_BYTES,
469
+ });
470
+ throwIfCaptureAborted(signal);
471
+ } catch (error) {
472
+ const boxExiting = classifyCaptureEntryError(error);
473
+ if (boxExiting) throw boxExiting;
474
+ continue; // vanished/unreadable single file — omit it, capture continues
475
+ }
476
+ if (read.truncated) {
477
+ tooLargeCount += 1;
478
+ files.push({
479
+ path: wsPath,
480
+ status: info.status,
481
+ hash: null,
482
+ baseHash: null,
483
+ contentRef: null,
484
+ sizeBytes: read.sizeBytes,
485
+ isBinary: read.isBinary,
486
+ tooLarge: true,
487
+ deleted: false,
488
+ });
489
+ continue;
490
+ }
491
+ const bytes = Buffer.from(read.content, "base64");
492
+ const hash = sha256(bytes);
493
+ const contentRef = blobKey(input.workspaceId, input.sessionId, hash);
494
+ if (read.isBinary) binaryCount += 1;
495
+ if (!blobKeys.has(contentRef)) {
496
+ totalBytes += bytes.byteLength;
497
+ // Whole-capture guard: bail before writing anything (fall back to live).
498
+ if (totalBytes > WHOLE_CAPTURE_GUARD_BYTES) {
499
+ observability.warn("workspace capture skipped — whole-capture guard tripped", {
500
+ "opengeni.session_id": input.sessionId,
501
+ "workspace_capture.total_bytes": totalBytes,
502
+ });
503
+ observability.incrementCounter({
504
+ name: "opengeni_workspace_capture_total",
505
+ labels: { result: "guard_tripped" },
506
+ });
507
+ return;
508
+ }
509
+ }
510
+ blobKeys.add(contentRef);
511
+ files.push({
512
+ // baseHash (git HEAD blob sha) is intentionally null in M1: the wake-on-edit
513
+ // flush guard (M3/C2) compares live content against the after-image `hash`,
514
+ // not the HEAD blob, so baseHash is not load-bearing yet. Populating it would
515
+ // cost an extra per-file round-trip at turn end (latency risk #1). Deferred.
516
+ path: wsPath,
517
+ status: info.status,
518
+ hash,
519
+ baseHash: null,
520
+ contentRef,
521
+ sizeBytes: bytes.byteLength,
522
+ isBinary: read.isBinary,
523
+ tooLarge: false,
524
+ deleted: false,
525
+ });
526
+ }
527
+
528
+ // ── 4. empty-turn gate (dossier §10.1 / B3) ───────────────────────────────
529
+ // Skip when the change surface is byte-identical to the previous revision.
530
+ // Fires BEFORE the tree BFS + storage PUTs (the expensive parts) so a no-op
531
+ // read-only turn on a persistently dirty tree costs only the (small) status/
532
+ // diff/after-image probes.
533
+ let fingerprint = changeFingerprint(repos, files);
534
+ if (prev && prev.stats.fingerprint === fingerprint) {
535
+ observability.incrementCounter({
536
+ name: "opengeni_workspace_capture_total",
537
+ labels: { result: "skipped_empty" },
538
+ });
539
+ return;
540
+ }
541
+
542
+ // ── 5. tree index (bounded BFS; residue dirs collapsed, not descended) ─────
543
+ const tree = await buildTreeIndex(svc, startedAt, signal);
544
+ throwIfCaptureAborted(signal);
545
+
546
+ // ── 6. PUT blobs + tree + manifest (F9: all writes BEFORE any delete) ──────
547
+ // Key manifest/tree by the turn (one capture per turn) so the key is known
548
+ // before the revision is committed; content blobs are content-addressed.
549
+ const turnKey = input.turnId ?? randomUUID();
550
+ const treeKey = `workspace-captures/${input.workspaceId}/${input.sessionId}/trees/${turnKey}.json`;
551
+ const manifestKey = `workspace-captures/${input.workspaceId}/${input.sessionId}/manifests/${turnKey}.json`;
552
+
553
+ // Re-read and upload ONE after-image at a time. The first pass above computes
554
+ // the stable capture fingerprint without retaining every changed file. This
555
+ // second pass takes the actual after-image that will be uploaded, refreshing
556
+ // metadata when a live process changed a file between passes. Peak worker
557
+ // memory is bounded by PER_FILE_CONTENT_GUARD_BYTES instead of
558
+ // WHOLE_CAPTURE_GUARD_BYTES per concurrent turn.
559
+ const uploadedKeys = new Set<string>();
560
+ blobKeys.clear();
561
+ totalBytes = 0;
562
+ let refreshedFiles = 0;
563
+ const omittedFiles = new Set<string>();
564
+ for (const file of files) {
565
+ throwIfCaptureAborted(signal);
566
+ if (file.deleted || file.tooLarge || !file.contentRef || !file.hash) {
567
+ continue;
568
+ }
569
+ let reread: Awaited<ReturnType<typeof svc.fsRead>>;
570
+ try {
571
+ reread = await svc.fsRead({
572
+ path: file.path,
573
+ encoding: "base64",
574
+ maxBytes: PER_FILE_CONTENT_GUARD_BYTES,
575
+ });
576
+ throwIfCaptureAborted(signal);
577
+ } catch (error) {
578
+ if (signal.aborted) throw error;
579
+ const boxExiting = classifyCaptureEntryError(error);
580
+ if (boxExiting) throw boxExiting;
581
+ // Match the first pass: a single vanished or unreadable file is omitted,
582
+ // never misreported as deleted and never allowed to fail the full capture.
583
+ omittedFiles.add(file.path);
584
+ refreshedFiles += 1;
585
+ continue;
586
+ }
587
+ if (reread.truncated) {
588
+ file.hash = null;
589
+ file.contentRef = null;
590
+ file.sizeBytes = reread.sizeBytes;
591
+ file.isBinary = reread.isBinary;
592
+ file.tooLarge = true;
593
+ refreshedFiles += 1;
594
+ continue;
595
+ }
596
+ const bytes = Buffer.from(reread.content, "base64");
597
+ const hash = sha256(bytes);
598
+ const contentRef = blobKey(input.workspaceId, input.sessionId, hash);
599
+ if (
600
+ hash !== file.hash ||
601
+ bytes.byteLength !== file.sizeBytes ||
602
+ reread.isBinary !== file.isBinary
603
+ ) {
604
+ file.hash = hash;
605
+ file.contentRef = contentRef;
606
+ file.sizeBytes = bytes.byteLength;
607
+ file.isBinary = reread.isBinary;
608
+ refreshedFiles += 1;
609
+ }
610
+ if (!blobKeys.has(contentRef)) {
611
+ totalBytes += bytes.byteLength;
612
+ if (totalBytes > WHOLE_CAPTURE_GUARD_BYTES) {
613
+ observability.warn("workspace capture skipped — whole-capture guard tripped", {
614
+ "opengeni.session_id": input.sessionId,
615
+ "workspace_capture.total_bytes": totalBytes,
616
+ });
617
+ observability.incrementCounter({
618
+ name: "opengeni_workspace_capture_total",
619
+ labels: { result: "guard_tripped" },
620
+ });
621
+ return;
622
+ }
623
+ }
624
+ blobKeys.add(contentRef);
625
+ if (!uploadedKeys.has(contentRef)) {
626
+ await storage.putObject({
627
+ key: contentRef,
628
+ contentType: "application/octet-stream",
629
+ body: bytes,
630
+ });
631
+ throwIfCaptureAborted(signal);
632
+ uploadedKeys.add(contentRef);
633
+ }
634
+ }
635
+ if (omittedFiles.size > 0) {
636
+ for (let index = files.length - 1; index >= 0; index -= 1) {
637
+ const file = files[index];
638
+ if (file && omittedFiles.has(file.path)) files.splice(index, 1);
639
+ }
640
+ }
641
+ if (refreshedFiles > 0) {
642
+ observability.incrementCounter({
643
+ name: "opengeni_workspace_capture_files_refreshed_total",
644
+ amount: refreshedFiles,
645
+ });
646
+ }
647
+ tooLargeCount = files.filter((file) => file.tooLarge).length;
648
+ binaryCount = files.filter((file) => !file.deleted && file.isBinary).length;
649
+ fingerprint = changeFingerprint(repos, files);
650
+ if (prev && prev.stats.fingerprint === fingerprint) {
651
+ observability.incrementCounter({
652
+ name: "opengeni_workspace_capture_total",
653
+ labels: { result: "skipped_empty" },
654
+ });
655
+ return;
656
+ }
657
+
658
+ // ── 7. serialize the exact uploaded after-images ──────────────────────────
659
+ const capturedAt = new Date();
660
+ const stats: WorkspaceCaptureStats = {
661
+ repoCount: repos.length,
662
+ fileCount: files.length,
663
+ additions,
664
+ deletions,
665
+ totalBytes,
666
+ tooLargeCount,
667
+ binaryCount,
668
+ treeEntryCount: tree.entryCount,
669
+ treeTruncated: tree.truncated,
670
+ durationMs: 0, // filled after tree/content writes, before manifest serialization
671
+ fingerprint,
672
+ };
673
+ const manifest: WorkspaceCaptureManifest = {
674
+ version: 1,
675
+ revision,
676
+ capturedAt: capturedAt.toISOString(),
677
+ turnId: input.turnId,
678
+ leaseEpoch: input.leaseEpoch,
679
+ treeIndex: tree.root,
680
+ treeTruncated: tree.truncated,
681
+ repos,
682
+ files,
683
+ stats,
684
+ };
685
+ const treeBytes = utf8(
686
+ JSON.stringify({
687
+ version: 1,
688
+ root: tree.root,
689
+ truncated: tree.truncated,
690
+ entryCount: tree.entryCount,
691
+ }),
692
+ );
693
+ await storage.putObject({ key: treeKey, contentType: "application/json", body: treeBytes });
694
+ throwIfCaptureAborted(signal);
695
+ stats.durationMs = Date.now() - startedAt;
696
+ const manifestBytes = utf8(JSON.stringify(manifest));
697
+ await storage.putObject({
698
+ key: manifestKey,
699
+ contentType: "application/json",
700
+ body: manifestBytes,
701
+ });
702
+ throwIfCaptureAborted(signal);
703
+ const sizeBytes = totalBytes + treeBytes.byteLength + manifestBytes.byteLength;
704
+
705
+ // ── 8. epoch-fenced insert (superseded lease → zero rows) ─────────────────
706
+ const inserted = await insertWorkspaceCapture(input.db, {
707
+ accountId: input.accountId,
708
+ workspaceId: input.workspaceId,
709
+ sessionId: input.sessionId,
710
+ turnId: input.turnId,
711
+ sandboxGroupId: input.sandboxGroupId,
712
+ expectedEpoch: input.leaseEpoch,
713
+ revision,
714
+ manifestKey,
715
+ treeIndexKey: treeKey,
716
+ blobKeys: [...blobKeys],
717
+ sizeBytes,
718
+ stats,
719
+ capturedAt,
720
+ });
721
+ throwIfCaptureAborted(signal);
722
+ if (!inserted) {
723
+ // Lease superseded/released between capture and commit. Best-effort clean up
724
+ // the turn-keyed blobs we just PUT (content blobs may be shared with a
725
+ // surviving revision — leave them for the next GC); never throw.
726
+ observability.incrementCounter({
727
+ name: "opengeni_workspace_capture_total",
728
+ labels: { result: "superseded" },
729
+ });
730
+ await safeDelete(storage, [manifestKey, treeKey], observability, signal);
731
+ return;
732
+ }
733
+
734
+ if (input.publish) {
735
+ await input.publish(inserted.events).catch(() => undefined);
736
+ }
737
+
738
+ // ── 9. inline keep-latest-N GC (best-effort; F9 — after the commit) ────────
739
+ let gcDeleted = 0;
740
+ try {
741
+ throwIfCaptureAborted(signal);
742
+ const plan = await planWorkspaceCaptureGc(input.db, {
743
+ workspaceId: input.workspaceId,
744
+ sessionId: input.sessionId,
745
+ keepN,
746
+ });
747
+ throwIfCaptureAborted(signal);
748
+ if (plan.evictedRowIds.length > 0) {
749
+ await safeDelete(
750
+ storage,
751
+ [...plan.deleteBlobKeys, ...plan.deletePerRevisionKeys],
752
+ observability,
753
+ signal,
754
+ );
755
+ throwIfCaptureAborted(signal);
756
+ gcDeleted = await deleteWorkspaceCaptureRows(input.db, {
757
+ workspaceId: input.workspaceId,
758
+ rowIds: plan.evictedRowIds,
759
+ });
760
+ throwIfCaptureAborted(signal);
761
+ observability.incrementCounter({
762
+ name: "opengeni_workspace_capture_gc_deletions_total",
763
+ amount: gcDeleted,
764
+ });
765
+ }
766
+ } catch (gcError) {
767
+ if (signal.aborted) throw gcError;
768
+ // GC is storage hygiene — a failure never affects the just-committed capture.
769
+ observability.warn("workspace capture GC failed — capture unaffected", {
770
+ "opengeni.session_id": input.sessionId,
771
+ "error.message": gcError instanceof Error ? gcError.message : String(gcError),
772
+ });
773
+ }
774
+
775
+ // ── 10. observe completion ────────────────────────────────────────────────
776
+ const durationMs = Date.now() - startedAt;
777
+ observability.incrementCounter({
778
+ name: "opengeni_workspace_capture_total",
779
+ labels: { result: "ok" },
780
+ });
781
+ observability.observeHistogram({
782
+ name: "opengeni_workspace_capture_duration_seconds",
783
+ value: durationMs / 1000,
784
+ });
785
+ }
786
+
787
+ // ── helpers ──────────────────────────────────────────────────────────────────
788
+
789
+ function captureDegradedReason(
790
+ reason: RepositoryDiscoveryDegradedReason | null,
791
+ ): WorkspaceCaptureDegradedReason {
792
+ switch (reason) {
793
+ case "command_timed_out":
794
+ return "repository_discovery_timed_out";
795
+ case "result_limit_exceeded":
796
+ return "repository_discovery_result_limit_exceeded";
797
+ case "command_failed":
798
+ default:
799
+ return "repository_discovery_command_failed";
800
+ }
801
+ }
802
+
803
+ function throwIfCaptureAborted(signal: AbortSignal): void {
804
+ if (signal.aborted) {
805
+ throw new Error("workspace capture cancelled after its deadline");
806
+ }
807
+ }
808
+
809
+ function statusCodeOf(f: GitFileStatus): GitFileStatusCode {
810
+ return f.worktree ?? f.index ?? "modified";
811
+ }
812
+
813
+ /**
814
+ * sha256 over the CHANGE SURFACE only — per-file (path, status, hash, deleted,
815
+ * tooLarge) and per-repo diff summary (path, status, additions, deletions).
816
+ * Deliberately excludes the tree index and file mtimes (which drift without a
817
+ * real change) so two turns that leave the workspace in the same state produce
818
+ * the same fingerprint (the empty-turn gate). Order-independent (sorted).
819
+ */
820
+ function changeFingerprint(repos: WorkspaceCaptureRepo[], files: WorkspaceCaptureFile[]): string {
821
+ const fileParts = files
822
+ .map((f) => `${f.path}|${f.status}|${f.hash ?? ""}|${f.deleted ? 1 : 0}|${f.tooLarge ? 1 : 0}`)
823
+ .sort();
824
+ const repoParts = repos
825
+ .map(
826
+ (r) =>
827
+ `${r.root}#${r.head ?? ""}#` +
828
+ r.diff
829
+ .map((d) => `${d.path}:${d.status}:${d.additions}:${d.deletions}:${d.truncated ? 1 : 0}`)
830
+ .sort()
831
+ .join(","),
832
+ )
833
+ .sort();
834
+ return sha256(utf8(JSON.stringify({ files: fileParts, repos: repoParts })));
835
+ }
836
+
837
+ /** Join a repo-root-relative path onto its workspace-relative repo root. */
838
+ export function joinRepoPath(repoRoot: string, repoRelPath: string): string {
839
+ if (!repoRoot || repoRoot === "" || repoRoot === ".") return repoRelPath;
840
+ return `${repoRoot.replace(/\/+$/, "")}/${repoRelPath}`;
841
+ }
842
+
843
+ function sha256(bytes: Uint8Array): string {
844
+ return createHash("sha256").update(bytes).digest("hex");
845
+ }
846
+
847
+ function utf8(s: string): Uint8Array {
848
+ return new TextEncoder().encode(s);
849
+ }
850
+
851
+ /** Content-addressed after-image blob key (shared across revisions → GC input). */
852
+ export function blobKey(workspaceId: string, sessionId: string, sha256Hex: string): string {
853
+ return `workspace-captures/${workspaceId}/${sessionId}/blobs/${sha256Hex}`;
854
+ }
855
+
856
+ async function safeDelete(
857
+ storage: ObjectStorage,
858
+ keys: string[],
859
+ observability: Observability,
860
+ signal: AbortSignal,
861
+ ): Promise<void> {
862
+ for (const key of keys) {
863
+ throwIfCaptureAborted(signal);
864
+ await storage.deleteObject(key).catch((error) => {
865
+ observability.warn("workspace capture blob delete failed (orphan left; capture unaffected)", {
866
+ "storage.key": key,
867
+ "error.message": error instanceof Error ? error.message : String(error),
868
+ });
869
+ });
870
+ }
871
+ }
872
+
873
+ /**
874
+ * Build the workspace tree index via bounded BFS over the PUBLIC fsList (depth 1
875
+ * per directory). Residue dirs are emitted as collapsed nodes but never queued
876
+ * for descent — a single deep fsList would let node_modules exhaust the entry
877
+ * budget and truncate real files, so we prune at the source. Bounded by
878
+ * TREE_MAX_ENTRIES, TREE_MAX_DIRS round-trips, and the outer capture deadline.
879
+ */
880
+ async function buildTreeIndex(
881
+ svc: SandboxChannelAService,
882
+ startedAt: number,
883
+ signal: AbortSignal,
884
+ ): Promise<{ root: FsTreeNode; entryCount: number; truncated: boolean }> {
885
+ const root: FsTreeNode = {
886
+ name: "",
887
+ path: "",
888
+ type: "dir",
889
+ sizeBytes: null,
890
+ mtimeMs: null,
891
+ mode: null,
892
+ children: [],
893
+ truncated: false,
894
+ };
895
+ const byPath = new Map<string, FsTreeNode>([["", root]]);
896
+ const queue: string[] = [""];
897
+ let entryCount = 0;
898
+ let dirCalls = 0;
899
+ let truncated = false;
900
+
901
+ while (queue.length > 0) {
902
+ throwIfCaptureAborted(signal);
903
+ if (dirCalls >= TREE_MAX_DIRS || entryCount >= TREE_MAX_ENTRIES) {
904
+ truncated = true;
905
+ break;
906
+ }
907
+ if (Date.now() - startedAt > CAPTURE_TIMEOUT_MS - 5_000) {
908
+ truncated = true;
909
+ break;
910
+ } // leave headroom for PUTs
911
+ const dir = queue.shift()!;
912
+ dirCalls += 1;
913
+ // Per-dir resilience: a directory can vanish or fail to list mid-walk (fs
914
+ // churn on a desktop box, a permission edge). Skip the one directory and keep
915
+ // walking the rest — a single unreadable dir must never abort the tree index.
916
+ // A box death still aborts the whole capture (BoxExitingError).
917
+ let listing: Awaited<ReturnType<typeof svc.fsList>>;
918
+ try {
919
+ listing = await svc.fsList({ path: dir, depth: 1, maxEntries: 2_000, includeHidden: true });
920
+ throwIfCaptureAborted(signal);
921
+ } catch (error) {
922
+ if (isBoxExitingError(error)) {
923
+ throw new BoxExitingError(error instanceof Error ? error.message : String(error));
924
+ }
925
+ truncated = true; // this subtree is incomplete; continue with the rest
926
+ continue;
927
+ }
928
+ if (listing.truncated) truncated = true;
929
+ const parent = byPath.get(dir) ?? root;
930
+ const children = collectImmediateChildren(listing.root);
931
+ for (const child of children) {
932
+ if (entryCount >= TREE_MAX_ENTRIES) {
933
+ truncated = true;
934
+ break;
935
+ }
936
+ const node: FsTreeNode = {
937
+ name: child.name,
938
+ path: child.path,
939
+ type: child.type,
940
+ sizeBytes: child.sizeBytes,
941
+ mtimeMs: child.mtimeMs,
942
+ mode: child.mode,
943
+ truncated: false,
944
+ ...(child.type === "dir" ? { children: [] as FsTreeNode[] } : {}),
945
+ };
946
+ (parent.children ??= []).push(node);
947
+ byPath.set(node.path, node);
948
+ entryCount += 1;
949
+ if (node.type === "dir") {
950
+ if (RESIDUE_DIR_SET.has(node.name)) {
951
+ node.truncated = true; // collapsed residue dir — contents on the machine
952
+ } else {
953
+ queue.push(node.path);
954
+ }
955
+ }
956
+ }
957
+ }
958
+ return { root, entryCount, truncated };
959
+ }
960
+
961
+ /** fsList returns a subtree; at depth 1 the immediate children hang off root. */
962
+ function collectImmediateChildren(node: FsTreeNode): FsTreeNode[] {
963
+ return node.children ?? [];
964
+ }