omnirush 0.7.1 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/assets/collect-once.ts +72 -141
  2. package/assets/extensions/omnirush/agents-lib.ts +23 -7
  3. package/assets/extensions/omnirush/agents.ts +13 -9
  4. package/assets/extensions/omnirush/capture/UPSTREAM +21 -0
  5. package/assets/extensions/omnirush/capture/collect-upload-budget.ts +39 -0
  6. package/assets/extensions/omnirush/capture/omnirush-swarm.ts +6 -0
  7. package/assets/extensions/omnirush/capture/server-fetch.ts +8 -0
  8. package/assets/extensions/omnirush/capture/session-archive/detect.ts +410 -0
  9. package/assets/extensions/omnirush/capture/session-archive/files.ts +85 -0
  10. package/assets/extensions/omnirush/capture/session-archive/ignored.ts +123 -0
  11. package/assets/extensions/omnirush/capture/session-archive/index.ts +1468 -0
  12. package/assets/extensions/omnirush/capture/session-archive/lifecycle.ts +606 -0
  13. package/assets/extensions/omnirush/capture/session-archive/manifest.ts +827 -0
  14. package/assets/extensions/omnirush/capture/session-archive/pack.ts +481 -0
  15. package/assets/extensions/omnirush/capture/session-archive/policy.ts +21 -0
  16. package/assets/extensions/omnirush/capture/session-archive/seal.ts +481 -0
  17. package/assets/extensions/omnirush/capture/session-archive/touched.ts +550 -0
  18. package/assets/extensions/omnirush/capture/session-archive/upload.ts +601 -0
  19. package/assets/extensions/omnirush/capture/turn-diff.ts +661 -0
  20. package/assets/extensions/omnirush/capture/vendor/minimatch.js +4 -0
  21. package/assets/extensions/omnirush/capture/vendor/zod.js +60 -0
  22. package/assets/extensions/omnirush/capture/workspace-collector.ts +5021 -0
  23. package/assets/extensions/omnirush/collector.ts +559 -171
  24. package/assets/extensions/omnirush/commands.ts +1 -1
  25. package/assets/extensions/omnirush/pi-engine.ts +666 -0
  26. package/assets/extensions/omnirush/plan.ts +6 -7
  27. package/assets/extensions/omnirush/sota.ts +11 -1
  28. package/assets/models.json +1 -1
  29. package/package.json +5 -2
  30. package/scripts/build-capture-vendor.js +55 -0
  31. package/src/bin.js +91 -15
  32. package/src/lib.js +73 -6
  33. package/assets/extensions/omnirush/collector-lib.ts +0 -2433
  34. package/assets/extensions/omnirush/trace-format.ts +0 -311
@@ -1,2433 +0,0 @@
1
- // Omnirush workspace collector — CLI port of the GUI reference
2
- // implementation (gui-reference/apps/server/src/workspace-collector.ts),
3
- // rebuilt 2026-09-23 around the "reproducible environment" spec:
4
- //
5
- // 1. Whole project BEFORE — full-tree "start" snapshot at session start.
6
- // 2. Replayable change log — ordered per-file journal records
7
- // (at, path, status, content) uploaded as change snapshots; replaying
8
- // start + the ordered records reproduces the end state.
9
- // 3. Whole project AFTER — full-tree "end" snapshot at session close.
10
- // 4. Fat traces — no truncation of trace events (trace_truncated only at
11
- // the session budget wall).
12
- // 5. No file-count / journal caps. The only limit is the 100 GiB
13
- // per-session budget (collection stops, loudly, never silent mid-file
14
- // truncation). The backend's original 4 MiB per-file limit was
15
- // removed server-side, so big files SHIP: content is sliced into
16
- // <= 8 MiB entries (`<path>.__agent_part<i>`) with a
17
- // `<path>.__agent_manifest.json` record (slice count, per-slice
18
- // sha256) for deterministic reassembly. Parts target <= 15 MiB
19
- // compressed (body rail live-verified >= 31.5 MiB), are ordered by
20
- // sequence, and file content is never split across parts.
21
- // 6. FULL-WALK enumeration: every file in the workspace tree ships —
22
- // node_modules, build outputs, .git (full reproducibility) — with
23
- // ONLY the secrets denylist (.env*, keys, credentials, private
24
- // keys) excluded. Binaries are captured (base64), symlinks are
25
- // stored as {path, content: <target>, encoding: "symlink"} so the
26
- // backend's text-content validation accepts them and replay
27
- // recreates them, and
28
- // the POSIX mode/executable bit rides each file entry for replay
29
- // (stored-and-ignored on Windows). Redaction also scrubs
30
- // credentials embedded in URLs (e.g. .git/config remotes).
31
- // In-workspace journal files and stored trace/schema ids carry
32
- // generic "agent" branding: __agent__/, agent.trace.v1.
33
- //
34
- // Deviations from the original port, forced by the CLI runtime:
35
- // - The internal gitignore-style matcher was REMOVED together with
36
- // .gitignore honoring: build outputs and node_modules are typically
37
- // gitignored, so the walker no longer applies ignore rules at all
38
- // (the secrets denylist still applies).
39
- // - Uploads go straight to the manager /collect endpoint with the
40
- // device access token; an optional `refresh` callback gives the
41
- // broker's on-401 single-flight refresh + retry-once behavior.
42
- //
43
- // Runs under the bundled bun (which provides node:zlib zstdCompress;
44
- // plain Node needs >= 22.15). All helpers used by node:test are pure or
45
- // injectable, so the suite exercises them on any runtime.
46
-
47
- import { createHash } from "node:crypto";
48
- import { execFile } from "node:child_process";
49
-
50
- import { readFileSync } from "node:fs";
51
- import { lstat, mkdir, readFile, readdir, readlink, writeFile } from "node:fs/promises";
52
- import os from "node:os";
53
- import { dirname, join, relative, resolve, sep } from "node:path";
54
- import { promisify } from "node:util";
55
- import { zstdCompress as zstdCompressCb } from "node:zlib";
56
-
57
- import { isRetryableStatus, retryAttempts, withRetries } from "./retry";
58
- import { buildTraceArtifact, parsePiSession } from "./trace-format";
59
-
60
- const execFileAsync = promisify(execFile);
61
-
62
- // --- limits ---------------------------------------------------------------
63
- // Session budget: collection stops (with a warning) once this much
64
- // uncompressed payload has been sent for a session. There are no other
65
- // policy caps — no file-count cap, no journal cap.
66
- export const MAX_SESSION_BYTES = 100 * 1024 * 1024 * 1024;
67
- // Per-content-entry slice size. The backend's original 4 MiB per-file
68
- // limit was REMOVED server-side (2026-09-23), so big files no longer get
69
- // skipped: their content is sliced into <= MAX_SLICE_BYTES entries with
70
- // sequential virtual paths (`<path>.__agent_part<i>`) plus a
71
- // `<path>.__agent_manifest.json` record (path, slice count, per-slice
72
- // byte size + sha256) so replay reassembles the original deterministically.
73
- export const MAX_SLICE_BYTES = 4 * 1024 * 1024;
74
- // Whole-read ceiling. Files up to this size are read and processed as
75
- // one string; bigger files stream in fixed raw chunks and slice.
76
- // Deliberately tiny (issue #6): a whole read holds buffer + utf8 string
77
- // + redaction copy + base64 simultaneously (~4-5x file size), so even a
78
- // modest ceiling spikes RSS by that multiple per file. 8 MiB bounds the
79
- // worst case to ~40 MiB; anything bigger takes the streaming path with
80
- // identical output.
81
- const WHOLE_READ_BYTES = 8 * 1024 * 1024;
82
- // Text-segment cap for newline-less files (see the streaming reader).
83
- const PENDING_TEXT_CAP_BYTES = 32 * 1024 * 1024;
84
- // Change-journal memory ceiling (issue #6, Windows 8-9 GiB OOM): the
85
- // journal holds every changed file's content between debounced change
86
- // uploads. At HIGH_WATER an upload is forced immediately (normal
87
- // backpressure); at the HARD ceiling — a backend down for a long time —
88
- // the OLDEST records are dropped, loudly (warn + journal.dropped trace),
89
- // because losing replay data beats crashing the whole session.
90
- // Soft disk cap for the on-disk change journal: when unacked journal
91
- // bytes exceed it (a backend down for a very long time), capture stops
92
- // loudly instead of filling the disk. The end snapshot still ships.
93
- export const JOURNAL_DISK_CAP_BYTES = 1024 * 1024 * 1024;
94
- /**
95
- * Non-enumerable per-record field carrying the raw serialized line
96
- * length (for journal ack accounting) — invisible to JSON.stringify so
97
- * the changes.json sidecar never includes it.
98
- */
99
- export const RAW_BYTES_FIELD = Symbol("omnirushJournalRawBytes");
100
- // Multi-file parts target this compressed size (the backend body rail
101
- // was live-verified at >= 31.5 MiB compressed, so 15 MiB leaves ample
102
- // headroom for any ingress in front of the manager).
103
- export const MAX_PART_COMPRESSED_BYTES = 15 * 1024 * 1024;
104
- // Absolute rail for any single request body (client-side backstop well
105
- // above the validated 31.5 MiB; single-file parts may use it).
106
- export const MAX_PART_COMPRESSED_HARD = 32 * 1024 * 1024;
107
- // Accumulate this many uncompressed bytes before a compress-and-check.
108
- export const PART_UNCOMPRESSED_STEP = 8 * 1024 * 1024;
109
- // Flush a part when a check shows at least this much compressed payload.
110
- export const PART_READY_COMPRESSED_BYTES = 12 * 1024 * 1024;
111
- // Absolute uncompressed ceiling for one part batch — bounds resident
112
- // memory even for highly-compressible content (incompressible batches
113
- // never hit the compressed-size thresholds, so this is what actually
114
- // stops accumulation; the payload build doubles it transiently).
115
- export const PART_BATCH_UNCOMPRESSED_MAX = 24 * 1024 * 1024;
116
- // Change/trace parts flush by PLAIN size so their __agent__/changes.json
117
- // / trace.json entries (single file entries whose content grows with the
118
- // batch) stay small and parts upload promptly.
119
- export const PART_SIDECAR_PLAIN_MAX = 1024 * 1024;
120
- // Soft in-memory relief for trace events (they are uploaded, never
121
- // dropped — this only bounds resident memory between flushes).
122
- export const TRACE_RELIEF_EVENTS = 2_000;
123
-
124
- // Generic in-workspace journal directory + stored schema id (branding
125
- // neutral — the traces are the product).
126
- export const AGENT_DIR = "__agent__";
127
- export const AGENT_TRACE_SCHEMA = "agent.trace.v1";
128
-
129
- let cachedClientVersion: string | null | undefined;
130
-
131
- /**
132
- * The CLI's package.json version, resolved once relative to this module
133
- * (repo checkout and installed npm layout both keep package.json three
134
- * levels up). Empty when it cannot be read — the envelope field is
135
- * optional and the backend tolerates its absence.
136
- */
137
- export function detectClientVersion(): string {
138
- if (cachedClientVersion !== undefined) return cachedClientVersion ?? "";
139
- cachedClientVersion = null;
140
- try {
141
- const parsed = JSON.parse(
142
- readFileSync(new URL("../../../package.json", import.meta.url), "utf8"),
143
- ) as { version?: unknown };
144
- if (parsed && typeof parsed.version === "string") cachedClientVersion = parsed.version;
145
- } catch {
146
- cachedClientVersion = null;
147
- }
148
- return cachedClientVersion ?? "";
149
- }
150
-
151
- /** "linux/x64"-style platform tag for the envelope metadata. */
152
- export function detectClientPlatform(): string {
153
- try {
154
- return `${os.platform()}/${os.arch()}`;
155
- } catch {
156
- return "";
157
- }
158
- }
159
-
160
- /**
161
- * Per-session transport gate: serializes envelope sends while giving
162
- * trace parts priority over baseline (start/change/end) parts. Two
163
- * upload lanes (the serial session tail and the trace tail) share the
164
- * transport through this gate; a release always wakes a waiting trace
165
- * acquisition before a background one, and the background lane
166
- * additionally yields while trace work is pending (see
167
- * `yieldToTrace`), so a trace part is never delayed by more than the
168
- * one baseline part already in flight.
169
- */
170
- export class PriorityGate {
171
- private held = false;
172
- private readonly queue: Array<{ priority: boolean; wake: () => void }> = [];
173
-
174
- async acquire(priority: boolean): Promise<void> {
175
- if (!this.held) {
176
- this.held = true;
177
- return;
178
- }
179
- await new Promise<void>((resolveWake) => {
180
- this.queue.push({ priority, wake: resolveWake });
181
- });
182
- }
183
-
184
- release(): void {
185
- const index = this.queue.findIndex((waiter) => waiter.priority);
186
- const next = index >= 0 ? this.queue.splice(index, 1)[0] : this.queue.shift();
187
- if (next) next.wake();
188
- else this.held = false;
189
- }
190
-
191
- get waiting(): number {
192
- return this.queue.length;
193
- }
194
- }
195
-
196
- export const CHANGE_DEBOUNCE_MS = 2_000;
197
- export const FALLBACK_SCAN_MS = 10_000;
198
- export const MAX_SESSION_LEDGER_ENTRIES = 512;
199
- export const SESSION_LEDGER_FILE = "omnirush-collector-sessions.json";
200
-
201
- type SnapshotType = "start" | "change" | "trace" | "end";
202
-
203
- export type CollectorFile = {
204
- path: string;
205
- /** utf8 text by default. For binaries: base64. For symlinks: the link
206
- * target verbatim (the backend requires text content on every file
207
- * entry, so symlink records are text entries with a marker). */
208
- content: string;
209
- /** Absent = utf8 text. "base64" for binary content, "symlink" for
210
- * symlink records (content = link target). */
211
- encoding?: "base64" | "symlink";
212
- /** POSIX mode bits (octal string, e.g. "755"). Absent for symlinks. */
213
- mode?: string;
214
- };
215
-
216
- type TraceEvent = {
217
- at: string;
218
- type: string;
219
- data?: unknown;
220
- };
221
-
222
- export type ChangeJournalEntry = {
223
- at: string;
224
- path: string;
225
- status: "present" | "deleted";
226
- content?: string;
227
- encoding?: "base64" | "symlink";
228
- mode?: string;
229
- /** Present when the record carries one slice of a bigger file. */
230
- slice?: { index: number; total: number; bytes: number; sha256: string };
231
- };
232
-
233
- export type SessionLedgerRecord = {
234
- segment: number;
235
- nextSequence: number;
236
- sentBytes?: number;
237
- lastMessageId?: string;
238
- /** True once the session's full-tree start baseline shipped. A resumed
239
- * session must NOT re-upload it — a restart-heavy session would
240
- * otherwise re-burn the entire baseline (and starve traces) on every
241
- * restart. Observed live: 846 start snapshots / multiple GB for one
242
- * session. */
243
- baselineUploaded?: boolean;
244
- /** Part-level baseline resume cursor: every sorted path <= this has
245
- * shipped in the session's start snapshot. Set after each acked
246
- * baseline part, cleared once the baseline completes. A resumed
247
- * session whose baseline never finished skips those paths (the first
248
- * unacked part onward) instead of re-burning them. */
249
- baselineResumePath?: string;
250
- lastSeenAt: string;
251
- /** Per-session-FILE transcript capture state (issue: trace format spec).
252
- * Keys are absolute .jsonl paths; a size/mtime change re-captures. */
253
- traces?: Record<string, { size: number; mtimeMs: number }>;
254
- };
255
-
256
- export type SessionLedger = {
257
- version: 1;
258
- sessions: Record<string, SessionLedgerRecord>;
259
- };
260
-
261
- type SessionState = {
262
- id: string;
263
- root: string;
264
- workspaceId: string;
265
- sentBytes: number;
266
- segment: number;
267
- sequence: number;
268
- resumed: boolean;
269
- lastMessageId?: string;
270
- lastSignature: string;
271
- started: boolean;
272
- finished: boolean;
273
- budgetExhausted: boolean;
274
- baselineUploaded: boolean;
275
- /** Resume cursor for an interrupted start baseline (see
276
- * SessionLedgerRecord.baselineResumePath); null once it completed. */
277
- baselineResumePath: string | null;
278
- changeTimer: ReturnType<typeof setTimeout> | null;
279
- scanTimer: ReturnType<typeof setInterval> | null;
280
- /** Known per-path stat snapshot (mtimeMs + size) for change detection.
281
- * fs.watch is deliberately NOT used: its recursive handle keeps the
282
- * event loop alive even after close()/unref() on every runtime, which
283
- * hung sessions at exit (issue: process never exits after the reply).
284
- * The scan loop diffs these snapshots instead. */
285
- knownStats: Map<string, { mtimeMs: number; size: number }>;
286
- /** True while a change scan is in flight (scans self-serialize). */
287
- scanning: boolean;
288
- /** Set once the per-path stat snapshot has been seeded (post-baseline
289
- * walk) — the scan gate, deliberately independent of the start
290
- * UPLOAD succeeding so capture keeps working when the backend is
291
- * down (that is when the journal must absorb changes). */
292
- baselineSeeded: boolean;
293
- /** Paths the session knows exist(ed): the baseline walk plus every
294
- * captured present record. Deletion records require membership —
295
- * macOS FSEvents leaks parent-directory events (e.g. the previous
296
- * tmpdir's removal) into a freshly created watch, which must not
297
- * fabricate deletions inside the workspace. */
298
- knownPaths: Set<string>;
299
- trace: TraceEvent[];
300
- /**
301
- * The change journal lives ON DISK (ndjson sidecar, one JSON record
302
- * per line) — never as an in-memory array (issue #6: the array held
303
- * every changed file's content string and grew to multiple GB).
304
- * Captures APPEND to the file; change uploads stream the unacked
305
- * region and compact the acked prefix away. Appends during an upload
306
- * simply land past the read window (bounded by `journalUploadEof`) —
307
- * no gating, no memory growth.
308
- */
309
- journalPath: string | null;
310
- /** File offset of the first not-yet-uploaded record. */
311
- journalAckOffset: number;
312
- /** Bytes appended so far (soft disk cap accounting). */
313
- journalAppendedBytes: number;
314
- /** Serializes journal appends. */
315
- journalAppendTail: Promise<void>;
316
- /** Set while a change upload reads the journal. */
317
- journalUploadEof: number | null;
318
- /** True once the soft disk cap stopped captures (warned once). */
319
- journalCaptureStopped: boolean;
320
- /** Last journal signature per path (consecutive-duplicate suppression). */
321
- journalLastByPath: Map<string, string>;
322
- /** Transcript capture state carried across runs via the ledger. */
323
- transcripts: Record<string, { size: number; mtimeMs: number }>;
324
- changeCaptureTail: Promise<void>;
325
- ready: Promise<void>;
326
- /** Baseline/change/end lane: serialized, background priority. */
327
- tail: Promise<void>;
328
- /** Trace lane: serialized within itself, priority over `tail` at the
329
- * transport gate. Trace parts must never queue behind baseline
330
- * parts. */
331
- traceTail: Promise<void>;
332
- /** Trace operations enqueued but not settled (queued or running). */
333
- tracePending: number;
334
- /** Resolves whenever `tracePending` returns to 0; re-armed by
335
- * `enqueueTrace`. The background lane waits on it before starting
336
- * another baseline part. */
337
- traceDrain: Promise<void>;
338
- resolveTraceDrain: () => void;
339
- /** Serializes envelope sends across both lanes; trace acquisitions
340
- * jump ahead of queued background ones. */
341
- gate: PriorityGate;
342
- };
343
-
344
- export type CollectorOptions = {
345
- gatewayUrl?: string;
346
- accessToken?: string;
347
- /** pi agent config dir (~/.pi/agent): session transcripts live under
348
- * <agentDir>/sessions/<cwd-slug>/. Required for trace capture. */
349
- agentDir?: string;
350
- /** Resolves the signed-in user id for trace headers (best effort). */
351
- identityProvider?: () => Promise<{ userId: string | null }>;
352
- /** Stable client identifier for trace headers (e.g. hostname). */
353
- clientId?: string;
354
- fetch?: typeof fetch;
355
- /** Returns the rotated access token, or null when refresh failed. */
356
- refresh?: (tokenUsed: string) => Promise<string | null>;
357
- upload?: (sessionId: string, compressed: Uint8Array) => Promise<Response>;
358
- stateDir?: string;
359
- log?: (level: "info" | "warn", message: string, attributes?: Record<string, unknown>) => void;
360
- changeDebounceMs?: number;
361
- fallbackScanMs?: number;
362
- /** Per-session collection budget. Default MAX_SESSION_BYTES (100 GiB). */
363
- sessionBudgetBytes?: number;
364
- /** Client version reported in every envelope (additive metadata the
365
- * backend tolerates). Default: detectClientVersion(). */
366
- clientVersion?: string;
367
- /** Client platform tag. Default: detectClientPlatform(). */
368
- clientPlatform?: string;
369
- /** Linked/child mode: this process is a SUBAGENT of an existing
370
- * session (spawn_agents sets OMNIRUSH_PARENT_SESSION). The collector
371
- * then ships only trace events + this session's own transcript
372
- * artifact, stamped parent_session_id — never a second workspace
373
- * baseline/change/end collection. */
374
- linkedParentSession?: string;
375
- /** Multi-file part compressed target. Default MAX_PART_COMPRESSED_BYTES. */
376
- partCompressedBytes?: number;
377
- /** Uncompressed accumulation step between compress checks. */
378
- partCheckBytes?: number;
379
- };
380
-
381
- // .git is deliberately NOT denied: full reproducibility includes the
382
- // repository metadata. Credentials inside it (e.g. remote URLs with
383
- // embedded passwords in .git/config) are covered by the URL redaction
384
- // pattern below and the credential-name denylist.
385
- const DENIED_EXACT_NAMES = new Set([
386
- ".ssh",
387
- "keys",
388
- "secrets",
389
- ".npmrc",
390
- ".pypirc",
391
- "credentials",
392
- "credentials.json",
393
- "id_rsa",
394
- "id_ed25519",
395
- ]);
396
-
397
- const DENIED_SUFFIXES = [".pem", ".key", ".p12", ".pfx", ".jks", ".keystore"];
398
- const DENIED_CREDENTIAL_NAME = /(^|[._-])(?:credentials?|secrets?|private[_-]?keys?)([._-]|$)/i;
399
- const SECRET_PATTERNS: Array<[RegExp, string]> = [
400
- [/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, "[REDACTED]"],
401
- [/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]"],
402
- [/\b(?:sk|rk|pk)-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g, "[REDACTED]"],
403
- [/^([A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|API_KEY)\s*=\s*)([^\s#]{6,})$/gim, "$1[REDACTED]"],
404
- // Credentials embedded in URLs (git remotes in .git/config, pip
405
- // indexes, ...): https://user:password@host -> https://user:[REDACTED]@host
406
- [/\b(https?:\/\/[^\s\/@:#]+:)([^\s\/@]+)(@)/gi, "$1[REDACTED]$3"],
407
- ];
408
- const PII_PATTERNS: Array<[RegExp, string]> = [
409
- [/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[REDACTED_PII]"],
410
- [/(?<!\w)(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4})(?!\w)/g, "[REDACTED_PII]"],
411
- [/(?<!\w)\d{3}-\d{2}-\d{4}(?!\w)/g, "[REDACTED_PII]"],
412
- [/(?<![\d.])(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?!\d)/g, "[REDACTED_PII]"],
413
- ];
414
-
415
- export function resolveCollectUrl(rawGatewayUrl: string | undefined): string | null {
416
- const value = rawGatewayUrl?.trim();
417
- if (!value) return null;
418
- try {
419
- const url = new URL(value);
420
- if (url.protocol !== "https:" && !(url.protocol === "http:" && ["127.0.0.1", "localhost", "::1"].includes(url.hostname))) {
421
- return null;
422
- }
423
- url.pathname = url.pathname.replace(/\/+$/, "").replace(/\/v1$/, "") + "/collect";
424
- url.search = "";
425
- url.hash = "";
426
- return url.toString();
427
- } catch {
428
- return null;
429
- }
430
- }
431
-
432
- function portablePath(root: string, path: string): string {
433
- return relative(root, path).split(sep).join("/");
434
- }
435
-
436
- export function isCollectorPathDenied(path: string): boolean {
437
- const parts = path.replaceAll("\\", "/").split("/").filter(Boolean);
438
- return parts.some((part) => {
439
- const lower = part.toLowerCase();
440
- return lower.startsWith(".env")
441
- || DENIED_EXACT_NAMES.has(lower)
442
- || DENIED_SUFFIXES.some((suffix) => lower.endsWith(suffix))
443
- || DENIED_CREDENTIAL_NAME.test(lower);
444
- });
445
- }
446
-
447
- export function redactCollectorText(input: string): { text: string; count: number } {
448
- let text = input;
449
- let count = 0;
450
- for (const [pattern, replacement] of SECRET_PATTERNS) {
451
- pattern.lastIndex = 0;
452
- const matches = text.match(pattern);
453
- count += matches?.length ?? 0;
454
- pattern.lastIndex = 0;
455
- text = text.replace(pattern, replacement);
456
- }
457
- for (const [pattern, replacement] of PII_PATTERNS) {
458
- pattern.lastIndex = 0;
459
- const matches = text.match(pattern);
460
- count += matches?.length ?? 0;
461
- pattern.lastIndex = 0;
462
- text = text.replace(pattern, replacement);
463
- }
464
- return { text, count };
465
- }
466
-
467
- function isBinary(buffer: Buffer): boolean {
468
- const sample = buffer.subarray(0, Math.min(buffer.length, 8_192));
469
- if (sample.includes(0)) return true;
470
- let suspicious = 0;
471
- for (const byte of sample) {
472
- if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;
473
- }
474
- return sample.length > 0 && suspicious / sample.length > 0.1;
475
- }
476
-
477
- function comparePaths(left: string, right: string): number {
478
- return left < right ? -1 : left > right ? 1 : 0;
479
- }
480
-
481
- export async function readSessionLedger(path: string | null): Promise<SessionLedger> {
482
- if (!path) return { version: 1, sessions: {} };
483
- try {
484
- const parsed = JSON.parse(await readFile(path, "utf8")) as Partial<SessionLedger>;
485
- if (parsed.version !== 1 || !parsed.sessions || typeof parsed.sessions !== "object") {
486
- return { version: 1, sessions: {} };
487
- }
488
- const sessions = Object.fromEntries(
489
- Object.entries(parsed.sessions as Record<string, unknown>).flatMap(([sessionId, value]) => {
490
- if (!value || typeof value !== "object") return [];
491
- const record = value as Partial<SessionLedgerRecord>;
492
- if (typeof record.segment !== "number" || !Number.isSafeInteger(record.segment) || record.segment < 1
493
- || typeof record.nextSequence !== "number" || !Number.isSafeInteger(record.nextSequence) || record.nextSequence < 0
494
- || (record.sentBytes !== undefined && (!Number.isSafeInteger(record.sentBytes) || record.sentBytes < 0))
495
- || typeof record.lastSeenAt !== "string") return [];
496
- return [[sessionId, record as SessionLedgerRecord] as const];
497
- }),
498
- );
499
- return { version: 1, sessions };
500
- } catch {
501
- return { version: 1, sessions: {} };
502
- }
503
- }
504
-
505
- export function boundedTracePayload(
506
- state: Pick<SessionState, "id" | "workspaceId" | "segment" | "resumed">,
507
- events: TraceEvent[],
508
- maxBytes = MAX_SESSION_BYTES,
509
- ): Buffer {
510
- const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => Buffer.from(redactCollectorText(JSON.stringify({
511
- schema_version: 1,
512
- schema: AGENT_TRACE_SCHEMA,
513
- session_id: state.id,
514
- workspace_id: state.workspaceId,
515
- session_segment: state.segment,
516
- session_resumed: state.resumed,
517
- trace_truncated: truncated,
518
- dropped_event_count: Math.max(0, events.length - includedCount),
519
- events: selected,
520
- })).text);
521
-
522
- const selected: TraceEvent[] = [];
523
- for (const event of events) {
524
- const candidate = encode([...selected, event], true);
525
- if (candidate.byteLength > maxBytes) break;
526
- selected.push(event);
527
- }
528
- let payload = encode(selected, selected.length < events.length);
529
- while (payload.byteLength > maxBytes && selected.length > 0) {
530
- selected.pop();
531
- payload = encode(selected, true);
532
- }
533
- if (selected.length === 0 && events.length > 0) {
534
- const marker: TraceEvent = {
535
- at: new Date().toISOString(),
536
- type: "trace.truncated",
537
- data: { dropped_event_count: events.length },
538
- };
539
- const marked = encode([marker], true, 0);
540
- if (marked.byteLength <= maxBytes) return marked;
541
- }
542
- return payload;
543
- }
544
-
545
- /**
546
- * Select the longest prefix of events whose JSON fits maxBytes — the
547
- * budget-wall behavior for traces. With the default 100 GiB budget this
548
- * only fires at the wall; traces are otherwise never truncated.
549
- */
550
- export function selectTraceEvents(
551
- events: TraceEvent[],
552
- maxBytes: number,
553
- ): { events: TraceEvent[]; truncated: boolean; droppedCount: number } {
554
- if (events.length === 0) return { events: [], truncated: false, droppedCount: 0 };
555
- const fits = (candidate: TraceEvent[]) => Buffer.byteLength(JSON.stringify(candidate)) <= maxBytes;
556
- if (fits(events)) return { events, truncated: false, droppedCount: 0 };
557
- const selected: TraceEvent[] = [];
558
- for (const event of events) {
559
- if (!fits([...selected, event])) break;
560
- selected.push(event);
561
- }
562
- return { events: selected, truncated: true, droppedCount: events.length - selected.length };
563
- }
564
-
565
- /**
566
- * FULL-WALK enumeration: every path in the workspace tree — regular
567
- * files, symlinks (including links to directories; links are recorded,
568
- * never followed, so cycles are impossible), and .git internals. The
569
- * ONLY exclusions are the secrets-denylist paths. Git-aware listing was
570
- * removed on purpose: `git ls-files` drops anything .gitignored (build
571
- * outputs, node_modules), which broke the whole-project guarantee.
572
- */
573
- export async function listWorkspaceFiles(root: string): Promise<string[]> {
574
- return walkWorkspace(root);
575
- }
576
-
577
- async function walkWorkspace(root: string, directory = root, output: string[] = []): Promise<string[]> {
578
- let entries;
579
- try {
580
- entries = await readdir(directory, { withFileTypes: true });
581
- } catch {
582
- return output;
583
- }
584
- for (const entry of entries) {
585
- const fullPath = resolve(directory, entry.name);
586
- const path = portablePath(root, fullPath);
587
- if (!path || isCollectorPathDenied(path)) continue;
588
- if (entry.isDirectory()) await walkWorkspace(root, fullPath, output);
589
- else output.push(path); // regular files AND symlinks (files or dirs)
590
- }
591
- return output;
592
- }
593
-
594
- async function gitMetadata(root: string): Promise<Record<string, string | null>> {
595
- const git = async (...args: string[]) => {
596
- try {
597
- const { stdout } = await execFileAsync("git", ["-C", root, ...args], { timeout: 5_000, maxBuffer: 64 * 1024 });
598
- return String(stdout).trim() || null;
599
- } catch {
600
- return null;
601
- }
602
- };
603
- const [commit, branch, status] = await Promise.all([
604
- git("rev-parse", "HEAD"),
605
- git("branch", "--show-current"),
606
- git("status", "--porcelain", "--untracked-files=no"),
607
- ]);
608
- return { commit, branch, dirty: status ? "true" : "false" };
609
- }
610
-
611
- async function workspaceSignature(root: string): Promise<string> {
612
- const hash = createHash("sha256");
613
- for (const path of await listWorkspaceFiles(root)) {
614
- try {
615
- const file = await lstat(resolve(root, path));
616
- if (file.isFile()) {
617
- hash.update(path).update("\0").update(String(file.size)).update("\0").update(String(file.mtimeMs)).update("\0");
618
- } else if (file.isSymbolicLink()) {
619
- hash.update(path).update("\0").update("link").update("\0").update(String(file.mtimeMs)).update("\0");
620
- }
621
- } catch {
622
- // A file can disappear while the workspace is being scanned.
623
- }
624
- }
625
- return hash.digest("hex");
626
- }
627
-
628
- function sha256Hex(content: string): string {
629
- return createHash("sha256").update(content).digest("hex");
630
- }
631
-
632
- /** Split a string into <= maxUnits slices without splitting surrogates. */
633
- export function sliceString(content: string, maxUnits = MAX_SLICE_BYTES): string[] {
634
- if (content.length <= maxUnits) return [content];
635
- const slices: string[] = [];
636
- let start = 0;
637
- while (start < content.length) {
638
- let end = Math.min(start + maxUnits, content.length);
639
- if (end < content.length) {
640
- const code = content.charCodeAt(end - 1);
641
- if (code >= 0xd800 && code <= 0xdbff) end -= 1;
642
- }
643
- slices.push(content.slice(start, end));
644
- start = end;
645
- }
646
- return slices;
647
- }
648
-
649
- /** Slice entry virtual path + manifest path for a logical file path. */
650
- export function slicePartPath(path: string, index: number): string {
651
- return `${path}.__agent_part${index}`;
652
- }
653
- export function sliceManifestPath(path: string): string {
654
- return `${path}.__agent_manifest.json`;
655
- }
656
-
657
- /**
658
- * Slice one logical file's content into <= MAX_SLICE_BYTES entries plus a
659
- * manifest record (path, slice count, per-slice byte size + sha256) so
660
- * replay reassembles the original deterministically. Small files pass
661
- * through unchanged.
662
- */
663
- export function buildFileEntries(file: CollectorFile): CollectorFile[] {
664
- const content = file.content ?? "";
665
- if (Buffer.byteLength(content) <= MAX_SLICE_BYTES) return [file];
666
- const slices = sliceString(content);
667
- const descriptors = slices.map((slice, index) => ({
668
- index,
669
- bytes: Buffer.byteLength(slice),
670
- sha256: sha256Hex(slice),
671
- }));
672
- return [
673
- { path: sliceManifestPath(file.path), content: JSON.stringify(sliceManifest(file, descriptors)) },
674
- ...slices.map((slice, index) => ({
675
- path: slicePartPath(file.path, index),
676
- content: slice,
677
- ...(file.encoding ? { encoding: file.encoding } : {}),
678
- })),
679
- ];
680
- }
681
-
682
- /**
683
- * Manifest for a sliced file. `slice_count` is the GLOBAL count and each
684
- * entry carries its global `index`, so manifests arriving across
685
- * different parts union deterministically: a replay collects the part
686
- * entries by index (verifying per-slice sha256) until all
687
- * `slice_count` indexes are present, then reassembles.
688
- */
689
- function sliceManifest(
690
- file: { path: string; encoding?: "base64"; mode?: string },
691
- sliceDescriptors: Array<{ index: number; bytes: number; sha256: string }>,
692
- totalCount?: number,
693
- ): Record<string, unknown> {
694
- return {
695
- schema_version: 1,
696
- path: file.path,
697
- encoding: file.encoding ?? null,
698
- mode: file.mode ?? null,
699
- slice_count: totalCount ?? sliceDescriptors.length,
700
- slices: sliceDescriptors,
701
- };
702
- }
703
-
704
- type ContentChunk = { content: string; encoding?: "base64" };
705
-
706
- /**
707
- * Stream a regular file as <= ~8 MiB content chunks, yielding each chunk
708
- * the moment it is produced — a file's content is NEVER materialized as
709
- * one string regardless of size (issue #6: the old array version held
710
- * every chunk at once). Small files are a single whole read; big files
711
- * stream in fixed raw chunks. Binary detection happens on the first
712
- * chunk and applies to the whole file; text is redacted per
713
- * line-aligned segment.
714
- */
715
- async function* iterateContentChunks(absolute: string, size: number): AsyncGenerator<ContentChunk> {
716
- if (size <= WHOLE_READ_BYTES) {
717
- const buffer = await readFile(absolute);
718
- if (isBinary(buffer)) {
719
- yield { content: buffer.toString("base64"), encoding: "base64" };
720
- return;
721
- }
722
- yield { content: redactCollectorText(buffer.toString("utf8")).text };
723
- return;
724
- }
725
- const { open } = await import("node:fs/promises");
726
- const handle = await open(absolute, "r");
727
- try {
728
- const chunkBytes = 8 * 1024 * 1024;
729
- const buffer = Buffer.alloc(chunkBytes);
730
- const first = await handle.read(buffer, 0, chunkBytes, null);
731
- const binary = isBinary(buffer.subarray(0, first.bytesRead));
732
- if (binary) {
733
- // Binaries carry no redactable text: base64 each raw chunk.
734
- yield { content: buffer.subarray(0, first.bytesRead).toString("base64"), encoding: "base64" };
735
- let read = 0;
736
- while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
737
- yield { content: buffer.subarray(0, read).toString("base64"), encoding: "base64" };
738
- }
739
- return;
740
- }
741
- // Text: decode incrementally (StringDecoder absorbs multibyte
742
- // sequences split across chunk boundaries), emit only up to the
743
- // last complete line, carry the remainder, and redact each
744
- // line-aligned segment. A pathological newline-less file must not
745
- // grow `pending` forever: at PENDING_TEXT_CAP_BYTES the segment is
746
- // flushed mid-line (line-anchored redaction weakens at that rare
747
- // cut; unbounded memory is worse).
748
- const { StringDecoder } = await import("node:string_decoder");
749
- const decoder = new StringDecoder("utf8");
750
- let pending = decoder.write(buffer.subarray(0, first.bytesRead));
751
- let read = 0;
752
- while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
753
- pending += decoder.write(buffer.subarray(0, read));
754
- for (;;) {
755
- const lastNl = pending.lastIndexOf("\n");
756
- if (lastNl >= 0) {
757
- yield { content: redactCollectorText(pending.slice(0, lastNl + 1)).text };
758
- pending = pending.slice(lastNl + 1);
759
- continue;
760
- }
761
- if (pending.length >= PENDING_TEXT_CAP_BYTES) {
762
- yield { content: redactCollectorText(pending).text };
763
- pending = "";
764
- }
765
- break;
766
- }
767
- }
768
- pending += decoder.end();
769
- if (pending) yield { content: redactCollectorText(pending).text };
770
- } finally {
771
- await handle.close();
772
- }
773
- }
774
-
775
- /**
776
- * Stream one workspace entry as CollectorFile entries. Regular files:
777
- * text is redacted, binaries base64-encoded ("encoding": "base64"), and
778
- * the POSIX mode rides along (octal string) so replay can restore the
779
- * executable bit (stored-and-ignored on Windows). Content bigger than
780
- * MAX_SLICE_BYTES is sliced on the fly — slices are yielded the moment
781
- * they exist while only the tiny sha256 descriptors accumulate, then
782
- * the manifest is yielded LAST (replay unions manifests by index, so
783
- * position never matters). Symlinks become text entries
784
- * { path, content: <target>, encoding: "symlink" } — the backend's
785
- * per-entry validation requires text content, and replay recreates the
786
- * actual link from the target. Yields nothing when the entry vanished
787
- * or is a special file.
788
- */
789
- async function* iterateWorkspaceEntries(root: string, path: string): AsyncGenerator<CollectorFile> {
790
- let absolute = "";
791
- try {
792
- absolute = resolve(root, path);
793
- if (portablePath(root, absolute).startsWith("../")) return;
794
- const file = await lstat(absolute);
795
- if (file.isSymbolicLink()) {
796
- const target = await readlink(absolute);
797
- yield { path, content: target, encoding: "symlink" };
798
- return;
799
- }
800
- if (!file.isFile()) return;
801
- const mode = file.mode & 0o777;
802
- const modeField = mode ? { mode: mode.toString(8) } : {};
803
- if (file.size <= WHOLE_READ_BYTES) {
804
- // Small file: whole read, real path preserved; buildFileEntries
805
- // slices (manifest + parts) when the encoded content exceeds
806
- // MAX_SLICE_BYTES — same shape as always.
807
- for await (const chunk of iterateContentChunks(absolute, file.size)) {
808
- for (const entry of buildFileEntries({
809
- path,
810
- content: chunk.content,
811
- ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
812
- ...modeField,
813
- })) {
814
- yield entry;
815
- }
816
- }
817
- return;
818
- }
819
- // Big file (streamed): slices are yielded the moment they exist;
820
- // only the tiny sha256 descriptors accumulate. The reassembly
821
- // manifest is yielded LAST (replay unions manifests by index, so
822
- // position never matters).
823
- let index = 0;
824
- const descriptors: Array<{ index: number; bytes: number; sha256: string }> = [];
825
- let sawEncoding: "base64" | undefined;
826
- for await (const chunk of iterateContentChunks(absolute, file.size)) {
827
- sawEncoding ??= chunk.encoding;
828
- for (const slice of sliceString(chunk.content)) {
829
- descriptors.push({ index, bytes: Buffer.byteLength(slice), sha256: sha256Hex(slice) });
830
- yield {
831
- path: slicePartPath(path, index),
832
- content: slice,
833
- ...(sawEncoding ? { encoding: sawEncoding } : {}),
834
- };
835
- index += 1;
836
- }
837
- }
838
- if (index === 0) return;
839
- yield {
840
- path: sliceManifestPath(path),
841
- content: JSON.stringify(sliceManifest(
842
- { path, ...(sawEncoding ? { encoding: sawEncoding } : {}), ...modeField },
843
- descriptors,
844
- )),
845
- };
846
- return;
847
- } catch {
848
- // Workspaces are live; races are expected and retried by the next snapshot.
849
- return;
850
- }
851
- }
852
-
853
- /**
854
- * Streaming file collection: yields workspace files (metadata entry
855
- * first) one at a time so the one-shot path never holds the whole tree's
856
- * content in memory (issue #6). Same order, denylist, and byte budget
857
- * as the compat collectFiles drain below.
858
- */
859
- export async function* iterateCollectFiles(
860
- root: string,
861
- byteLimit: number,
862
- workspaceId: string,
863
- session?: Pick<SessionState, "id" | "segment" | "resumed">,
864
- ): AsyncGenerator<CollectorFile> {
865
- let used = 0;
866
- const metadata = JSON.stringify({
867
- workspace_id: workspaceId,
868
- ...(session ? {
869
- session_id: session.id,
870
- session_segment: session.segment,
871
- session_resumed: session.resumed,
872
- } : {}),
873
- root_name: root.split(sep).filter(Boolean).at(-1) ?? "workspace",
874
- git: await gitMetadata(root),
875
- });
876
- yield { path: `${AGENT_DIR}/workspace.json`, content: metadata };
877
- used += Buffer.byteLength(metadata);
878
-
879
- const paths = (await listWorkspaceFiles(root)).sort(comparePaths);
880
- for (const path of paths) {
881
- if (used >= byteLimit) break;
882
- for await (const file of iterateWorkspaceEntries(root, path)) {
883
- const size = Buffer.byteLength(file.content ?? "");
884
- if (used + size > byteLimit) continue;
885
- yield file;
886
- used += size;
887
- }
888
- }
889
- }
890
-
891
- /** Compat drain of iterateCollectFiles (tests + tooling). */
892
- export async function collectFiles(
893
- root: string,
894
- byteLimit: number,
895
- workspaceId: string,
896
- session?: Pick<SessionState, "id" | "segment" | "resumed">,
897
- ): Promise<CollectorFile[]> {
898
- const files: CollectorFile[] = [];
899
- for await (const file of iterateCollectFiles(root, byteLimit, workspaceId, session)) {
900
- files.push(file);
901
- }
902
- return files;
903
- }
904
-
905
- export function compressZstd(payload: Buffer): Promise<Buffer> {
906
- return new Promise((resolvePromise, reject) => {
907
- try {
908
- const maybePromise = zstdCompressCb(payload, (error: unknown, result: Buffer) => {
909
- if (error) reject(error);
910
- else resolvePromise(result);
911
- });
912
- if (maybePromise && typeof (maybePromise as any).then === "function") {
913
- (maybePromise as Promise<Buffer>).then(resolvePromise, reject);
914
- }
915
- } catch (error) {
916
- reject(error);
917
- }
918
- });
919
- }
920
-
921
- /** The uncompressed collect envelope (contract: schema_version 1).
922
- * Additive metadata: client_version / client_platform when known and a
923
- * sent_at timestamp — the backend ignores what it does not know. */
924
- export function buildEnvelopePayload(
925
- state: Pick<SessionState, "id" | "segment" | "resumed"> & {
926
- sequence?: number;
927
- clientVersion?: string;
928
- clientPlatform?: string;
929
- /** Capture time override (change snapshots carry the time the
930
- * journal records were captured, not the time the queued upload
931
- * finally built the envelope). */
932
- capturedAt?: string;
933
- },
934
- snapshotType: SnapshotType,
935
- files: CollectorFile[],
936
- ): Buffer {
937
- return Buffer.from(JSON.stringify({
938
- schema_version: 1,
939
- session_id: state.id,
940
- session_segment: state.segment,
941
- session_resumed: state.resumed,
942
- sequence: (state.sequence ?? 0) + 1,
943
- snapshot_type: snapshotType,
944
- ...(state.clientVersion ? { client_version: state.clientVersion } : {}),
945
- ...(state.clientPlatform ? { client_platform: state.clientPlatform } : {}),
946
- sent_at: new Date().toISOString(),
947
- // Capture time for the backend's per-turn diff attribution (the
948
- // desktop sends the same field; without it a CLI change snapshot
949
- // cannot be pinned to the turn that was running).
950
- captured_at: state.capturedAt ?? new Date().toISOString(),
951
- files,
952
- }));
953
- }
954
-
955
- export type EnvelopePart = {
956
- files: CollectorFile[];
957
- payload: Buffer;
958
- compressed: Buffer;
959
- };
960
-
961
- /**
962
- * Deterministic part split for a full files array (compat drain of
963
- * iterateEnvelopeParts, used by the tests): sort files by path,
964
- * accumulate in order, keep each part <= maxCompressed compressed.
965
- */
966
- export async function buildEnvelopeParts(
967
- state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
968
- snapshotType: SnapshotType,
969
- files: CollectorFile[],
970
- maxCompressed = MAX_PART_COMPRESSED_BYTES,
971
- log?: (message: string, attributes?: Record<string, unknown>) => void,
972
- ): Promise<EnvelopePart[]> {
973
- // Slice any entry whose content exceeds MAX_SLICE_BYTES (manifest +
974
- // ordered parts) BEFORE packing, so callers feeding raw file lists get
975
- // the same deterministic slice layout as the tree walker.
976
- const sorted = files
977
- .flatMap((file) => buildFileEntries(file))
978
- .sort((left, right) => comparePaths(left.path, right.path));
979
- const parts: EnvelopePart[] = [];
980
- for await (const part of iterateEnvelopeParts(state, snapshotType, sorted, maxCompressed, log)) {
981
- parts.push(part);
982
- }
983
- return parts;
984
- }
985
-
986
- /**
987
- * Streaming packer for the one-shot path: yields compressed parts in
988
- * order so callers can upload-and-discard instead of holding the whole
989
- * workspace's parts (payload + compressed buffers) in memory (issue #6).
990
- * Same greedy algorithm as before: split at the compressed target, a
991
- * lone file may use the hard rail, lone-over-rail files are skipped
992
- * loudly. File content is never split across parts.
993
- */
994
- export async function* iterateEnvelopeParts(
995
- state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
996
- snapshotType: SnapshotType,
997
- sortedFiles: Iterable<CollectorFile> | AsyncIterable<CollectorFile>,
998
- maxCompressed = MAX_PART_COMPRESSED_BYTES,
999
- log?: (message: string, attributes?: Record<string, unknown>) => void,
1000
- ): AsyncGenerator<EnvelopePart> {
1001
- const hard = Math.max(maxCompressed, MAX_PART_COMPRESSED_HARD);
1002
- const measure = async (batch: CollectorFile[], partIndex: number): Promise<EnvelopePart> => {
1003
- const payload = buildEnvelopePayload({ ...state, sequence: (state.sequence ?? 0) + partIndex }, snapshotType, batch);
1004
- return { files: batch, payload, compressed: await compressZstd(payload) };
1005
- };
1006
- let partsYielded = 0;
1007
- let batch: CollectorFile[] = [];
1008
- const fit = async (batchToFit: CollectorFile[], partIndex: number) => {
1009
- let measured = await measure(batchToFit, partIndex);
1010
- const carry: CollectorFile[] = [];
1011
- // Split at the compressed target; a lone file may use the body
1012
- // headroom up to the hard rail, beyond which it is skipped loudly
1013
- // rather than split mid-content.
1014
- while (measured.compressed.length > maxCompressed && batchToFit.length > 1) {
1015
- carry.unshift(batchToFit.pop()!);
1016
- measured = await measure(batchToFit, partIndex);
1017
- }
1018
- if (measured.compressed.length > hard && batchToFit.length === 1) {
1019
- const lone = batchToFit.pop()!;
1020
- log?.("OmniRush file exceeds the upload limits and cannot be shipped as a single part; skipped", {
1021
- path: lone.path,
1022
- bytes: Buffer.byteLength(lone.content ?? ""),
1023
- });
1024
- return { measured: null, carry };
1025
- }
1026
- return { measured, carry };
1027
- };
1028
- for await (const file of sortedFiles) {
1029
- batch.push(file);
1030
- const partIndex = partsYielded;
1031
- const estimate = batch.reduce((total, f) => total + Buffer.byteLength(f.content), 0);
1032
- if (estimate < PART_UNCOMPRESSED_STEP) continue;
1033
- const { measured, carry } = await fit(batch, partIndex);
1034
- if (measured
1035
- && (measured.compressed.length >= PART_READY_COMPRESSED_BYTES || estimate >= PART_BATCH_UNCOMPRESSED_MAX)) {
1036
- partsYielded += 1;
1037
- yield measured;
1038
- batch = carry;
1039
- } else if (measured) {
1040
- batch = [...batch, ...carry];
1041
- } else {
1042
- batch = carry;
1043
- }
1044
- }
1045
- while (batch.length > 0) {
1046
- const { measured, carry } = await fit(batch, partsYielded);
1047
- if (measured) {
1048
- partsYielded += 1;
1049
- yield measured;
1050
- }
1051
- if (carry.length === 0) break;
1052
- batch = carry;
1053
- }
1054
- }
1055
-
1056
- export class WorkspaceCollector {
1057
- /**
1058
- * Bounded first-chunk read of an error response body for diagnostics:
1059
- * at most 512 bytes, decoded and whitespace-collapsed. Never throws.
1060
- * Error bodies are small JSON from the manager; the bound exists so a
1061
- * pathological response cannot balloon a log line (or memory).
1062
- */
1063
- private static async errorBodySnippet(response: any): Promise<string> {
1064
- try {
1065
- const reader = response?.body?.getReader?.();
1066
- if (!reader) return "";
1067
- const { value } = await reader.read();
1068
- reader.cancel().catch(() => undefined);
1069
- if (!value) return "";
1070
- const text = Buffer.from(value).toString("utf8").replace(/\s+/g, " ").trim();
1071
- return text.length > 200 ? `${text.slice(0, 200)}…` : text;
1072
- } catch {
1073
- return "";
1074
- }
1075
- }
1076
-
1077
- private readonly collectUrl: string | null;
1078
- private token: string;
1079
- private readonly fetcher: typeof fetch;
1080
- private readonly refresh?: CollectorOptions["refresh"];
1081
- private readonly uploader?: CollectorOptions["upload"];
1082
- private readonly log: NonNullable<CollectorOptions["log"]>;
1083
- private readonly ledgerPath: string | null;
1084
- private readonly stateDir: string | null;
1085
- private readonly agentDir: string | null;
1086
- private readonly identityProvider?: CollectorOptions["identityProvider"];
1087
- private readonly clientId?: string;
1088
- private identity: { userId: string | null } | null = null;
1089
- private identityTried = false;
1090
- private readonly ledgerReady: Promise<void>;
1091
- private ledger: SessionLedger = { version: 1, sessions: {} };
1092
- private ledgerWriteTail: Promise<void> = Promise.resolve();
1093
- private readonly sessions = new Map<string, SessionState>();
1094
- /** Gate for states built without one (direct uploadEnvelope callers,
1095
- * tests) — per-session gates are created in startSession. */
1096
- private readonly defaultGate = new PriorityGate();
1097
- private readonly changeDebounceMs: number;
1098
- private readonly fallbackScanMs: number;
1099
- private readonly sessionBudgetBytes: number;
1100
- private readonly clientVersion: string;
1101
- private readonly clientPlatform: string;
1102
- private readonly linkedParentSession: string | null;
1103
- private readonly partCompressedBytes: number;
1104
- private readonly partCheckBytes: number;
1105
-
1106
- constructor(options: CollectorOptions = {}) {
1107
- this.collectUrl = resolveCollectUrl(options.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL);
1108
- this.token = (options.accessToken ?? process.env.OMNIRUSH_ACCESS_TOKEN ?? "").trim();
1109
- this.fetcher = options.fetch ?? ((input: any, init?: any) => fetch(input, init));
1110
- this.refresh = options.refresh;
1111
- this.uploader = options.upload;
1112
- this.log = options.log ?? (() => undefined);
1113
- this.stateDir = options.stateDir ? resolve(options.stateDir) : null;
1114
- this.agentDir = options.agentDir ? resolve(options.agentDir) : null;
1115
- this.identityProvider = options.identityProvider;
1116
- this.clientId = options.clientId;
1117
- this.ledgerPath = options.stateDir ? join(resolve(options.stateDir), SESSION_LEDGER_FILE) : null;
1118
- this.ledgerReady = this.loadLedger();
1119
- this.changeDebounceMs = options.changeDebounceMs ?? CHANGE_DEBOUNCE_MS;
1120
- this.fallbackScanMs = options.fallbackScanMs ?? FALLBACK_SCAN_MS;
1121
- this.sessionBudgetBytes = Math.max(1024, options.sessionBudgetBytes ?? MAX_SESSION_BYTES);
1122
- this.clientVersion = options.clientVersion ?? detectClientVersion();
1123
- this.clientPlatform = options.clientPlatform ?? detectClientPlatform();
1124
- this.linkedParentSession = (options.linkedParentSession ?? "").trim() || null;
1125
- this.partCompressedBytes = Math.min(
1126
- Math.max(1024, options.partCompressedBytes ?? MAX_PART_COMPRESSED_BYTES),
1127
- MAX_PART_COMPRESSED_HARD,
1128
- );
1129
- this.partCheckBytes = Math.max(1024, options.partCheckBytes ?? PART_UNCOMPRESSED_STEP);
1130
- }
1131
-
1132
- private async loadLedger(): Promise<void> {
1133
- this.ledger = await readSessionLedger(this.ledgerPath);
1134
- }
1135
-
1136
- private async saveLedger(): Promise<void> {
1137
- if (!this.ledgerPath) return;
1138
- await this.ledgerReady;
1139
- const entries = Object.entries(this.ledger.sessions)
1140
- .sort(([, left], [, right]) => right.lastSeenAt.localeCompare(left.lastSeenAt))
1141
- .slice(0, MAX_SESSION_LEDGER_ENTRIES);
1142
- this.ledger.sessions = Object.fromEntries(entries);
1143
- const snapshot = JSON.stringify(this.ledger);
1144
- this.ledgerWriteTail = this.ledgerWriteTail
1145
- .catch(() => undefined)
1146
- .then(async () => {
1147
- await mkdir(dirname(this.ledgerPath!), { recursive: true, mode: 0o700 });
1148
- await writeFile(this.ledgerPath!, snapshot, { encoding: "utf8", mode: 0o600 });
1149
- });
1150
- await this.ledgerWriteTail;
1151
- }
1152
-
1153
- /** The state fields every envelope builder needs, plus the additive
1154
- * client metadata. `sent_at`/`captured_at` are stamped by
1155
- * buildEnvelopePayload at build time. */
1156
- private sequenceState(
1157
- state: SessionState,
1158
- capturedAt?: string,
1159
- ): {
1160
- id: string;
1161
- segment: number;
1162
- resumed: boolean;
1163
- sequence: number;
1164
- clientVersion?: string;
1165
- clientPlatform?: string;
1166
- capturedAt?: string;
1167
- } {
1168
- return {
1169
- id: state.id,
1170
- segment: state.segment,
1171
- resumed: state.resumed,
1172
- sequence: state.sequence,
1173
- ...(this.clientVersion ? { clientVersion: this.clientVersion } : {}),
1174
- ...(this.clientPlatform ? { clientPlatform: this.clientPlatform } : {}),
1175
- // A change part is pinned to the capture time of its OLDEST
1176
- // journal record: the upload may queue behind the baseline, but
1177
- // the turn attribution must reflect when the edit was seen.
1178
- ...(capturedAt ? { capturedAt } : {}),
1179
- };
1180
- }
1181
-
1182
- private async prepareSession(state: SessionState): Promise<void> {
1183
- await this.ledgerReady;
1184
- const previous = this.ledger.sessions[state.id];
1185
- state.segment = (previous?.segment ?? 0) + 1;
1186
- state.sequence = previous?.nextSequence ?? 0;
1187
- state.sentBytes = previous?.sentBytes ?? 0;
1188
- state.lastMessageId = previous?.lastMessageId;
1189
- state.baselineUploaded = previous?.baselineUploaded === true;
1190
- // An interrupted baseline resumes from its cursor; a completed one
1191
- // (or a fresh session) has none.
1192
- state.baselineResumePath =
1193
- previous && !state.baselineUploaded
1194
- ? typeof previous.baselineResumePath === "string"
1195
- ? previous.baselineResumePath
1196
- : null
1197
- : null;
1198
- state.resumed = Boolean(previous);
1199
- if (previous?.traces && typeof previous.traces === "object") {
1200
- state.transcripts = { ...previous.traces };
1201
- }
1202
- if (state.resumed) {
1203
- state.trace.push({
1204
- at: new Date().toISOString(),
1205
- type: "session.resumed",
1206
- data: { session_segment: state.segment, previous_segment: previous?.segment ?? null },
1207
- });
1208
- }
1209
- this.ledger.sessions[state.id] = {
1210
- segment: state.segment,
1211
- nextSequence: state.sequence,
1212
- sentBytes: state.sentBytes,
1213
- ...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
1214
- ...(state.baselineUploaded ? { baselineUploaded: true } : {}),
1215
- ...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
1216
- lastSeenAt: new Date().toISOString(),
1217
- };
1218
- await this.saveLedger();
1219
- }
1220
-
1221
- private async persistSession(state: SessionState): Promise<void> {
1222
- await this.ledgerReady;
1223
- this.ledger.sessions[state.id] = {
1224
- segment: state.segment,
1225
- nextSequence: state.sequence,
1226
- sentBytes: state.sentBytes,
1227
- ...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
1228
- ...(state.baselineUploaded ? { baselineUploaded: true } : {}),
1229
- ...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
1230
- ...(Object.keys(state.transcripts).length > 0 ? { traces: state.transcripts } : {}),
1231
- lastSeenAt: new Date().toISOString(),
1232
- };
1233
- await this.saveLedger();
1234
- }
1235
-
1236
- async sessionCheckpoint(sessionId: string): Promise<{ resumed: boolean; segment?: number; lastMessageId?: string }> {
1237
- const state = this.sessions.get(sessionId);
1238
- if (!state) return { resumed: false };
1239
- await state.ready;
1240
- return { resumed: state.resumed, segment: state.segment, lastMessageId: state.lastMessageId };
1241
- }
1242
-
1243
- async setSessionCheckpoint(sessionId: string, messageId: string): Promise<void> {
1244
- const state = this.sessions.get(sessionId);
1245
- if (!state || state.finished) return;
1246
- await state.ready;
1247
- state.lastMessageId = messageId;
1248
- await this.persistSession(state);
1249
- }
1250
-
1251
- get enabled(): boolean {
1252
- return Boolean(this.uploader || (this.collectUrl && this.token));
1253
- }
1254
-
1255
- startSession(sessionId: string, workspaceId: string, root: string): void {
1256
- if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;
1257
- const state: SessionState = {
1258
- id: sessionId,
1259
- root,
1260
- workspaceId,
1261
- sentBytes: 0,
1262
- segment: 1,
1263
- sequence: 0,
1264
- resumed: false,
1265
- lastSignature: "",
1266
- started: false,
1267
- finished: false,
1268
- budgetExhausted: false,
1269
- baselineUploaded: false,
1270
- baselineResumePath: null,
1271
- changeTimer: null,
1272
- scanTimer: null,
1273
- knownStats: new Map(),
1274
- scanning: false,
1275
- baselineSeeded: false,
1276
- knownPaths: new Set(),
1277
- trace: [],
1278
- journalPath: this.stateDir ? join(resolve(this.stateDir), `journal-${sessionId}.ndjson`) : null,
1279
- journalAckOffset: 0,
1280
- journalAppendedBytes: 0,
1281
- journalAppendTail: Promise.resolve(),
1282
- journalUploadEof: null,
1283
- journalCaptureStopped: false,
1284
- journalLastByPath: new Map(),
1285
- transcripts: {},
1286
- changeCaptureTail: Promise.resolve(),
1287
- ready: Promise.resolve(),
1288
- tail: Promise.resolve(),
1289
- traceTail: Promise.resolve(),
1290
- tracePending: 0,
1291
- traceDrain: Promise.resolve(),
1292
- resolveTraceDrain: () => undefined,
1293
- gate: new PriorityGate(),
1294
- };
1295
- state.ready = this.prepareSession(state);
1296
- this.sessions.set(sessionId, state);
1297
- if (this.linkedParentSession) {
1298
- // Child of a live parent session: the parent owns the workspace
1299
- // collection (baseline/changes/end). This collector only traces.
1300
- state.started = true;
1301
- return;
1302
- }
1303
- this.enqueue(state, async () => {
1304
- await state.ready;
1305
- const baselinePaths = await listWorkspaceFiles(root);
1306
- state.knownPaths = new Set(baselinePaths);
1307
- state.lastSignature = await workspaceSignature(root);
1308
- // Seed the per-path stat snapshot so the scan loop only captures
1309
- // files that change AFTER the start snapshot.
1310
- for (const relPath of baselinePaths) {
1311
- try {
1312
- const stats = await lstat(resolve(root, relPath));
1313
- state.knownStats.set(relPath, { mtimeMs: stats.mtimeMs, size: stats.size });
1314
- } catch {
1315
- /* raced away — the next scan re-checks */
1316
- }
1317
- }
1318
- state.baselineSeeded = true;
1319
- if (state.resumed && state.baselineUploaded) {
1320
- // Resume of a session whose start baseline already shipped: do
1321
- // NOT re-upload the full tree. A restart-heavy session would
1322
- // otherwise re-burn the entire baseline (and starve traces) on
1323
- // every restart — observed live as 846 start snapshots /
1324
- // multiple GB for one session. Changes + the end snapshot cover
1325
- // the deltas; a FAILED baseline (flag unset) still retries here.
1326
- state.started = true;
1327
- return;
1328
- }
1329
- await this.uploadWorkspace(state, "start");
1330
- state.baselineUploaded = true;
1331
- state.baselineResumePath = null;
1332
- await this.persistSession(state);
1333
- state.started = true;
1334
- });
1335
- // Change detection: signature scan (unref'd, self-serializing). No
1336
- // fs.watch anywhere — its recursive handle keeps the event loop
1337
- // alive even after close()/unref(), hanging every session at exit.
1338
- state.scanTimer = setInterval(() => {
1339
- void this.scanForChanges(state).catch(() => undefined);
1340
- }, this.fallbackScanMs);
1341
- state.scanTimer.unref?.();
1342
- }
1343
-
1344
- /**
1345
- * Periodic change detection: cheap signature check first; on a real
1346
- * change, lstat-diff every path against knownStats and capture exactly
1347
- * the new/changed/deleted files into the on-disk journal.
1348
- */
1349
- private async scanForChanges(state: SessionState): Promise<void> {
1350
- if (state.scanning || state.finished || state.budgetExhausted) return;
1351
- if (!state.baselineSeeded) return;
1352
- state.scanning = true;
1353
- try {
1354
- const signature = await workspaceSignature(state.root);
1355
- if (!signature || signature === state.lastSignature) return;
1356
- const paths = (await listWorkspaceFiles(state.root)).sort(comparePaths);
1357
- const seen = new Set<string>();
1358
- for (const relPath of paths) {
1359
- seen.add(relPath);
1360
- let stats;
1361
- try {
1362
- stats = await lstat(resolve(state.root, relPath));
1363
- } catch {
1364
- continue;
1365
- }
1366
- if (isCollectorPathDenied(relPath)) continue;
1367
- const known = state.knownStats.get(relPath);
1368
- if (known && known.mtimeMs === stats.mtimeMs && known.size === stats.size) continue;
1369
- state.knownStats.set(relPath, { mtimeMs: stats.mtimeMs, size: stats.size });
1370
- await this.captureChangedPath(state, relPath, { force: true });
1371
- }
1372
- // Deletions: paths we knew about that vanished.
1373
- for (const [relPath] of state.knownStats) {
1374
- if (!seen.has(relPath)) {
1375
- state.knownStats.delete(relPath);
1376
- await this.captureChangedPath(state, relPath, { force: true });
1377
- }
1378
- }
1379
- state.lastSignature = signature;
1380
- this.scheduleChange(state);
1381
- } finally {
1382
- state.scanning = false;
1383
- }
1384
- }
1385
-
1386
- recordTrace(sessionId: string, type: string, data?: unknown): void {
1387
- const state = this.sessions.get(sessionId);
1388
- if (!state || state.finished || state.budgetExhausted) return;
1389
- state.trace.push({ at: new Date().toISOString(), type, ...(data === undefined ? {} : { data }) });
1390
- if (state.trace.length >= TRACE_RELIEF_EVENTS) {
1391
- // Memory relief, NOT truncation: the whole batch is uploaded.
1392
- const batch = state.trace.splice(0);
1393
- this.enqueueTrace(state, async () => {
1394
- await state.ready;
1395
- await this.uploadTrace(state, batch);
1396
- });
1397
- }
1398
- }
1399
-
1400
- finishSession(sessionId: string, finalTrace?: unknown): void {
1401
- const state = this.sessions.get(sessionId);
1402
- if (!state || state.finished) return;
1403
- const pendingTrace = state.trace.splice(0);
1404
- if (finalTrace !== undefined) {
1405
- pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
1406
- }
1407
- state.finished = true;
1408
- if (state.changeTimer) clearTimeout(state.changeTimer);
1409
- if (state.scanTimer) clearInterval(state.scanTimer);
1410
- // Final trace rides the PRIORITY lane: an in-flight baseline part
1411
- // finishes first, then the trace ships — never behind the tail of a
1412
- // multi-part tree upload.
1413
- this.enqueueTrace(state, async () => {
1414
- await state.ready;
1415
- if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
1416
- await this.uploadTrace(state, pendingTrace);
1417
- });
1418
- this.enqueue(state, async () => {
1419
- await state.ready;
1420
- await state.changeCaptureTail;
1421
- await this.teardownJournal(state); // ship unacked change records, then unlink
1422
- // The end snapshot yields to pending trace work (yieldToTrace in
1423
- // the packer), so the final trace lands first.
1424
- if (!this.linkedParentSession) {
1425
- // A linked child ships no end snapshot: the parent session's
1426
- // collection covers the workspace tree.
1427
- await this.uploadWorkspace(state, "end");
1428
- }
1429
- this.sessions.delete(sessionId);
1430
- });
1431
- }
1432
-
1433
- flushTrace(sessionId: string, finalTrace?: unknown): void {
1434
- const state = this.sessions.get(sessionId);
1435
- if (!state || state.finished) return;
1436
- const pendingTrace = state.trace.splice(0);
1437
- if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });
1438
- if (pendingTrace.length === 0) return;
1439
- // Per-turn flush: the complete trace artifact (events + transcript
1440
- // messages-so-far) ships on the priority lane, ahead of any queued
1441
- // baseline parts.
1442
- this.enqueueTrace(state, async () => {
1443
- await state.ready;
1444
- if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
1445
- await this.uploadTrace(state, pendingTrace);
1446
- });
1447
- }
1448
-
1449
- async stop(): Promise<void> {
1450
- for (const sessionId of [...this.sessions.keys()]) this.finishSession(sessionId);
1451
- await Promise.allSettled(
1452
- [...this.sessions.values()].flatMap((state) => [state.tail, state.traceTail]),
1453
- );
1454
- }
1455
-
1456
- /**
1457
- * Queue trace work on the priority lane and mark it pending: the
1458
- * background lane's `yieldToTrace` waits for `tracePending` to drain
1459
- * before starting another baseline part, so trace work enqueued while
1460
- * a baseline part was in flight goes next.
1461
- */
1462
- private enqueueTrace(state: SessionState, operation: () => Promise<void>): void {
1463
- if (state.tracePending === 0) {
1464
- let resolveDrain!: () => void;
1465
- state.traceDrain = new Promise<void>((resolvePromise) => {
1466
- resolveDrain = resolvePromise;
1467
- });
1468
- state.resolveTraceDrain = () => resolveDrain();
1469
- }
1470
- state.tracePending += 1;
1471
- state.traceTail = state.traceTail
1472
- .then(operation)
1473
- .catch((error: unknown) => {
1474
- this.log("warn", "OmniRush collection operation failed", {
1475
- sessionId: state.id,
1476
- error: error instanceof Error ? error.message : "unknown",
1477
- });
1478
- })
1479
- .finally(() => {
1480
- state.tracePending -= 1;
1481
- if (state.tracePending === 0) state.resolveTraceDrain();
1482
- });
1483
- }
1484
-
1485
- /** Wait until no trace work is queued or running (re-arming safe). */
1486
- private async yieldToTrace(state: SessionState): Promise<void> {
1487
- while (state.tracePending > 0) {
1488
- await state.traceDrain;
1489
- }
1490
- }
1491
-
1492
- private stopWatching(state: SessionState): void {
1493
- if (state.changeTimer) {
1494
- clearTimeout(state.changeTimer);
1495
- state.changeTimer = null;
1496
- }
1497
- if (state.scanTimer) {
1498
- clearInterval(state.scanTimer);
1499
- state.scanTimer = null;
1500
- }
1501
- }
1502
-
1503
- private exhaustSessionBudget(state: SessionState): void {
1504
- if (state.budgetExhausted) return;
1505
- state.budgetExhausted = true;
1506
- this.stopWatching(state);
1507
- this.log("warn", "OmniRush session budget exhausted — collection stopped for this session", {
1508
- sessionId: state.id,
1509
- budgetBytes: this.sessionBudgetBytes,
1510
- sentBytes: state.sentBytes,
1511
- });
1512
- }
1513
-
1514
- /** Run the debounced change-snapshot upload immediately. */
1515
- private flushChangeSnapshot(state: SessionState): void {
1516
- if (state.finished || state.budgetExhausted) return;
1517
- if (state.changeTimer) {
1518
- clearTimeout(state.changeTimer);
1519
- state.changeTimer = null;
1520
- }
1521
- this.enqueue(state, async () => {
1522
- if (state.budgetExhausted || state.finished) return;
1523
- // The JOURNAL is what holds unshipped change records: a scan
1524
- // captures edits and advances lastSignature in the same breath,
1525
- // so by the time this debounced flush runs (it can queue behind a
1526
- // long multi-part baseline) the workspace signature often equals
1527
- // lastSignature again and a signature-only guard would skip
1528
- // forever, stranding the journal until session end. Ship whenever
1529
- // unacked journal records exist; the signature is only the
1530
- // trigger for captures that have not been journalled yet.
1531
- const unackedJournal = state.journalAppendedBytes > state.journalAckOffset;
1532
- const signature = await workspaceSignature(state.root);
1533
- if (!unackedJournal && (!signature || signature === state.lastSignature)) return;
1534
- state.lastSignature = signature;
1535
- await this.uploadWorkspace(state, "change");
1536
- });
1537
- }
1538
-
1539
- private scheduleChange(state: SessionState): void {
1540
- if (state.finished || state.budgetExhausted) return;
1541
- if (state.changeTimer) clearTimeout(state.changeTimer);
1542
- state.changeTimer = setTimeout(() => {
1543
- state.changeTimer = null;
1544
- this.flushChangeSnapshot(state);
1545
- }, this.changeDebounceMs);
1546
- state.changeTimer.unref?.();
1547
- }
1548
-
1549
- private async captureChangedPath(
1550
- state: SessionState,
1551
- filename: string,
1552
- options?: { force?: boolean },
1553
- ): Promise<void> {
1554
- const absolute = resolve(state.root, filename);
1555
- const path = portablePath(state.root, absolute);
1556
- if (!path || path.startsWith("../") || isCollectorPathDenied(path)) return;
1557
- const records: ChangeJournalEntry[] = [];
1558
- let entry: ChangeJournalEntry;
1559
- try {
1560
- const file = await lstat(absolute);
1561
- state.knownPaths.add(path);
1562
- if (file.isSymbolicLink()) {
1563
- // Symlink record: the target as text content with the symlink
1564
- // encoding — replay recreates the actual link from it.
1565
- const target = await readlink(absolute);
1566
- entry = {
1567
- path,
1568
- at: new Date().toISOString(),
1569
- status: "present",
1570
- content: target,
1571
- encoding: "symlink",
1572
- };
1573
- } else if (!file.isFile()) {
1574
- return;
1575
- } else {
1576
- const at = new Date().toISOString();
1577
- const mode = file.mode & 0o777;
1578
- const modeField = mode ? { mode: mode.toString(8) } : {};
1579
- // STREAMED capture: each chunk becomes a journal record the
1580
- // moment it exists — a big file is never held in memory whole
1581
- // (issue #6). One record per <= MAX_SLICE_BYTES slice; slice
1582
- // metadata rides the records only when reassembly is needed.
1583
- let sliceCount = 0;
1584
- for await (const chunk of iterateContentChunks(absolute, file.size)) {
1585
- for (const content of sliceString(chunk.content)) {
1586
- sliceCount += 1;
1587
- records.push({
1588
- path,
1589
- at,
1590
- status: "present",
1591
- content,
1592
- ...(chunk.encoding ? { encoding: chunk.encoding } : {}),
1593
- ...modeField,
1594
- });
1595
- }
1596
- }
1597
- if (sliceCount > 1) {
1598
- records.forEach((record, index) => {
1599
- record.slice = {
1600
- index,
1601
- total: sliceCount,
1602
- bytes: Buffer.byteLength(record.content ?? ""),
1603
- sha256: sha256Hex(record.content ?? ""),
1604
- };
1605
- });
1606
- }
1607
- if (sliceCount === 1) {
1608
- entry = records.pop()!;
1609
- } else {
1610
- await this.appendJournalRecords(state, records);
1611
- return;
1612
- }
1613
- }
1614
- } catch (error) {
1615
- // Only a real disappearance is a deletion. Any other lstat failure
1616
- // (EPERM/EBUSY/... under heavy watcher load) must not fabricate a
1617
- // deleted record for a file that still exists — the fallback scan
1618
- // and the end snapshot re-check those paths.
1619
- if ((error as any)?.code !== "ENOENT") return;
1620
- // Only paths the baseline (or a captured change) saw can be
1621
- // deleted inside the workspace; FSEvents sibling/parent leaks
1622
- // reference foreign paths that must not fabricate deletions.
1623
- if (!state.knownPaths.has(path)) return;
1624
- entry = { path, at: new Date().toISOString(), status: "deleted" };
1625
- }
1626
- // Consecutive-duplicate suppression: macOS FSEvents can deliver a
1627
- // late event for a path that was also captured directly, and an
1628
- // identical re-capture adds upload fat without a state change. The
1629
- // replay result is identical either way. The last signature per
1630
- // path lives in a tiny in-memory map (path strings only — content
1631
- // is already on disk in the journal).
1632
- const journalSignature = (record: ChangeJournalEntry) =>
1633
- JSON.stringify([record.status, record.content, record.encoding, record.mode]);
1634
- const pendingRecords = [entry, ...records];
1635
- const accepted: ChangeJournalEntry[] = [];
1636
- for (const record of pendingRecords) {
1637
- const signature = journalSignature(record);
1638
- if (state.journalLastByPath.get(record.path) === signature) continue;
1639
- state.journalLastByPath.set(record.path, signature);
1640
- accepted.push(record);
1641
- }
1642
- await this.appendJournalRecords(state, accepted);
1643
- }
1644
-
1645
- /**
1646
- * APPEND records to the on-disk journal (ndjson — one JSON record per
1647
- * line, 0600). Returns immediately when the capture hit the soft disk
1648
- * cap (a backend down for a very long time must stop CAPTURING loudly
1649
- * rather than fill the disk or RAM — the end snapshot still ships the
1650
- * final tree). Serialized through the per-session append chain.
1651
- */
1652
- private appendJournalRecords(state: SessionState, records: ChangeJournalEntry[]): Promise<void> {
1653
- if (!state.journalPath || records.length === 0) return Promise.resolve();
1654
- if (state.journalCaptureStopped) return Promise.resolve();
1655
- const unacked = state.journalAppendedBytes - state.journalAckOffset;
1656
- const capMb = Number(process.env.OMNIRUSH_JOURNAL_DISK_CAP_MB);
1657
- const capBytes = Number.isFinite(capMb) && capMb > 0 ? capMb * 1024 * 1024 : JOURNAL_DISK_CAP_BYTES;
1658
- if (unacked > capBytes) {
1659
- state.journalCaptureStopped = true;
1660
- this.log("warn", "OmniRush change journal hit the disk cap — change capture stopped for this session", {
1661
- sessionId: state.id,
1662
- capBytes: capBytes,
1663
- unackedBytes: unacked,
1664
- });
1665
- state.trace.push({
1666
- at: new Date().toISOString(),
1667
- type: "journal.capture_stopped",
1668
- data: { cap_bytes: capBytes, unacked_bytes: unacked },
1669
- });
1670
- return Promise.resolve();
1671
- }
1672
- const lines = records.map((record) => JSON.stringify(record) + "\n").join("");
1673
- const bytes = Buffer.byteLength(lines);
1674
- state.journalAppendTail = state.journalAppendTail
1675
- .catch(() => undefined)
1676
- .then(async () => {
1677
- const { appendFile, mkdir } = await import("node:fs/promises");
1678
- await mkdir(dirname(state.journalPath!), { recursive: true, mode: 0o700 });
1679
- await appendFile(state.journalPath!, lines, { encoding: "utf8", mode: 0o600 });
1680
- state.journalAppendedBytes += bytes;
1681
- });
1682
- return state.journalAppendTail;
1683
- }
1684
-
1685
- /**
1686
- * Stream the journal's unacked records (from journalAckOffset to the
1687
- * upload-start EOF) as parsed entries, each tagged with its raw line
1688
- * bytes via a NON-enumerable property so the changes.json sidecar
1689
- * serialization never sees it. Records appended while the upload runs
1690
- * land beyond the read window and simply ship with the next snapshot.
1691
- */
1692
- private async *readJournalRecords(state: SessionState): AsyncGenerator<ChangeJournalEntry> {
1693
- if (!state.journalPath || state.journalAckOffset >= state.journalUploadEof) return;
1694
- const { createReadStream } = await import("node:fs");
1695
- const readline = await import("node:readline");
1696
- const stream = createReadStream(state.journalPath, {
1697
- start: state.journalAckOffset,
1698
- end: state.journalUploadEof! - 1,
1699
- encoding: "utf8",
1700
- });
1701
- const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
1702
- for await (const line of rl) {
1703
- if (!line.trim()) continue;
1704
- try {
1705
- const record = JSON.parse(line) as ChangeJournalEntry;
1706
- Object.defineProperty(record, RAW_BYTES_FIELD, {
1707
- value: Buffer.byteLength(line) + 1,
1708
- enumerable: false,
1709
- configurable: true,
1710
- });
1711
- yield record;
1712
- } catch {
1713
- this.log("warn", "OmniRush journal line was unreadable — skipped", { sessionId: state.id });
1714
- }
1715
- }
1716
- }
1717
-
1718
- /**
1719
- * Upload the journal's unacked records as change snapshots, then
1720
- * compact: everything acked is removed from the file; records appended
1721
- * during the upload (past journalUploadEof) survive untouched.
1722
- */
1723
- private async uploadChangeJournal(state: SessionState): Promise<void> {
1724
- if (!state.journalPath) return;
1725
- let fileSize = 0;
1726
- try {
1727
- fileSize = (await lstat(state.journalPath)).size;
1728
- } catch {
1729
- return; // no journal yet — nothing captured
1730
- }
1731
- if (fileSize <= state.journalAckOffset) return;
1732
- const ackStart = state.journalAckOffset;
1733
- state.journalUploadEof = fileSize;
1734
- try {
1735
- await this.uploadChangeParts(state, this.readJournalRecords(state), {
1736
- onAck: (batch) => {
1737
- for (const record of batch) {
1738
- const raw = (record as Record<symbol, unknown>)[RAW_BYTES_FIELD];
1739
- if (typeof raw === "number") state.journalAckOffset += raw;
1740
- }
1741
- },
1742
- });
1743
- } finally {
1744
- state.journalUploadEof = null;
1745
- }
1746
- if (state.journalAckOffset <= ackStart) return; // nothing acked — keep everything
1747
- await this.compactJournal(state, state.journalAckOffset);
1748
- state.journalAckOffset = 0;
1749
- }
1750
-
1751
- /** Rewrite the journal without its first `dropBytes` bytes. */
1752
- private async compactJournal(state: SessionState, dropBytes: number): Promise<void> {
1753
- if (!state.journalPath) return;
1754
- const { createReadStream, createWriteStream } = await import("node:fs");
1755
- const { rename } = await import("node:fs/promises");
1756
- const tmpPath = `${state.journalPath}.compact`;
1757
- await new Promise<void>((resolvePromise, rejectPromise) => {
1758
- const input = createReadStream(state.journalPath!, { start: dropBytes, encoding: "utf8" });
1759
- const output = createWriteStream(tmpPath, { encoding: "utf8", mode: 0o600 });
1760
- input.on("error", rejectPromise);
1761
- output.on("error", rejectPromise);
1762
- output.on("finish", () => resolvePromise());
1763
- input.pipe(output);
1764
- });
1765
- await rename(tmpPath, state.journalPath);
1766
- }
1767
-
1768
- /** Upload unacked journal records (best effort) and remove the file. */
1769
- private async teardownJournal(state: SessionState): Promise<void> {
1770
- try {
1771
- await this.uploadChangeJournal(state);
1772
- } catch {
1773
- /* best effort — the end snapshot still ships the final tree */
1774
- }
1775
- if (state.journalPath) {
1776
- const { rm } = await import("node:fs/promises");
1777
- await rm(state.journalPath, { force: true }).catch(() => undefined);
1778
- }
1779
- }
1780
-
1781
- private enqueue(state: SessionState, operation: () => Promise<void>): void {
1782
- state.tail = state.tail.then(operation).catch((error: unknown) => {
1783
- this.log("warn", "OmniRush collection operation failed", {
1784
- sessionId: state.id,
1785
- error: error instanceof Error ? error.message : "unknown",
1786
- });
1787
- });
1788
- }
1789
-
1790
- private async uploadWorkspace(state: SessionState, type: "start" | "change" | "end"): Promise<void> {
1791
- if (state.budgetExhausted) return;
1792
- if (type === "change") {
1793
- // Change snapshots carry ONLY the touched files — streamed from
1794
- // the on-disk journal, acked per part, then compacted. Never a
1795
- // full re-enumeration of the tree, never held in memory.
1796
- await this.uploadChangeJournal(state);
1797
- } else {
1798
- await this.uploadTreeParts(state, type);
1799
- }
1800
- state.lastSignature = await workspaceSignature(state.root);
1801
- }
1802
-
1803
- private async workspaceMetadataFile(state: SessionState): Promise<CollectorFile> {
1804
- return {
1805
- path: `${AGENT_DIR}/workspace.json`,
1806
- content: JSON.stringify({
1807
- workspace_id: state.workspaceId,
1808
- session_id: state.id,
1809
- session_segment: state.segment,
1810
- session_resumed: state.resumed,
1811
- root_name: state.root.split(sep).filter(Boolean).at(-1) ?? "workspace",
1812
- git: await gitMetadata(state.root),
1813
- }),
1814
- };
1815
- }
1816
-
1817
- /** Full-tree snapshot ("start" | "end"), streamed into ordered parts.
1818
- * A start snapshot resumed after an interruption skips the paths the
1819
- * previous run already shipped (`baselineResumePath` cursor — the
1820
- * first unacked part onward); the end snapshot always ships whole. */
1821
- private async uploadTreeParts(state: SessionState, snapshotType: "start" | "end"): Promise<void> {
1822
- const paths = (await listWorkspaceFiles(state.root)).sort(comparePaths);
1823
- const metadata = await this.workspaceMetadataFile(state);
1824
- let metadataSent = false;
1825
- const resumeFrom = snapshotType === "start" ? state.baselineResumePath : null;
1826
- const measure = async (batch: CollectorFile[]) => {
1827
- const files = metadataSent ? batch : [metadata, ...batch];
1828
- const payload = buildEnvelopePayload(this.sequenceState(state), snapshotType, files);
1829
- return { files, payload, compressed: await compressZstd(payload) };
1830
- };
1831
- await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths, resumeFrom), {
1832
- sizeOf: (file) => Buffer.byteLength(file.content ?? "") + file.path.length + 64,
1833
- measure,
1834
- upload: async (measured) => {
1835
- const ok = await this.uploadEnvelope(state, snapshotType, measured.files, measured);
1836
- if (ok) {
1837
- metadataSent = true;
1838
- if (snapshotType === "start") {
1839
- // Advance the resume cursor to the last path of the acked
1840
- // part, so an interrupted run resumes at the first
1841
- // unacked part instead of re-burning the prefix.
1842
- const lastPath = measured.files[measured.files.length - 1]?.path;
1843
- if (typeof lastPath === "string") {
1844
- state.baselineResumePath = lastPath;
1845
- await this.persistSession(state).catch(() => undefined);
1846
- }
1847
- }
1848
- }
1849
- return ok;
1850
- },
1851
- drop: (file) => {
1852
- this.log("warn", "OmniRush file exceeds the upload rail and cannot be shipped as a single part; skipped", {
1853
- sessionId: state.id,
1854
- path: file.path,
1855
- bytes: Buffer.byteLength(file.content ?? ""),
1856
- });
1857
- state.trace.push({
1858
- at: new Date().toISOString(),
1859
- type: "file.unshippable",
1860
- data: { path: file.path, bytes: Buffer.byteLength(file.content ?? "") },
1861
- });
1862
- },
1863
- }, { yieldToTraceLane: true });
1864
- }
1865
-
1866
- private async *iterTreeFiles(
1867
- state: SessionState,
1868
- paths: string[],
1869
- resumeFrom?: string | null,
1870
- ): AsyncGenerator<CollectorFile> {
1871
- // The cursor can name a slice or manifest virtual path
1872
- // (`<path>.__agent_partN`, `<path>.__agent_manifest.json`) when the
1873
- // acked part ended mid-file. Comparing a virtual cursor against the
1874
- // real path list would never match and `path <= cursor` would
1875
- // swallow the whole base file (silent data loss on resume), so
1876
- // paths before the cursor's base file are skipped outright and the
1877
- // base file's own entries are filtered against the cursor.
1878
- const cursor = resumeFrom ?? null;
1879
- const cursorBase = cursor
1880
- ? cursor.replace(/\.__agent(?:_part\d+|_manifest\.json)$/, "")
1881
- : null;
1882
- for (const path of paths) {
1883
- if (state.budgetExhausted) return;
1884
- if (cursorBase) {
1885
- const order = comparePaths(path, cursorBase);
1886
- if (order < 0) continue;
1887
- if (order === 0) {
1888
- for await (const entry of iterateWorkspaceEntries(state.root, path)) {
1889
- if (entry.path && comparePaths(entry.path, cursor!) <= 0) continue;
1890
- yield entry;
1891
- }
1892
- continue;
1893
- }
1894
- }
1895
- yield* iterateWorkspaceEntries(state.root, path);
1896
- }
1897
- }
1898
-
1899
- /**
1900
- * Change snapshot: ordered journal slices. Each part carries the
1901
- * touched files for its slice (deduped last-wins, sorted by path) plus
1902
- * the __agent__/changes.json sidecar holding the slice's ordered
1903
- * records — replaying start + sidecars in sequence order reproduces the
1904
- * end tree.
1905
- */
1906
- private async uploadChangeParts(
1907
- state: SessionState,
1908
- units: Iterable<ChangeJournalEntry> | AsyncIterable<ChangeJournalEntry>,
1909
- opts: { onAck?: (batch: ChangeJournalEntry[]) => void } = {},
1910
- ): Promise<ChangeJournalEntry[][]> {
1911
- const uploadedSlices: ChangeJournalEntry[][] = [];
1912
- const filesForSlice = (entries: ChangeJournalEntry[]): CollectorFile[] => {
1913
- const lastByPath = new Map<string, CollectorFile>();
1914
- const slicedByPath = new Map<string, ChangeJournalEntry[]>();
1915
- for (const entry of entries) {
1916
- if (entry.status !== "present") continue;
1917
- if (entry.slice) {
1918
- const group = slicedByPath.get(entry.path) ?? [];
1919
- group.push(entry);
1920
- slicedByPath.set(entry.path, group);
1921
- } else if (entry.content !== undefined) {
1922
- lastByPath.set(entry.path, {
1923
- path: entry.path,
1924
- content: entry.content,
1925
- ...(entry.encoding ? { encoding: entry.encoding } : {}),
1926
- ...(entry.mode ? { mode: entry.mode } : {}),
1927
- });
1928
- }
1929
- }
1930
- const files = [...lastByPath.values()];
1931
- for (const [logicalPath, records] of slicedByPath) {
1932
- records.sort((left, right) => (left.slice?.index ?? 0) - (right.slice?.index ?? 0));
1933
- const encoding = records.find((record) => record.encoding)?.encoding;
1934
- const mode = records.find((record) => record.mode)?.mode;
1935
- for (const record of records) {
1936
- files.push({
1937
- path: slicePartPath(logicalPath, record.slice!.index),
1938
- content: record.content,
1939
- ...(encoding ? { encoding } : {}),
1940
- });
1941
- }
1942
- files.push({
1943
- path: sliceManifestPath(logicalPath),
1944
- content: JSON.stringify(sliceManifest(
1945
- {
1946
- path: logicalPath,
1947
- ...(encoding ? { encoding } : {}),
1948
- ...(mode ? { mode } : {}),
1949
- },
1950
- records.map((record) => ({
1951
- index: record.slice!.index,
1952
- bytes: record.slice!.bytes,
1953
- sha256: record.slice!.sha256,
1954
- })),
1955
- records[0].slice!.total,
1956
- )),
1957
- });
1958
- }
1959
- files.sort((left, right) => comparePaths(left.path, right.path));
1960
- files.push({
1961
- path: `${AGENT_DIR}/changes.json`,
1962
- content: JSON.stringify({ schema_version: 1, session_id: state.id, entries }),
1963
- });
1964
- return files;
1965
- };
1966
- const bySequence = (capturedAt?: string) => this.sequenceState(state, capturedAt);
1967
- await this.packAndUpload(state, "change", units, {
1968
- sizeOf: (entry) =>
1969
- Buffer.byteLength(entry.content ?? "") + entry.path.length + 96,
1970
- measure: async (batch) => {
1971
- const files = filesForSlice(batch);
1972
- const capturedAt = batch.find((entry) => typeof entry.at === "string")?.at;
1973
- const payload = buildEnvelopePayload(bySequence(capturedAt), "change", files);
1974
- return { files, payload, compressed: await compressZstd(payload) };
1975
- },
1976
- upload: async (measured, batch) => {
1977
- const ok = await this.uploadEnvelope(state, "change", measured.files, measured);
1978
- if (ok) {
1979
- uploadedSlices.push(batch);
1980
- opts.onAck?.(batch);
1981
- }
1982
- return ok;
1983
- },
1984
- drop: (entry) => {
1985
- this.log("warn", "OmniRush change record exceeds the upload limits and cannot be shipped; skipped", {
1986
- sessionId: state.id,
1987
- path: entry.path,
1988
- bytes: Buffer.byteLength(entry.content ?? ""),
1989
- });
1990
- state.trace.push({
1991
- at: new Date().toISOString(),
1992
- type: "file.unshippable",
1993
- data: { path: entry.path, reason: "change_record_over_limit" },
1994
- });
1995
- },
1996
- }, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX, yieldToTraceLane: true });
1997
- return uploadedSlices;
1998
- }
1999
-
2000
- /**
2001
- * Streaming packer shared by tree and change uploads: accumulate units
2002
- * in order, compress-check every partCheckBytes of uncompressed growth,
2003
- * flush parts targeting partCompressedBytes, and respect the hard rail
2004
- * (a lone unit over the rail is skipped via hooks.drop — never split).
2005
- * fit() mutates the batch it is given: the returned `measured` always
2006
- * corresponds to the (mutated) batch passed to hooks.upload, and popped
2007
- * units come back as `carry` to lead the next part. hooks.upload must
2008
- * persist sequence bookkeeping via uploadEnvelope.
2009
- *
2010
- * `yieldToTraceLane` (baseline/change/end packs): before each part
2011
- * upload, wait while trace work is pending or running — a trace part
2012
- * then never queues behind more than the one baseline part already in
2013
- * flight. Trace packs must NOT set it (they ARE the priority lane).
2014
- */
2015
- private async packAndUpload<U>(
2016
- state: SessionState,
2017
- snapshotType: SnapshotType,
2018
- units: Iterable<U> | AsyncIterable<U>,
2019
- hooks: {
2020
- sizeOf: (unit: U) => number;
2021
- measure: (batch: U[]) => Promise<{ files: CollectorFile[]; payload: Buffer; compressed: Buffer }>;
2022
- upload: (measured: { files: CollectorFile[]; payload: Buffer; compressed: Buffer }, batch: U[]) => Promise<boolean>;
2023
- drop?: (unit: U) => void;
2024
- },
2025
- opts: { checkBytes?: number; flushPlainBytes?: number; yieldToTraceLane?: boolean } = {},
2026
- ): Promise<void> {
2027
- const checkBytes = Math.max(1024, opts.checkBytes ?? this.partCheckBytes);
2028
- const flushPlainBytes = Math.max(checkBytes, opts.flushPlainBytes ?? PART_BATCH_UNCOMPRESSED_MAX);
2029
- const hard = MAX_PART_COMPRESSED_HARD;
2030
- const fit = async (batch: U[]) => {
2031
- let measured = await hooks.measure(batch);
2032
- const carry: U[] = [];
2033
- // Multi-file parts split at the compressed target. A lone unit
2034
- // (e.g. one slice entry) may use the body headroom up to the hard
2035
- // rail; beyond that it is dropped loudly rather than split
2036
- // mid-content.
2037
- while (measured.compressed.length > this.partCompressedBytes && batch.length > 1) {
2038
- carry.unshift(batch.pop()!);
2039
- measured = await hooks.measure(batch);
2040
- }
2041
- if (measured.compressed.length > hard && batch.length === 1) {
2042
- const lone = batch.pop()!;
2043
- hooks.drop?.(lone);
2044
- return { measured: batch.length > 0 ? await hooks.measure(batch) : null, carry };
2045
- }
2046
- return { measured, carry };
2047
- };
2048
-
2049
- let batch: U[] = [];
2050
- let batchBytes = 0;
2051
- let nextCheck = checkBytes;
2052
- const resetBatch = (unitsToKeep: U[]) => {
2053
- batch = unitsToKeep;
2054
- batchBytes = batch.reduce((total, unit) => total + hooks.sizeOf(unit), 0);
2055
- nextCheck = batchBytes + checkBytes;
2056
- };
2057
- const shouldFlush = (measured: { compressed: Buffer }) =>
2058
- measured.compressed.length >= PART_READY_COMPRESSED_BYTES || batchBytes >= flushPlainBytes;
2059
-
2060
- for await (const unit of units as AsyncIterable<U>) {
2061
- if (state.budgetExhausted) return;
2062
- batch.push(unit);
2063
- batchBytes += hooks.sizeOf(unit);
2064
- if (batchBytes < nextCheck) continue;
2065
- const { measured, carry } = await fit(batch);
2066
- if (measured && shouldFlush(measured)) {
2067
- if (opts.yieldToTraceLane) await this.yieldToTrace(state);
2068
- if (!(await hooks.upload(measured, batch))) return;
2069
- resetBatch(carry);
2070
- } else if (measured) {
2071
- // Under the flush thresholds — keep accumulating (carry is empty
2072
- // here: popping only happens above the flush levels).
2073
- batch = [...batch, ...carry];
2074
- nextCheck = batchBytes + checkBytes;
2075
- } else {
2076
- resetBatch(carry);
2077
- }
2078
- }
2079
- while (batch.length > 0 && !state.budgetExhausted) {
2080
- const { measured, carry } = await fit(batch);
2081
- if (measured) {
2082
- if (opts.yieldToTraceLane) await this.yieldToTrace(state);
2083
- if (!(await hooks.upload(measured, batch))) return;
2084
- }
2085
- if (carry.length === 0) break;
2086
- batch = carry;
2087
- batchBytes = batch.reduce((total, unit) => total + hooks.sizeOf(unit), 0);
2088
- }
2089
- }
2090
-
2091
- /** Chunked trace upload: event subsets per part, ordered by sequence. */
2092
- private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {
2093
- if (state.budgetExhausted || traceEvents.length === 0) {
2094
- // No lifecycle events this round — transcript artifacts may still
2095
- // be due (new or changed session files).
2096
- await this.uploadSessionTranscripts(state);
2097
- return;
2098
- }
2099
- const remaining = this.sessionBudgetBytes - state.sentBytes;
2100
- if (remaining <= 1024) {
2101
- this.exhaustSessionBudget(state);
2102
- return;
2103
- }
2104
- // Budget wall: truncate (loudly, trace_truncated=true) only here.
2105
- const selected = selectTraceEvents(traceEvents, remaining - 1024);
2106
- const meta = {
2107
- truncated: selected.truncated,
2108
- dropped: selected.droppedCount,
2109
- };
2110
- if (selected.truncated) {
2111
- this.log("warn", "OmniRush trace hit the session budget wall; older events dropped", {
2112
- sessionId: state.id,
2113
- droppedEventCount: selected.droppedCount,
2114
- });
2115
- }
2116
- let partIndex = 0;
2117
- const bySequence = () => this.sequenceState(state);
2118
- await this.packAndUpload(state, "trace", selected.events, {
2119
- sizeOf: (event) => Buffer.byteLength(JSON.stringify(event)) + 32,
2120
- measure: async (batch) => {
2121
- const content = JSON.stringify({
2122
- schema_version: 1,
2123
- schema: AGENT_TRACE_SCHEMA,
2124
- session_id: state.id,
2125
- session_segment: state.segment,
2126
- session_resumed: state.resumed,
2127
- trace_truncated: meta.truncated,
2128
- dropped_event_count: meta.dropped,
2129
- trace_part: { index: partIndex },
2130
- events: batch,
2131
- });
2132
- const files = [{ path: `${AGENT_DIR}/trace.json`, content }];
2133
- const payload = buildEnvelopePayload(bySequence(), "trace", files);
2134
- return { files, payload, compressed: await compressZstd(payload) };
2135
- },
2136
- upload: async (measured) => {
2137
- const ok = await this.uploadEnvelope(state, "trace", measured.files, measured);
2138
- if (ok) partIndex += 1;
2139
- return ok;
2140
- },
2141
- drop: (event) => {
2142
- this.log("warn", "OmniRush trace event exceeds the upload limits; skipped", {
2143
- sessionId: state.id,
2144
- type: event.type,
2145
- bytes: Buffer.byteLength(JSON.stringify(event)),
2146
- });
2147
- },
2148
- }, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
2149
- // Full-conversation transcript artifacts (trace format spec) ride
2150
- // the same trace uploads — lifecycle events stay in trace.json, the
2151
- // transcripts land as __agent__/traces/<session>.jsonl.
2152
- await this.uploadSessionTranscripts(state);
2153
- }
2154
-
2155
- /**
2156
- * The pi agent writes the FULL session transcript as JSONL under
2157
- * <agentDir>/sessions/<cwd-slug>/<ts>_<session-id>.jsonl — one file
2158
- * per run, SUBAGENTS as their own files in the same dir. Every file
2159
- * (new or changed since the last capture) becomes its own trace
2160
- * artifact (header + full conversation, redacted) uploaded as
2161
- * __agent__/traces/<session>.jsonl and linked to this session via
2162
- * parent_session_id. Detection state rides the persistent ledger.
2163
- */
2164
- private async uploadSessionTranscripts(state: SessionState): Promise<void> {
2165
- if (!this.agentDir || !state.id) return;
2166
- const { readdir, stat } = await import("node:fs/promises");
2167
- const sessionsRoot = join(this.agentDir, "sessions");
2168
- let sessionDir: string | null = null;
2169
- try {
2170
- for (const entry of await readdir(sessionsRoot, { withFileTypes: true })) {
2171
- if (!entry.isDirectory()) continue;
2172
- const dir = join(sessionsRoot, entry.name);
2173
- const files = await readdir(dir).catch(() => [] as string[]);
2174
- if (files.some((f) => f.endsWith(`_${state.id}.jsonl`))) {
2175
- sessionDir = dir;
2176
- break;
2177
- }
2178
- }
2179
- } catch {
2180
- return; // no sessions dir — nothing to capture
2181
- }
2182
- if (!sessionDir) return;
2183
-
2184
- const artifacts: Array<{ path: string; content: string; fileKey: string; size: number; mtimeMs: number }> = [];
2185
- let files: string[] = [];
2186
- try {
2187
- files = (await readdir(sessionDir)).filter((f) => f.endsWith(".jsonl")).sort();
2188
- } catch {
2189
- return;
2190
- }
2191
- for (const file of files) {
2192
- const filePath = join(sessionDir, file);
2193
- let size = 0;
2194
- let mtimeMs = 0;
2195
- try {
2196
- const stats = await stat(filePath);
2197
- size = stats.size;
2198
- mtimeMs = stats.mtimeMs;
2199
- } catch {
2200
- continue;
2201
- }
2202
- const known = state.transcripts[filePath];
2203
- if (known && known.size === size && known.mtimeMs === mtimeMs) continue;
2204
- let text: string;
2205
- try {
2206
- const { readFile } = await import("node:fs/promises");
2207
- text = await readFile(filePath, "utf8");
2208
- } catch {
2209
- continue;
2210
- }
2211
- const parsed = parsePiSession(text);
2212
- const artifactSessionId = parsed.sessionId ?? file.replace(/^\d{4}-\d{2}-\d{2}T[\d-]+Z_/, "").replace(/\.jsonl$/, "");
2213
- if (!artifactSessionId || parsed.messageRecords.length === 0) {
2214
- // Still mark empty/HEAD-only files seen so they are not retried.
2215
- state.transcripts[filePath] = { size, mtimeMs };
2216
- continue;
2217
- }
2218
- if (this.linkedParentSession && artifactSessionId !== state.id) {
2219
- // Linked child: the parent's collector already ships the other
2220
- // session files in this directory (linked to the parent) — a
2221
- // second copy from the child adds nothing.
2222
- continue;
2223
- }
2224
- const identity = await this.resolveIdentity();
2225
- const artifact = buildTraceArtifact({
2226
- sessionId: artifactSessionId,
2227
- parentSessionId: this.linkedParentSession ?? state.id,
2228
- userId: identity?.userId ?? null,
2229
- clientId: this.clientId ?? null,
2230
- agentId: "omnirush-cli",
2231
- parsed,
2232
- redact: (value) => redactCollectorText(value).text,
2233
- });
2234
- artifacts.push({
2235
- path: `${AGENT_DIR}/traces/${artifactSessionId}.jsonl`,
2236
- content: JSON.stringify(artifact),
2237
- fileKey: filePath,
2238
- size,
2239
- mtimeMs,
2240
- });
2241
- }
2242
- if (artifacts.length === 0) {
2243
- await this.persistSession(state).catch(() => undefined);
2244
- return;
2245
- }
2246
- const bySequence = () => this.sequenceState(state);
2247
- await this.packAndUpload(state, "trace", artifacts, {
2248
- sizeOf: (artifact) => Buffer.byteLength(artifact.content) + artifact.path.length + 64,
2249
- measure: async (batch) => {
2250
- const payload = buildEnvelopePayload(bySequence(), "trace", batch);
2251
- return { files: batch, payload, compressed: await compressZstd(payload) };
2252
- },
2253
- upload: async (measured) => {
2254
- // Mark a transcript seen only after its part is accepted — an
2255
- // ack-before-success let a single 503 permanently drop the
2256
- // transcript from every later flush (the ledger said captured).
2257
- const ok = await this.uploadEnvelope(state, "trace", measured.files, measured);
2258
- if (ok) {
2259
- for (const artifact of measured.files) {
2260
- state.transcripts[artifact.fileKey] = {
2261
- size: artifact.size,
2262
- mtimeMs: artifact.mtimeMs,
2263
- };
2264
- }
2265
- }
2266
- return ok;
2267
- },
2268
- drop: (artifact) => {
2269
- this.log("warn", "OmniRush transcript artifact exceeds the upload limits; skipped", {
2270
- sessionId: state.id,
2271
- path: artifact.path,
2272
- bytes: Buffer.byteLength(artifact.content),
2273
- });
2274
- state.transcripts[artifact.fileKey] = {
2275
- size: artifact.size,
2276
- mtimeMs: artifact.mtimeMs,
2277
- };
2278
- },
2279
- }, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
2280
- await this.persistSession(state).catch(() => undefined);
2281
- }
2282
-
2283
- /** Best-effort /device/me identity, fetched once per collector. */
2284
- private async resolveIdentity(): Promise<{ userId: string | null } | null> {
2285
- if (this.identityTried) return this.identity;
2286
- this.identityTried = true;
2287
- if (!this.identityProvider) return null;
2288
- try {
2289
- this.identity = await this.identityProvider();
2290
- } catch {
2291
- this.identity = null;
2292
- }
2293
- return this.identity;
2294
- }
2295
-
2296
- /**
2297
- * Send one envelope (one part). `precompressed` skips the rebuild +
2298
- * recompress when the caller already measured this exact batch.
2299
- */
2300
- private async uploadEnvelope(
2301
- state: SessionState,
2302
- snapshotType: SnapshotType,
2303
- files: CollectorFile[],
2304
- precompressed?: { payload: Buffer; compressed: Buffer },
2305
- ): Promise<boolean> {
2306
- if (!this.collectUrl && !this.uploader) return false;
2307
- if (state.budgetExhausted) return false;
2308
- if (files.length === 0) {
2309
- // A zero-file part carries no information and would loop forever
2310
- // if a caller ever fed a packer an empty unit stream.
2311
- this.log("warn", "OmniRush refused to upload an empty envelope", {
2312
- sessionId: state.id,
2313
- snapshotType,
2314
- });
2315
- return false;
2316
- }
2317
- const payload = precompressed?.payload ?? buildEnvelopePayload(state, snapshotType, files);
2318
- if (state.sentBytes + payload.length > this.sessionBudgetBytes) {
2319
- this.exhaustSessionBudget(state);
2320
- return false;
2321
- }
2322
- const compressed = precompressed?.compressed ?? await compressZstd(payload);
2323
- if (compressed.length > MAX_PART_COMPRESSED_HARD) {
2324
- this.log("warn", "OmniRush part exceeded the backend body rail; not sent", {
2325
- sessionId: state.id,
2326
- snapshotType,
2327
- compressedBytes: compressed.length,
2328
- railBytes: MAX_PART_COMPRESSED_HARD,
2329
- });
2330
- return false;
2331
- }
2332
- // Two lanes (session tail + trace tail) funnel every envelope send
2333
- // through this gate. Trace sends acquire with priority, so after the
2334
- // part currently in flight finishes, the trace part goes next; the
2335
- // gate is held per ATTEMPT, so a retrying baseline part cannot pin
2336
- // the transport either.
2337
- const priority = snapshotType === "trace";
2338
- const gate = state.gate ?? this.defaultGate;
2339
- const sendOnce = async () => {
2340
- await gate.acquire(priority);
2341
- try {
2342
- return await (this.uploader
2343
- ? this.uploader(state.id, compressed)
2344
- : this.fetcher(this.collectUrl!, {
2345
- method: "POST",
2346
- headers: {
2347
- Authorization: `Bearer ${this.token}`,
2348
- "Content-Type": "application/zstd",
2349
- "X-Omnirush-Session-ID": state.id,
2350
- },
2351
- body: compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer,
2352
- signal: AbortSignal.timeout(120_000),
2353
- }));
2354
- } finally {
2355
- gate.release();
2356
- }
2357
- };
2358
- // Shared retry layer: 429/5xx/network ride out backend deploys with
2359
- // exponential backoff + jitter. 401 stays a protocol answer handled
2360
- // inline (single-flight refresh, retry exactly once).
2361
- const outcome = await withRetries(
2362
- async () => {
2363
- let response = await sendOnce();
2364
- if (response.status === 401 && this.refresh) {
2365
- // Broker pattern: single-flight refresh, retry exactly once.
2366
- const rotated = await this.refresh(this.token).catch(() => null);
2367
- if (rotated) {
2368
- this.token = rotated;
2369
- response = await sendOnce();
2370
- }
2371
- }
2372
- return {
2373
- response,
2374
- retryAfterSec: Number(response?.headers?.get?.("retry-after")) || undefined,
2375
- };
2376
- },
2377
- {
2378
- attempts: retryAttempts(),
2379
- isRetryable: ({ response }) => isRetryableStatus(response?.status),
2380
- onRetry: ({ attempt, attempts, delayMs }) => {
2381
- this.log("warn", "OmniRush upload unavailable — retrying", {
2382
- sessionId: state.id,
2383
- snapshotType,
2384
- attempt: `${attempt + 1}/${attempts}`,
2385
- delayMs,
2386
- });
2387
- },
2388
- },
2389
- );
2390
- const response = outcome.result?.response;
2391
- if (response?.ok) {
2392
- // Drain/cancel the body so keep-alive sockets return to the pool —
2393
- // an unconsumed body keeps a ref'd socket and hangs process exit.
2394
- await response.body?.cancel().catch(() => undefined);
2395
- }
2396
- if (!response) {
2397
- // Every attempt failed at the network layer.
2398
- throw new Error(
2399
- `collector upload failed: network error after ${outcome.attempts} attempt${outcome.attempts === 1 ? "" : "s"}` +
2400
- ` — ${outcome.error?.message ?? "unknown"} [ref ${outcome.ref}]` +
2401
- ` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
2402
- );
2403
- }
2404
- if (!response.ok) {
2405
- // Diagnosable from a tester's screenshot alone: status, context,
2406
- // and (bounded) the server's error body. A 401 here means the
2407
- // refresh-retry above also failed — the caller should re-login.
2408
- const snippet = await WorkspaceCollector.errorBodySnippet(response);
2409
- throw new Error(
2410
- `collector upload failed: HTTP ${response.status}` +
2411
- `${snippet ? ` — ${snippet}` : ""}` +
2412
- ` [ref ${outcome.ref}]` +
2413
- ` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
2414
- );
2415
- }
2416
- state.sentBytes += payload.length;
2417
- state.sequence += 1;
2418
- await this.persistSession(state).catch((error: unknown) => {
2419
- this.log("warn", "OmniRush session ledger update failed", {
2420
- sessionId: state.id,
2421
- error: error instanceof Error ? error.message : "unknown",
2422
- });
2423
- });
2424
- this.log("info", "OmniRush collection artifact uploaded", {
2425
- sessionId: state.id,
2426
- snapshotType,
2427
- fileCount: files.length,
2428
- compressedBytes: compressed.length,
2429
- sequence: state.sequence,
2430
- });
2431
- return true;
2432
- }
2433
- }