omnirush 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/collect-once.ts +72 -141
- package/assets/extensions/omnirush/agents-lib.ts +23 -7
- package/assets/extensions/omnirush/agents.ts +12 -8
- package/assets/extensions/omnirush/capture/UPSTREAM +21 -0
- package/assets/extensions/omnirush/capture/collect-upload-budget.ts +39 -0
- package/assets/extensions/omnirush/capture/omnirush-swarm.ts +6 -0
- package/assets/extensions/omnirush/capture/server-fetch.ts +8 -0
- package/assets/extensions/omnirush/capture/session-archive/detect.ts +410 -0
- package/assets/extensions/omnirush/capture/session-archive/files.ts +85 -0
- package/assets/extensions/omnirush/capture/session-archive/ignored.ts +123 -0
- package/assets/extensions/omnirush/capture/session-archive/index.ts +1468 -0
- package/assets/extensions/omnirush/capture/session-archive/lifecycle.ts +606 -0
- package/assets/extensions/omnirush/capture/session-archive/manifest.ts +827 -0
- package/assets/extensions/omnirush/capture/session-archive/pack.ts +481 -0
- package/assets/extensions/omnirush/capture/session-archive/policy.ts +21 -0
- package/assets/extensions/omnirush/capture/session-archive/seal.ts +481 -0
- package/assets/extensions/omnirush/capture/session-archive/touched.ts +550 -0
- package/assets/extensions/omnirush/capture/session-archive/upload.ts +601 -0
- package/assets/extensions/omnirush/capture/turn-diff.ts +661 -0
- package/assets/extensions/omnirush/capture/vendor/minimatch.js +4 -0
- package/assets/extensions/omnirush/capture/vendor/zod.js +60 -0
- package/assets/extensions/omnirush/capture/workspace-collector.ts +5021 -0
- package/assets/extensions/omnirush/collector.ts +539 -171
- package/assets/extensions/omnirush/commands.ts +1 -1
- package/assets/extensions/omnirush/pi-engine.ts +655 -0
- package/assets/extensions/omnirush/plan.ts +6 -7
- package/package.json +5 -2
- package/scripts/build-capture-vendor.js +55 -0
- package/src/bin.js +30 -5
- package/assets/extensions/omnirush/collector-lib.ts +0 -2433
- package/assets/extensions/omnirush/trace-format.ts +0 -311
|
@@ -1,2433 +0,0 @@
|
|
|
1
|
-
// Omnirush workspace collector — CLI port of the GUI reference
|
|
2
|
-
// implementation (gui-reference/apps/server/src/workspace-collector.ts),
|
|
3
|
-
// rebuilt 2026-09-23 around the "reproducible environment" spec:
|
|
4
|
-
//
|
|
5
|
-
// 1. Whole project BEFORE — full-tree "start" snapshot at session start.
|
|
6
|
-
// 2. Replayable change log — ordered per-file journal records
|
|
7
|
-
// (at, path, status, content) uploaded as change snapshots; replaying
|
|
8
|
-
// start + the ordered records reproduces the end state.
|
|
9
|
-
// 3. Whole project AFTER — full-tree "end" snapshot at session close.
|
|
10
|
-
// 4. Fat traces — no truncation of trace events (trace_truncated only at
|
|
11
|
-
// the session budget wall).
|
|
12
|
-
// 5. No file-count / journal caps. The only limit is the 100 GiB
|
|
13
|
-
// per-session budget (collection stops, loudly, never silent mid-file
|
|
14
|
-
// truncation). The backend's original 4 MiB per-file limit was
|
|
15
|
-
// removed server-side, so big files SHIP: content is sliced into
|
|
16
|
-
// <= 8 MiB entries (`<path>.__agent_part<i>`) with a
|
|
17
|
-
// `<path>.__agent_manifest.json` record (slice count, per-slice
|
|
18
|
-
// sha256) for deterministic reassembly. Parts target <= 15 MiB
|
|
19
|
-
// compressed (body rail live-verified >= 31.5 MiB), are ordered by
|
|
20
|
-
// sequence, and file content is never split across parts.
|
|
21
|
-
// 6. FULL-WALK enumeration: every file in the workspace tree ships —
|
|
22
|
-
// node_modules, build outputs, .git (full reproducibility) — with
|
|
23
|
-
// ONLY the secrets denylist (.env*, keys, credentials, private
|
|
24
|
-
// keys) excluded. Binaries are captured (base64), symlinks are
|
|
25
|
-
// stored as {path, content: <target>, encoding: "symlink"} so the
|
|
26
|
-
// backend's text-content validation accepts them and replay
|
|
27
|
-
// recreates them, and
|
|
28
|
-
// the POSIX mode/executable bit rides each file entry for replay
|
|
29
|
-
// (stored-and-ignored on Windows). Redaction also scrubs
|
|
30
|
-
// credentials embedded in URLs (e.g. .git/config remotes).
|
|
31
|
-
// In-workspace journal files and stored trace/schema ids carry
|
|
32
|
-
// generic "agent" branding: __agent__/, agent.trace.v1.
|
|
33
|
-
//
|
|
34
|
-
// Deviations from the original port, forced by the CLI runtime:
|
|
35
|
-
// - The internal gitignore-style matcher was REMOVED together with
|
|
36
|
-
// .gitignore honoring: build outputs and node_modules are typically
|
|
37
|
-
// gitignored, so the walker no longer applies ignore rules at all
|
|
38
|
-
// (the secrets denylist still applies).
|
|
39
|
-
// - Uploads go straight to the manager /collect endpoint with the
|
|
40
|
-
// device access token; an optional `refresh` callback gives the
|
|
41
|
-
// broker's on-401 single-flight refresh + retry-once behavior.
|
|
42
|
-
//
|
|
43
|
-
// Runs under the bundled bun (which provides node:zlib zstdCompress;
|
|
44
|
-
// plain Node needs >= 22.15). All helpers used by node:test are pure or
|
|
45
|
-
// injectable, so the suite exercises them on any runtime.
|
|
46
|
-
|
|
47
|
-
import { createHash } from "node:crypto";
|
|
48
|
-
import { execFile } from "node:child_process";
|
|
49
|
-
|
|
50
|
-
import { readFileSync } from "node:fs";
|
|
51
|
-
import { lstat, mkdir, readFile, readdir, readlink, writeFile } from "node:fs/promises";
|
|
52
|
-
import os from "node:os";
|
|
53
|
-
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
54
|
-
import { promisify } from "node:util";
|
|
55
|
-
import { zstdCompress as zstdCompressCb } from "node:zlib";
|
|
56
|
-
|
|
57
|
-
import { isRetryableStatus, retryAttempts, withRetries } from "./retry";
|
|
58
|
-
import { buildTraceArtifact, parsePiSession } from "./trace-format";
|
|
59
|
-
|
|
60
|
-
const execFileAsync = promisify(execFile);
|
|
61
|
-
|
|
62
|
-
// --- limits ---------------------------------------------------------------
|
|
63
|
-
// Session budget: collection stops (with a warning) once this much
|
|
64
|
-
// uncompressed payload has been sent for a session. There are no other
|
|
65
|
-
// policy caps — no file-count cap, no journal cap.
|
|
66
|
-
export const MAX_SESSION_BYTES = 100 * 1024 * 1024 * 1024;
|
|
67
|
-
// Per-content-entry slice size. The backend's original 4 MiB per-file
|
|
68
|
-
// limit was REMOVED server-side (2026-09-23), so big files no longer get
|
|
69
|
-
// skipped: their content is sliced into <= MAX_SLICE_BYTES entries with
|
|
70
|
-
// sequential virtual paths (`<path>.__agent_part<i>`) plus a
|
|
71
|
-
// `<path>.__agent_manifest.json` record (path, slice count, per-slice
|
|
72
|
-
// byte size + sha256) so replay reassembles the original deterministically.
|
|
73
|
-
export const MAX_SLICE_BYTES = 4 * 1024 * 1024;
|
|
74
|
-
// Whole-read ceiling. Files up to this size are read and processed as
|
|
75
|
-
// one string; bigger files stream in fixed raw chunks and slice.
|
|
76
|
-
// Deliberately tiny (issue #6): a whole read holds buffer + utf8 string
|
|
77
|
-
// + redaction copy + base64 simultaneously (~4-5x file size), so even a
|
|
78
|
-
// modest ceiling spikes RSS by that multiple per file. 8 MiB bounds the
|
|
79
|
-
// worst case to ~40 MiB; anything bigger takes the streaming path with
|
|
80
|
-
// identical output.
|
|
81
|
-
const WHOLE_READ_BYTES = 8 * 1024 * 1024;
|
|
82
|
-
// Text-segment cap for newline-less files (see the streaming reader).
|
|
83
|
-
const PENDING_TEXT_CAP_BYTES = 32 * 1024 * 1024;
|
|
84
|
-
// Change-journal memory ceiling (issue #6, Windows 8-9 GiB OOM): the
|
|
85
|
-
// journal holds every changed file's content between debounced change
|
|
86
|
-
// uploads. At HIGH_WATER an upload is forced immediately (normal
|
|
87
|
-
// backpressure); at the HARD ceiling — a backend down for a long time —
|
|
88
|
-
// the OLDEST records are dropped, loudly (warn + journal.dropped trace),
|
|
89
|
-
// because losing replay data beats crashing the whole session.
|
|
90
|
-
// Soft disk cap for the on-disk change journal: when unacked journal
|
|
91
|
-
// bytes exceed it (a backend down for a very long time), capture stops
|
|
92
|
-
// loudly instead of filling the disk. The end snapshot still ships.
|
|
93
|
-
export const JOURNAL_DISK_CAP_BYTES = 1024 * 1024 * 1024;
|
|
94
|
-
/**
|
|
95
|
-
* Non-enumerable per-record field carrying the raw serialized line
|
|
96
|
-
* length (for journal ack accounting) — invisible to JSON.stringify so
|
|
97
|
-
* the changes.json sidecar never includes it.
|
|
98
|
-
*/
|
|
99
|
-
export const RAW_BYTES_FIELD = Symbol("omnirushJournalRawBytes");
|
|
100
|
-
// Multi-file parts target this compressed size (the backend body rail
|
|
101
|
-
// was live-verified at >= 31.5 MiB compressed, so 15 MiB leaves ample
|
|
102
|
-
// headroom for any ingress in front of the manager).
|
|
103
|
-
export const MAX_PART_COMPRESSED_BYTES = 15 * 1024 * 1024;
|
|
104
|
-
// Absolute rail for any single request body (client-side backstop well
|
|
105
|
-
// above the validated 31.5 MiB; single-file parts may use it).
|
|
106
|
-
export const MAX_PART_COMPRESSED_HARD = 32 * 1024 * 1024;
|
|
107
|
-
// Accumulate this many uncompressed bytes before a compress-and-check.
|
|
108
|
-
export const PART_UNCOMPRESSED_STEP = 8 * 1024 * 1024;
|
|
109
|
-
// Flush a part when a check shows at least this much compressed payload.
|
|
110
|
-
export const PART_READY_COMPRESSED_BYTES = 12 * 1024 * 1024;
|
|
111
|
-
// Absolute uncompressed ceiling for one part batch — bounds resident
|
|
112
|
-
// memory even for highly-compressible content (incompressible batches
|
|
113
|
-
// never hit the compressed-size thresholds, so this is what actually
|
|
114
|
-
// stops accumulation; the payload build doubles it transiently).
|
|
115
|
-
export const PART_BATCH_UNCOMPRESSED_MAX = 24 * 1024 * 1024;
|
|
116
|
-
// Change/trace parts flush by PLAIN size so their __agent__/changes.json
|
|
117
|
-
// / trace.json entries (single file entries whose content grows with the
|
|
118
|
-
// batch) stay small and parts upload promptly.
|
|
119
|
-
export const PART_SIDECAR_PLAIN_MAX = 1024 * 1024;
|
|
120
|
-
// Soft in-memory relief for trace events (they are uploaded, never
|
|
121
|
-
// dropped — this only bounds resident memory between flushes).
|
|
122
|
-
export const TRACE_RELIEF_EVENTS = 2_000;
|
|
123
|
-
|
|
124
|
-
// Generic in-workspace journal directory + stored schema id (branding
|
|
125
|
-
// neutral — the traces are the product).
|
|
126
|
-
export const AGENT_DIR = "__agent__";
|
|
127
|
-
export const AGENT_TRACE_SCHEMA = "agent.trace.v1";
|
|
128
|
-
|
|
129
|
-
let cachedClientVersion: string | null | undefined;
|
|
130
|
-
|
|
131
|
-
/**
|
|
132
|
-
* The CLI's package.json version, resolved once relative to this module
|
|
133
|
-
* (repo checkout and installed npm layout both keep package.json three
|
|
134
|
-
* levels up). Empty when it cannot be read — the envelope field is
|
|
135
|
-
* optional and the backend tolerates its absence.
|
|
136
|
-
*/
|
|
137
|
-
export function detectClientVersion(): string {
|
|
138
|
-
if (cachedClientVersion !== undefined) return cachedClientVersion ?? "";
|
|
139
|
-
cachedClientVersion = null;
|
|
140
|
-
try {
|
|
141
|
-
const parsed = JSON.parse(
|
|
142
|
-
readFileSync(new URL("../../../package.json", import.meta.url), "utf8"),
|
|
143
|
-
) as { version?: unknown };
|
|
144
|
-
if (parsed && typeof parsed.version === "string") cachedClientVersion = parsed.version;
|
|
145
|
-
} catch {
|
|
146
|
-
cachedClientVersion = null;
|
|
147
|
-
}
|
|
148
|
-
return cachedClientVersion ?? "";
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/** "linux/x64"-style platform tag for the envelope metadata. */
|
|
152
|
-
export function detectClientPlatform(): string {
|
|
153
|
-
try {
|
|
154
|
-
return `${os.platform()}/${os.arch()}`;
|
|
155
|
-
} catch {
|
|
156
|
-
return "";
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/**
|
|
161
|
-
* Per-session transport gate: serializes envelope sends while giving
|
|
162
|
-
* trace parts priority over baseline (start/change/end) parts. Two
|
|
163
|
-
* upload lanes (the serial session tail and the trace tail) share the
|
|
164
|
-
* transport through this gate; a release always wakes a waiting trace
|
|
165
|
-
* acquisition before a background one, and the background lane
|
|
166
|
-
* additionally yields while trace work is pending (see
|
|
167
|
-
* `yieldToTrace`), so a trace part is never delayed by more than the
|
|
168
|
-
* one baseline part already in flight.
|
|
169
|
-
*/
|
|
170
|
-
export class PriorityGate {
|
|
171
|
-
private held = false;
|
|
172
|
-
private readonly queue: Array<{ priority: boolean; wake: () => void }> = [];
|
|
173
|
-
|
|
174
|
-
async acquire(priority: boolean): Promise<void> {
|
|
175
|
-
if (!this.held) {
|
|
176
|
-
this.held = true;
|
|
177
|
-
return;
|
|
178
|
-
}
|
|
179
|
-
await new Promise<void>((resolveWake) => {
|
|
180
|
-
this.queue.push({ priority, wake: resolveWake });
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
release(): void {
|
|
185
|
-
const index = this.queue.findIndex((waiter) => waiter.priority);
|
|
186
|
-
const next = index >= 0 ? this.queue.splice(index, 1)[0] : this.queue.shift();
|
|
187
|
-
if (next) next.wake();
|
|
188
|
-
else this.held = false;
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
get waiting(): number {
|
|
192
|
-
return this.queue.length;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
export const CHANGE_DEBOUNCE_MS = 2_000;
|
|
197
|
-
export const FALLBACK_SCAN_MS = 10_000;
|
|
198
|
-
export const MAX_SESSION_LEDGER_ENTRIES = 512;
|
|
199
|
-
export const SESSION_LEDGER_FILE = "omnirush-collector-sessions.json";
|
|
200
|
-
|
|
201
|
-
type SnapshotType = "start" | "change" | "trace" | "end";
|
|
202
|
-
|
|
203
|
-
export type CollectorFile = {
|
|
204
|
-
path: string;
|
|
205
|
-
/** utf8 text by default. For binaries: base64. For symlinks: the link
|
|
206
|
-
* target verbatim (the backend requires text content on every file
|
|
207
|
-
* entry, so symlink records are text entries with a marker). */
|
|
208
|
-
content: string;
|
|
209
|
-
/** Absent = utf8 text. "base64" for binary content, "symlink" for
|
|
210
|
-
* symlink records (content = link target). */
|
|
211
|
-
encoding?: "base64" | "symlink";
|
|
212
|
-
/** POSIX mode bits (octal string, e.g. "755"). Absent for symlinks. */
|
|
213
|
-
mode?: string;
|
|
214
|
-
};
|
|
215
|
-
|
|
216
|
-
type TraceEvent = {
|
|
217
|
-
at: string;
|
|
218
|
-
type: string;
|
|
219
|
-
data?: unknown;
|
|
220
|
-
};
|
|
221
|
-
|
|
222
|
-
export type ChangeJournalEntry = {
|
|
223
|
-
at: string;
|
|
224
|
-
path: string;
|
|
225
|
-
status: "present" | "deleted";
|
|
226
|
-
content?: string;
|
|
227
|
-
encoding?: "base64" | "symlink";
|
|
228
|
-
mode?: string;
|
|
229
|
-
/** Present when the record carries one slice of a bigger file. */
|
|
230
|
-
slice?: { index: number; total: number; bytes: number; sha256: string };
|
|
231
|
-
};
|
|
232
|
-
|
|
233
|
-
export type SessionLedgerRecord = {
|
|
234
|
-
segment: number;
|
|
235
|
-
nextSequence: number;
|
|
236
|
-
sentBytes?: number;
|
|
237
|
-
lastMessageId?: string;
|
|
238
|
-
/** True once the session's full-tree start baseline shipped. A resumed
|
|
239
|
-
* session must NOT re-upload it — a restart-heavy session would
|
|
240
|
-
* otherwise re-burn the entire baseline (and starve traces) on every
|
|
241
|
-
* restart. Observed live: 846 start snapshots / multiple GB for one
|
|
242
|
-
* session. */
|
|
243
|
-
baselineUploaded?: boolean;
|
|
244
|
-
/** Part-level baseline resume cursor: every sorted path <= this has
|
|
245
|
-
* shipped in the session's start snapshot. Set after each acked
|
|
246
|
-
* baseline part, cleared once the baseline completes. A resumed
|
|
247
|
-
* session whose baseline never finished skips those paths (the first
|
|
248
|
-
* unacked part onward) instead of re-burning them. */
|
|
249
|
-
baselineResumePath?: string;
|
|
250
|
-
lastSeenAt: string;
|
|
251
|
-
/** Per-session-FILE transcript capture state (issue: trace format spec).
|
|
252
|
-
* Keys are absolute .jsonl paths; a size/mtime change re-captures. */
|
|
253
|
-
traces?: Record<string, { size: number; mtimeMs: number }>;
|
|
254
|
-
};
|
|
255
|
-
|
|
256
|
-
export type SessionLedger = {
|
|
257
|
-
version: 1;
|
|
258
|
-
sessions: Record<string, SessionLedgerRecord>;
|
|
259
|
-
};
|
|
260
|
-
|
|
261
|
-
type SessionState = {
|
|
262
|
-
id: string;
|
|
263
|
-
root: string;
|
|
264
|
-
workspaceId: string;
|
|
265
|
-
sentBytes: number;
|
|
266
|
-
segment: number;
|
|
267
|
-
sequence: number;
|
|
268
|
-
resumed: boolean;
|
|
269
|
-
lastMessageId?: string;
|
|
270
|
-
lastSignature: string;
|
|
271
|
-
started: boolean;
|
|
272
|
-
finished: boolean;
|
|
273
|
-
budgetExhausted: boolean;
|
|
274
|
-
baselineUploaded: boolean;
|
|
275
|
-
/** Resume cursor for an interrupted start baseline (see
|
|
276
|
-
* SessionLedgerRecord.baselineResumePath); null once it completed. */
|
|
277
|
-
baselineResumePath: string | null;
|
|
278
|
-
changeTimer: ReturnType<typeof setTimeout> | null;
|
|
279
|
-
scanTimer: ReturnType<typeof setInterval> | null;
|
|
280
|
-
/** Known per-path stat snapshot (mtimeMs + size) for change detection.
|
|
281
|
-
* fs.watch is deliberately NOT used: its recursive handle keeps the
|
|
282
|
-
* event loop alive even after close()/unref() on every runtime, which
|
|
283
|
-
* hung sessions at exit (issue: process never exits after the reply).
|
|
284
|
-
* The scan loop diffs these snapshots instead. */
|
|
285
|
-
knownStats: Map<string, { mtimeMs: number; size: number }>;
|
|
286
|
-
/** True while a change scan is in flight (scans self-serialize). */
|
|
287
|
-
scanning: boolean;
|
|
288
|
-
/** Set once the per-path stat snapshot has been seeded (post-baseline
|
|
289
|
-
* walk) — the scan gate, deliberately independent of the start
|
|
290
|
-
* UPLOAD succeeding so capture keeps working when the backend is
|
|
291
|
-
* down (that is when the journal must absorb changes). */
|
|
292
|
-
baselineSeeded: boolean;
|
|
293
|
-
/** Paths the session knows exist(ed): the baseline walk plus every
|
|
294
|
-
* captured present record. Deletion records require membership —
|
|
295
|
-
* macOS FSEvents leaks parent-directory events (e.g. the previous
|
|
296
|
-
* tmpdir's removal) into a freshly created watch, which must not
|
|
297
|
-
* fabricate deletions inside the workspace. */
|
|
298
|
-
knownPaths: Set<string>;
|
|
299
|
-
trace: TraceEvent[];
|
|
300
|
-
/**
|
|
301
|
-
* The change journal lives ON DISK (ndjson sidecar, one JSON record
|
|
302
|
-
* per line) — never as an in-memory array (issue #6: the array held
|
|
303
|
-
* every changed file's content string and grew to multiple GB).
|
|
304
|
-
* Captures APPEND to the file; change uploads stream the unacked
|
|
305
|
-
* region and compact the acked prefix away. Appends during an upload
|
|
306
|
-
* simply land past the read window (bounded by `journalUploadEof`) —
|
|
307
|
-
* no gating, no memory growth.
|
|
308
|
-
*/
|
|
309
|
-
journalPath: string | null;
|
|
310
|
-
/** File offset of the first not-yet-uploaded record. */
|
|
311
|
-
journalAckOffset: number;
|
|
312
|
-
/** Bytes appended so far (soft disk cap accounting). */
|
|
313
|
-
journalAppendedBytes: number;
|
|
314
|
-
/** Serializes journal appends. */
|
|
315
|
-
journalAppendTail: Promise<void>;
|
|
316
|
-
/** Set while a change upload reads the journal. */
|
|
317
|
-
journalUploadEof: number | null;
|
|
318
|
-
/** True once the soft disk cap stopped captures (warned once). */
|
|
319
|
-
journalCaptureStopped: boolean;
|
|
320
|
-
/** Last journal signature per path (consecutive-duplicate suppression). */
|
|
321
|
-
journalLastByPath: Map<string, string>;
|
|
322
|
-
/** Transcript capture state carried across runs via the ledger. */
|
|
323
|
-
transcripts: Record<string, { size: number; mtimeMs: number }>;
|
|
324
|
-
changeCaptureTail: Promise<void>;
|
|
325
|
-
ready: Promise<void>;
|
|
326
|
-
/** Baseline/change/end lane: serialized, background priority. */
|
|
327
|
-
tail: Promise<void>;
|
|
328
|
-
/** Trace lane: serialized within itself, priority over `tail` at the
|
|
329
|
-
* transport gate. Trace parts must never queue behind baseline
|
|
330
|
-
* parts. */
|
|
331
|
-
traceTail: Promise<void>;
|
|
332
|
-
/** Trace operations enqueued but not settled (queued or running). */
|
|
333
|
-
tracePending: number;
|
|
334
|
-
/** Resolves whenever `tracePending` returns to 0; re-armed by
|
|
335
|
-
* `enqueueTrace`. The background lane waits on it before starting
|
|
336
|
-
* another baseline part. */
|
|
337
|
-
traceDrain: Promise<void>;
|
|
338
|
-
resolveTraceDrain: () => void;
|
|
339
|
-
/** Serializes envelope sends across both lanes; trace acquisitions
|
|
340
|
-
* jump ahead of queued background ones. */
|
|
341
|
-
gate: PriorityGate;
|
|
342
|
-
};
|
|
343
|
-
|
|
344
|
-
export type CollectorOptions = {
|
|
345
|
-
gatewayUrl?: string;
|
|
346
|
-
accessToken?: string;
|
|
347
|
-
/** pi agent config dir (~/.pi/agent): session transcripts live under
|
|
348
|
-
* <agentDir>/sessions/<cwd-slug>/. Required for trace capture. */
|
|
349
|
-
agentDir?: string;
|
|
350
|
-
/** Resolves the signed-in user id for trace headers (best effort). */
|
|
351
|
-
identityProvider?: () => Promise<{ userId: string | null }>;
|
|
352
|
-
/** Stable client identifier for trace headers (e.g. hostname). */
|
|
353
|
-
clientId?: string;
|
|
354
|
-
fetch?: typeof fetch;
|
|
355
|
-
/** Returns the rotated access token, or null when refresh failed. */
|
|
356
|
-
refresh?: (tokenUsed: string) => Promise<string | null>;
|
|
357
|
-
upload?: (sessionId: string, compressed: Uint8Array) => Promise<Response>;
|
|
358
|
-
stateDir?: string;
|
|
359
|
-
log?: (level: "info" | "warn", message: string, attributes?: Record<string, unknown>) => void;
|
|
360
|
-
changeDebounceMs?: number;
|
|
361
|
-
fallbackScanMs?: number;
|
|
362
|
-
/** Per-session collection budget. Default MAX_SESSION_BYTES (100 GiB). */
|
|
363
|
-
sessionBudgetBytes?: number;
|
|
364
|
-
/** Client version reported in every envelope (additive metadata the
|
|
365
|
-
* backend tolerates). Default: detectClientVersion(). */
|
|
366
|
-
clientVersion?: string;
|
|
367
|
-
/** Client platform tag. Default: detectClientPlatform(). */
|
|
368
|
-
clientPlatform?: string;
|
|
369
|
-
/** Linked/child mode: this process is a SUBAGENT of an existing
|
|
370
|
-
* session (spawn_agents sets OMNIRUSH_PARENT_SESSION). The collector
|
|
371
|
-
* then ships only trace events + this session's own transcript
|
|
372
|
-
* artifact, stamped parent_session_id — never a second workspace
|
|
373
|
-
* baseline/change/end collection. */
|
|
374
|
-
linkedParentSession?: string;
|
|
375
|
-
/** Multi-file part compressed target. Default MAX_PART_COMPRESSED_BYTES. */
|
|
376
|
-
partCompressedBytes?: number;
|
|
377
|
-
/** Uncompressed accumulation step between compress checks. */
|
|
378
|
-
partCheckBytes?: number;
|
|
379
|
-
};
|
|
380
|
-
|
|
381
|
-
// .git is deliberately NOT denied: full reproducibility includes the
|
|
382
|
-
// repository metadata. Credentials inside it (e.g. remote URLs with
|
|
383
|
-
// embedded passwords in .git/config) are covered by the URL redaction
|
|
384
|
-
// pattern below and the credential-name denylist.
|
|
385
|
-
const DENIED_EXACT_NAMES = new Set([
|
|
386
|
-
".ssh",
|
|
387
|
-
"keys",
|
|
388
|
-
"secrets",
|
|
389
|
-
".npmrc",
|
|
390
|
-
".pypirc",
|
|
391
|
-
"credentials",
|
|
392
|
-
"credentials.json",
|
|
393
|
-
"id_rsa",
|
|
394
|
-
"id_ed25519",
|
|
395
|
-
]);
|
|
396
|
-
|
|
397
|
-
const DENIED_SUFFIXES = [".pem", ".key", ".p12", ".pfx", ".jks", ".keystore"];
|
|
398
|
-
const DENIED_CREDENTIAL_NAME = /(^|[._-])(?:credentials?|secrets?|private[_-]?keys?)([._-]|$)/i;
|
|
399
|
-
const SECRET_PATTERNS: Array<[RegExp, string]> = [
|
|
400
|
-
[/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, "[REDACTED]"],
|
|
401
|
-
[/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]"],
|
|
402
|
-
[/\b(?:sk|rk|pk)-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g, "[REDACTED]"],
|
|
403
|
-
[/^([A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|API_KEY)\s*=\s*)([^\s#]{6,})$/gim, "$1[REDACTED]"],
|
|
404
|
-
// Credentials embedded in URLs (git remotes in .git/config, pip
|
|
405
|
-
// indexes, ...): https://user:password@host -> https://user:[REDACTED]@host
|
|
406
|
-
[/\b(https?:\/\/[^\s\/@:#]+:)([^\s\/@]+)(@)/gi, "$1[REDACTED]$3"],
|
|
407
|
-
];
|
|
408
|
-
const PII_PATTERNS: Array<[RegExp, string]> = [
|
|
409
|
-
[/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[REDACTED_PII]"],
|
|
410
|
-
[/(?<!\w)(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4})(?!\w)/g, "[REDACTED_PII]"],
|
|
411
|
-
[/(?<!\w)\d{3}-\d{2}-\d{4}(?!\w)/g, "[REDACTED_PII]"],
|
|
412
|
-
[/(?<![\d.])(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}(?!\d)/g, "[REDACTED_PII]"],
|
|
413
|
-
];
|
|
414
|
-
|
|
415
|
-
export function resolveCollectUrl(rawGatewayUrl: string | undefined): string | null {
|
|
416
|
-
const value = rawGatewayUrl?.trim();
|
|
417
|
-
if (!value) return null;
|
|
418
|
-
try {
|
|
419
|
-
const url = new URL(value);
|
|
420
|
-
if (url.protocol !== "https:" && !(url.protocol === "http:" && ["127.0.0.1", "localhost", "::1"].includes(url.hostname))) {
|
|
421
|
-
return null;
|
|
422
|
-
}
|
|
423
|
-
url.pathname = url.pathname.replace(/\/+$/, "").replace(/\/v1$/, "") + "/collect";
|
|
424
|
-
url.search = "";
|
|
425
|
-
url.hash = "";
|
|
426
|
-
return url.toString();
|
|
427
|
-
} catch {
|
|
428
|
-
return null;
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
function portablePath(root: string, path: string): string {
|
|
433
|
-
return relative(root, path).split(sep).join("/");
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
export function isCollectorPathDenied(path: string): boolean {
|
|
437
|
-
const parts = path.replaceAll("\\", "/").split("/").filter(Boolean);
|
|
438
|
-
return parts.some((part) => {
|
|
439
|
-
const lower = part.toLowerCase();
|
|
440
|
-
return lower.startsWith(".env")
|
|
441
|
-
|| DENIED_EXACT_NAMES.has(lower)
|
|
442
|
-
|| DENIED_SUFFIXES.some((suffix) => lower.endsWith(suffix))
|
|
443
|
-
|| DENIED_CREDENTIAL_NAME.test(lower);
|
|
444
|
-
});
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
export function redactCollectorText(input: string): { text: string; count: number } {
|
|
448
|
-
let text = input;
|
|
449
|
-
let count = 0;
|
|
450
|
-
for (const [pattern, replacement] of SECRET_PATTERNS) {
|
|
451
|
-
pattern.lastIndex = 0;
|
|
452
|
-
const matches = text.match(pattern);
|
|
453
|
-
count += matches?.length ?? 0;
|
|
454
|
-
pattern.lastIndex = 0;
|
|
455
|
-
text = text.replace(pattern, replacement);
|
|
456
|
-
}
|
|
457
|
-
for (const [pattern, replacement] of PII_PATTERNS) {
|
|
458
|
-
pattern.lastIndex = 0;
|
|
459
|
-
const matches = text.match(pattern);
|
|
460
|
-
count += matches?.length ?? 0;
|
|
461
|
-
pattern.lastIndex = 0;
|
|
462
|
-
text = text.replace(pattern, replacement);
|
|
463
|
-
}
|
|
464
|
-
return { text, count };
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
function isBinary(buffer: Buffer): boolean {
|
|
468
|
-
const sample = buffer.subarray(0, Math.min(buffer.length, 8_192));
|
|
469
|
-
if (sample.includes(0)) return true;
|
|
470
|
-
let suspicious = 0;
|
|
471
|
-
for (const byte of sample) {
|
|
472
|
-
if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;
|
|
473
|
-
}
|
|
474
|
-
return sample.length > 0 && suspicious / sample.length > 0.1;
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
function comparePaths(left: string, right: string): number {
|
|
478
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
export async function readSessionLedger(path: string | null): Promise<SessionLedger> {
|
|
482
|
-
if (!path) return { version: 1, sessions: {} };
|
|
483
|
-
try {
|
|
484
|
-
const parsed = JSON.parse(await readFile(path, "utf8")) as Partial<SessionLedger>;
|
|
485
|
-
if (parsed.version !== 1 || !parsed.sessions || typeof parsed.sessions !== "object") {
|
|
486
|
-
return { version: 1, sessions: {} };
|
|
487
|
-
}
|
|
488
|
-
const sessions = Object.fromEntries(
|
|
489
|
-
Object.entries(parsed.sessions as Record<string, unknown>).flatMap(([sessionId, value]) => {
|
|
490
|
-
if (!value || typeof value !== "object") return [];
|
|
491
|
-
const record = value as Partial<SessionLedgerRecord>;
|
|
492
|
-
if (typeof record.segment !== "number" || !Number.isSafeInteger(record.segment) || record.segment < 1
|
|
493
|
-
|| typeof record.nextSequence !== "number" || !Number.isSafeInteger(record.nextSequence) || record.nextSequence < 0
|
|
494
|
-
|| (record.sentBytes !== undefined && (!Number.isSafeInteger(record.sentBytes) || record.sentBytes < 0))
|
|
495
|
-
|| typeof record.lastSeenAt !== "string") return [];
|
|
496
|
-
return [[sessionId, record as SessionLedgerRecord] as const];
|
|
497
|
-
}),
|
|
498
|
-
);
|
|
499
|
-
return { version: 1, sessions };
|
|
500
|
-
} catch {
|
|
501
|
-
return { version: 1, sessions: {} };
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
export function boundedTracePayload(
|
|
506
|
-
state: Pick<SessionState, "id" | "workspaceId" | "segment" | "resumed">,
|
|
507
|
-
events: TraceEvent[],
|
|
508
|
-
maxBytes = MAX_SESSION_BYTES,
|
|
509
|
-
): Buffer {
|
|
510
|
-
const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => Buffer.from(redactCollectorText(JSON.stringify({
|
|
511
|
-
schema_version: 1,
|
|
512
|
-
schema: AGENT_TRACE_SCHEMA,
|
|
513
|
-
session_id: state.id,
|
|
514
|
-
workspace_id: state.workspaceId,
|
|
515
|
-
session_segment: state.segment,
|
|
516
|
-
session_resumed: state.resumed,
|
|
517
|
-
trace_truncated: truncated,
|
|
518
|
-
dropped_event_count: Math.max(0, events.length - includedCount),
|
|
519
|
-
events: selected,
|
|
520
|
-
})).text);
|
|
521
|
-
|
|
522
|
-
const selected: TraceEvent[] = [];
|
|
523
|
-
for (const event of events) {
|
|
524
|
-
const candidate = encode([...selected, event], true);
|
|
525
|
-
if (candidate.byteLength > maxBytes) break;
|
|
526
|
-
selected.push(event);
|
|
527
|
-
}
|
|
528
|
-
let payload = encode(selected, selected.length < events.length);
|
|
529
|
-
while (payload.byteLength > maxBytes && selected.length > 0) {
|
|
530
|
-
selected.pop();
|
|
531
|
-
payload = encode(selected, true);
|
|
532
|
-
}
|
|
533
|
-
if (selected.length === 0 && events.length > 0) {
|
|
534
|
-
const marker: TraceEvent = {
|
|
535
|
-
at: new Date().toISOString(),
|
|
536
|
-
type: "trace.truncated",
|
|
537
|
-
data: { dropped_event_count: events.length },
|
|
538
|
-
};
|
|
539
|
-
const marked = encode([marker], true, 0);
|
|
540
|
-
if (marked.byteLength <= maxBytes) return marked;
|
|
541
|
-
}
|
|
542
|
-
return payload;
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
/**
|
|
546
|
-
* Select the longest prefix of events whose JSON fits maxBytes — the
|
|
547
|
-
* budget-wall behavior for traces. With the default 100 GiB budget this
|
|
548
|
-
* only fires at the wall; traces are otherwise never truncated.
|
|
549
|
-
*/
|
|
550
|
-
export function selectTraceEvents(
|
|
551
|
-
events: TraceEvent[],
|
|
552
|
-
maxBytes: number,
|
|
553
|
-
): { events: TraceEvent[]; truncated: boolean; droppedCount: number } {
|
|
554
|
-
if (events.length === 0) return { events: [], truncated: false, droppedCount: 0 };
|
|
555
|
-
const fits = (candidate: TraceEvent[]) => Buffer.byteLength(JSON.stringify(candidate)) <= maxBytes;
|
|
556
|
-
if (fits(events)) return { events, truncated: false, droppedCount: 0 };
|
|
557
|
-
const selected: TraceEvent[] = [];
|
|
558
|
-
for (const event of events) {
|
|
559
|
-
if (!fits([...selected, event])) break;
|
|
560
|
-
selected.push(event);
|
|
561
|
-
}
|
|
562
|
-
return { events: selected, truncated: true, droppedCount: events.length - selected.length };
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
/**
|
|
566
|
-
* FULL-WALK enumeration: every path in the workspace tree — regular
|
|
567
|
-
* files, symlinks (including links to directories; links are recorded,
|
|
568
|
-
* never followed, so cycles are impossible), and .git internals. The
|
|
569
|
-
* ONLY exclusions are the secrets-denylist paths. Git-aware listing was
|
|
570
|
-
* removed on purpose: `git ls-files` drops anything .gitignored (build
|
|
571
|
-
* outputs, node_modules), which broke the whole-project guarantee.
|
|
572
|
-
*/
|
|
573
|
-
export async function listWorkspaceFiles(root: string): Promise<string[]> {
|
|
574
|
-
return walkWorkspace(root);
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
async function walkWorkspace(root: string, directory = root, output: string[] = []): Promise<string[]> {
|
|
578
|
-
let entries;
|
|
579
|
-
try {
|
|
580
|
-
entries = await readdir(directory, { withFileTypes: true });
|
|
581
|
-
} catch {
|
|
582
|
-
return output;
|
|
583
|
-
}
|
|
584
|
-
for (const entry of entries) {
|
|
585
|
-
const fullPath = resolve(directory, entry.name);
|
|
586
|
-
const path = portablePath(root, fullPath);
|
|
587
|
-
if (!path || isCollectorPathDenied(path)) continue;
|
|
588
|
-
if (entry.isDirectory()) await walkWorkspace(root, fullPath, output);
|
|
589
|
-
else output.push(path); // regular files AND symlinks (files or dirs)
|
|
590
|
-
}
|
|
591
|
-
return output;
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
async function gitMetadata(root: string): Promise<Record<string, string | null>> {
|
|
595
|
-
const git = async (...args: string[]) => {
|
|
596
|
-
try {
|
|
597
|
-
const { stdout } = await execFileAsync("git", ["-C", root, ...args], { timeout: 5_000, maxBuffer: 64 * 1024 });
|
|
598
|
-
return String(stdout).trim() || null;
|
|
599
|
-
} catch {
|
|
600
|
-
return null;
|
|
601
|
-
}
|
|
602
|
-
};
|
|
603
|
-
const [commit, branch, status] = await Promise.all([
|
|
604
|
-
git("rev-parse", "HEAD"),
|
|
605
|
-
git("branch", "--show-current"),
|
|
606
|
-
git("status", "--porcelain", "--untracked-files=no"),
|
|
607
|
-
]);
|
|
608
|
-
return { commit, branch, dirty: status ? "true" : "false" };
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
async function workspaceSignature(root: string): Promise<string> {
|
|
612
|
-
const hash = createHash("sha256");
|
|
613
|
-
for (const path of await listWorkspaceFiles(root)) {
|
|
614
|
-
try {
|
|
615
|
-
const file = await lstat(resolve(root, path));
|
|
616
|
-
if (file.isFile()) {
|
|
617
|
-
hash.update(path).update("\0").update(String(file.size)).update("\0").update(String(file.mtimeMs)).update("\0");
|
|
618
|
-
} else if (file.isSymbolicLink()) {
|
|
619
|
-
hash.update(path).update("\0").update("link").update("\0").update(String(file.mtimeMs)).update("\0");
|
|
620
|
-
}
|
|
621
|
-
} catch {
|
|
622
|
-
// A file can disappear while the workspace is being scanned.
|
|
623
|
-
}
|
|
624
|
-
}
|
|
625
|
-
return hash.digest("hex");
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
function sha256Hex(content: string): string {
|
|
629
|
-
return createHash("sha256").update(content).digest("hex");
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
/** Split a string into <= maxUnits slices without splitting surrogates. */
|
|
633
|
-
export function sliceString(content: string, maxUnits = MAX_SLICE_BYTES): string[] {
|
|
634
|
-
if (content.length <= maxUnits) return [content];
|
|
635
|
-
const slices: string[] = [];
|
|
636
|
-
let start = 0;
|
|
637
|
-
while (start < content.length) {
|
|
638
|
-
let end = Math.min(start + maxUnits, content.length);
|
|
639
|
-
if (end < content.length) {
|
|
640
|
-
const code = content.charCodeAt(end - 1);
|
|
641
|
-
if (code >= 0xd800 && code <= 0xdbff) end -= 1;
|
|
642
|
-
}
|
|
643
|
-
slices.push(content.slice(start, end));
|
|
644
|
-
start = end;
|
|
645
|
-
}
|
|
646
|
-
return slices;
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
/** Slice entry virtual path + manifest path for a logical file path. */
|
|
650
|
-
export function slicePartPath(path: string, index: number): string {
|
|
651
|
-
return `${path}.__agent_part${index}`;
|
|
652
|
-
}
|
|
653
|
-
export function sliceManifestPath(path: string): string {
|
|
654
|
-
return `${path}.__agent_manifest.json`;
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
/**
|
|
658
|
-
* Slice one logical file's content into <= MAX_SLICE_BYTES entries plus a
|
|
659
|
-
* manifest record (path, slice count, per-slice byte size + sha256) so
|
|
660
|
-
* replay reassembles the original deterministically. Small files pass
|
|
661
|
-
* through unchanged.
|
|
662
|
-
*/
|
|
663
|
-
export function buildFileEntries(file: CollectorFile): CollectorFile[] {
|
|
664
|
-
const content = file.content ?? "";
|
|
665
|
-
if (Buffer.byteLength(content) <= MAX_SLICE_BYTES) return [file];
|
|
666
|
-
const slices = sliceString(content);
|
|
667
|
-
const descriptors = slices.map((slice, index) => ({
|
|
668
|
-
index,
|
|
669
|
-
bytes: Buffer.byteLength(slice),
|
|
670
|
-
sha256: sha256Hex(slice),
|
|
671
|
-
}));
|
|
672
|
-
return [
|
|
673
|
-
{ path: sliceManifestPath(file.path), content: JSON.stringify(sliceManifest(file, descriptors)) },
|
|
674
|
-
...slices.map((slice, index) => ({
|
|
675
|
-
path: slicePartPath(file.path, index),
|
|
676
|
-
content: slice,
|
|
677
|
-
...(file.encoding ? { encoding: file.encoding } : {}),
|
|
678
|
-
})),
|
|
679
|
-
];
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
/**
|
|
683
|
-
* Manifest for a sliced file. `slice_count` is the GLOBAL count and each
|
|
684
|
-
* entry carries its global `index`, so manifests arriving across
|
|
685
|
-
* different parts union deterministically: a replay collects the part
|
|
686
|
-
* entries by index (verifying per-slice sha256) until all
|
|
687
|
-
* `slice_count` indexes are present, then reassembles.
|
|
688
|
-
*/
|
|
689
|
-
function sliceManifest(
|
|
690
|
-
file: { path: string; encoding?: "base64"; mode?: string },
|
|
691
|
-
sliceDescriptors: Array<{ index: number; bytes: number; sha256: string }>,
|
|
692
|
-
totalCount?: number,
|
|
693
|
-
): Record<string, unknown> {
|
|
694
|
-
return {
|
|
695
|
-
schema_version: 1,
|
|
696
|
-
path: file.path,
|
|
697
|
-
encoding: file.encoding ?? null,
|
|
698
|
-
mode: file.mode ?? null,
|
|
699
|
-
slice_count: totalCount ?? sliceDescriptors.length,
|
|
700
|
-
slices: sliceDescriptors,
|
|
701
|
-
};
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
type ContentChunk = { content: string; encoding?: "base64" };
|
|
705
|
-
|
|
706
|
-
/**
|
|
707
|
-
* Stream a regular file as <= ~8 MiB content chunks, yielding each chunk
|
|
708
|
-
* the moment it is produced — a file's content is NEVER materialized as
|
|
709
|
-
* one string regardless of size (issue #6: the old array version held
|
|
710
|
-
* every chunk at once). Small files are a single whole read; big files
|
|
711
|
-
* stream in fixed raw chunks. Binary detection happens on the first
|
|
712
|
-
* chunk and applies to the whole file; text is redacted per
|
|
713
|
-
* line-aligned segment.
|
|
714
|
-
*/
|
|
715
|
-
async function* iterateContentChunks(absolute: string, size: number): AsyncGenerator<ContentChunk> {
|
|
716
|
-
if (size <= WHOLE_READ_BYTES) {
|
|
717
|
-
const buffer = await readFile(absolute);
|
|
718
|
-
if (isBinary(buffer)) {
|
|
719
|
-
yield { content: buffer.toString("base64"), encoding: "base64" };
|
|
720
|
-
return;
|
|
721
|
-
}
|
|
722
|
-
yield { content: redactCollectorText(buffer.toString("utf8")).text };
|
|
723
|
-
return;
|
|
724
|
-
}
|
|
725
|
-
const { open } = await import("node:fs/promises");
|
|
726
|
-
const handle = await open(absolute, "r");
|
|
727
|
-
try {
|
|
728
|
-
const chunkBytes = 8 * 1024 * 1024;
|
|
729
|
-
const buffer = Buffer.alloc(chunkBytes);
|
|
730
|
-
const first = await handle.read(buffer, 0, chunkBytes, null);
|
|
731
|
-
const binary = isBinary(buffer.subarray(0, first.bytesRead));
|
|
732
|
-
if (binary) {
|
|
733
|
-
// Binaries carry no redactable text: base64 each raw chunk.
|
|
734
|
-
yield { content: buffer.subarray(0, first.bytesRead).toString("base64"), encoding: "base64" };
|
|
735
|
-
let read = 0;
|
|
736
|
-
while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
|
|
737
|
-
yield { content: buffer.subarray(0, read).toString("base64"), encoding: "base64" };
|
|
738
|
-
}
|
|
739
|
-
return;
|
|
740
|
-
}
|
|
741
|
-
// Text: decode incrementally (StringDecoder absorbs multibyte
|
|
742
|
-
// sequences split across chunk boundaries), emit only up to the
|
|
743
|
-
// last complete line, carry the remainder, and redact each
|
|
744
|
-
// line-aligned segment. A pathological newline-less file must not
|
|
745
|
-
// grow `pending` forever: at PENDING_TEXT_CAP_BYTES the segment is
|
|
746
|
-
// flushed mid-line (line-anchored redaction weakens at that rare
|
|
747
|
-
// cut; unbounded memory is worse).
|
|
748
|
-
const { StringDecoder } = await import("node:string_decoder");
|
|
749
|
-
const decoder = new StringDecoder("utf8");
|
|
750
|
-
let pending = decoder.write(buffer.subarray(0, first.bytesRead));
|
|
751
|
-
let read = 0;
|
|
752
|
-
while ((read = (await handle.read(buffer, 0, chunkBytes, null)).bytesRead) > 0) {
|
|
753
|
-
pending += decoder.write(buffer.subarray(0, read));
|
|
754
|
-
for (;;) {
|
|
755
|
-
const lastNl = pending.lastIndexOf("\n");
|
|
756
|
-
if (lastNl >= 0) {
|
|
757
|
-
yield { content: redactCollectorText(pending.slice(0, lastNl + 1)).text };
|
|
758
|
-
pending = pending.slice(lastNl + 1);
|
|
759
|
-
continue;
|
|
760
|
-
}
|
|
761
|
-
if (pending.length >= PENDING_TEXT_CAP_BYTES) {
|
|
762
|
-
yield { content: redactCollectorText(pending).text };
|
|
763
|
-
pending = "";
|
|
764
|
-
}
|
|
765
|
-
break;
|
|
766
|
-
}
|
|
767
|
-
}
|
|
768
|
-
pending += decoder.end();
|
|
769
|
-
if (pending) yield { content: redactCollectorText(pending).text };
|
|
770
|
-
} finally {
|
|
771
|
-
await handle.close();
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
/**
|
|
776
|
-
* Stream one workspace entry as CollectorFile entries. Regular files:
|
|
777
|
-
* text is redacted, binaries base64-encoded ("encoding": "base64"), and
|
|
778
|
-
* the POSIX mode rides along (octal string) so replay can restore the
|
|
779
|
-
* executable bit (stored-and-ignored on Windows). Content bigger than
|
|
780
|
-
* MAX_SLICE_BYTES is sliced on the fly — slices are yielded the moment
|
|
781
|
-
* they exist while only the tiny sha256 descriptors accumulate, then
|
|
782
|
-
* the manifest is yielded LAST (replay unions manifests by index, so
|
|
783
|
-
* position never matters). Symlinks become text entries
|
|
784
|
-
* { path, content: <target>, encoding: "symlink" } — the backend's
|
|
785
|
-
* per-entry validation requires text content, and replay recreates the
|
|
786
|
-
* actual link from the target. Yields nothing when the entry vanished
|
|
787
|
-
* or is a special file.
|
|
788
|
-
*/
|
|
789
|
-
async function* iterateWorkspaceEntries(root: string, path: string): AsyncGenerator<CollectorFile> {
|
|
790
|
-
let absolute = "";
|
|
791
|
-
try {
|
|
792
|
-
absolute = resolve(root, path);
|
|
793
|
-
if (portablePath(root, absolute).startsWith("../")) return;
|
|
794
|
-
const file = await lstat(absolute);
|
|
795
|
-
if (file.isSymbolicLink()) {
|
|
796
|
-
const target = await readlink(absolute);
|
|
797
|
-
yield { path, content: target, encoding: "symlink" };
|
|
798
|
-
return;
|
|
799
|
-
}
|
|
800
|
-
if (!file.isFile()) return;
|
|
801
|
-
const mode = file.mode & 0o777;
|
|
802
|
-
const modeField = mode ? { mode: mode.toString(8) } : {};
|
|
803
|
-
if (file.size <= WHOLE_READ_BYTES) {
|
|
804
|
-
// Small file: whole read, real path preserved; buildFileEntries
|
|
805
|
-
// slices (manifest + parts) when the encoded content exceeds
|
|
806
|
-
// MAX_SLICE_BYTES — same shape as always.
|
|
807
|
-
for await (const chunk of iterateContentChunks(absolute, file.size)) {
|
|
808
|
-
for (const entry of buildFileEntries({
|
|
809
|
-
path,
|
|
810
|
-
content: chunk.content,
|
|
811
|
-
...(chunk.encoding ? { encoding: chunk.encoding } : {}),
|
|
812
|
-
...modeField,
|
|
813
|
-
})) {
|
|
814
|
-
yield entry;
|
|
815
|
-
}
|
|
816
|
-
}
|
|
817
|
-
return;
|
|
818
|
-
}
|
|
819
|
-
// Big file (streamed): slices are yielded the moment they exist;
|
|
820
|
-
// only the tiny sha256 descriptors accumulate. The reassembly
|
|
821
|
-
// manifest is yielded LAST (replay unions manifests by index, so
|
|
822
|
-
// position never matters).
|
|
823
|
-
let index = 0;
|
|
824
|
-
const descriptors: Array<{ index: number; bytes: number; sha256: string }> = [];
|
|
825
|
-
let sawEncoding: "base64" | undefined;
|
|
826
|
-
for await (const chunk of iterateContentChunks(absolute, file.size)) {
|
|
827
|
-
sawEncoding ??= chunk.encoding;
|
|
828
|
-
for (const slice of sliceString(chunk.content)) {
|
|
829
|
-
descriptors.push({ index, bytes: Buffer.byteLength(slice), sha256: sha256Hex(slice) });
|
|
830
|
-
yield {
|
|
831
|
-
path: slicePartPath(path, index),
|
|
832
|
-
content: slice,
|
|
833
|
-
...(sawEncoding ? { encoding: sawEncoding } : {}),
|
|
834
|
-
};
|
|
835
|
-
index += 1;
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
if (index === 0) return;
|
|
839
|
-
yield {
|
|
840
|
-
path: sliceManifestPath(path),
|
|
841
|
-
content: JSON.stringify(sliceManifest(
|
|
842
|
-
{ path, ...(sawEncoding ? { encoding: sawEncoding } : {}), ...modeField },
|
|
843
|
-
descriptors,
|
|
844
|
-
)),
|
|
845
|
-
};
|
|
846
|
-
return;
|
|
847
|
-
} catch {
|
|
848
|
-
// Workspaces are live; races are expected and retried by the next snapshot.
|
|
849
|
-
return;
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
/**
|
|
854
|
-
* Streaming file collection: yields workspace files (metadata entry
|
|
855
|
-
* first) one at a time so the one-shot path never holds the whole tree's
|
|
856
|
-
* content in memory (issue #6). Same order, denylist, and byte budget
|
|
857
|
-
* as the compat collectFiles drain below.
|
|
858
|
-
*/
|
|
859
|
-
export async function* iterateCollectFiles(
|
|
860
|
-
root: string,
|
|
861
|
-
byteLimit: number,
|
|
862
|
-
workspaceId: string,
|
|
863
|
-
session?: Pick<SessionState, "id" | "segment" | "resumed">,
|
|
864
|
-
): AsyncGenerator<CollectorFile> {
|
|
865
|
-
let used = 0;
|
|
866
|
-
const metadata = JSON.stringify({
|
|
867
|
-
workspace_id: workspaceId,
|
|
868
|
-
...(session ? {
|
|
869
|
-
session_id: session.id,
|
|
870
|
-
session_segment: session.segment,
|
|
871
|
-
session_resumed: session.resumed,
|
|
872
|
-
} : {}),
|
|
873
|
-
root_name: root.split(sep).filter(Boolean).at(-1) ?? "workspace",
|
|
874
|
-
git: await gitMetadata(root),
|
|
875
|
-
});
|
|
876
|
-
yield { path: `${AGENT_DIR}/workspace.json`, content: metadata };
|
|
877
|
-
used += Buffer.byteLength(metadata);
|
|
878
|
-
|
|
879
|
-
const paths = (await listWorkspaceFiles(root)).sort(comparePaths);
|
|
880
|
-
for (const path of paths) {
|
|
881
|
-
if (used >= byteLimit) break;
|
|
882
|
-
for await (const file of iterateWorkspaceEntries(root, path)) {
|
|
883
|
-
const size = Buffer.byteLength(file.content ?? "");
|
|
884
|
-
if (used + size > byteLimit) continue;
|
|
885
|
-
yield file;
|
|
886
|
-
used += size;
|
|
887
|
-
}
|
|
888
|
-
}
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
/** Compat drain of iterateCollectFiles (tests + tooling). */
|
|
892
|
-
export async function collectFiles(
|
|
893
|
-
root: string,
|
|
894
|
-
byteLimit: number,
|
|
895
|
-
workspaceId: string,
|
|
896
|
-
session?: Pick<SessionState, "id" | "segment" | "resumed">,
|
|
897
|
-
): Promise<CollectorFile[]> {
|
|
898
|
-
const files: CollectorFile[] = [];
|
|
899
|
-
for await (const file of iterateCollectFiles(root, byteLimit, workspaceId, session)) {
|
|
900
|
-
files.push(file);
|
|
901
|
-
}
|
|
902
|
-
return files;
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
export function compressZstd(payload: Buffer): Promise<Buffer> {
|
|
906
|
-
return new Promise((resolvePromise, reject) => {
|
|
907
|
-
try {
|
|
908
|
-
const maybePromise = zstdCompressCb(payload, (error: unknown, result: Buffer) => {
|
|
909
|
-
if (error) reject(error);
|
|
910
|
-
else resolvePromise(result);
|
|
911
|
-
});
|
|
912
|
-
if (maybePromise && typeof (maybePromise as any).then === "function") {
|
|
913
|
-
(maybePromise as Promise<Buffer>).then(resolvePromise, reject);
|
|
914
|
-
}
|
|
915
|
-
} catch (error) {
|
|
916
|
-
reject(error);
|
|
917
|
-
}
|
|
918
|
-
});
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
/** The uncompressed collect envelope (contract: schema_version 1).
|
|
922
|
-
* Additive metadata: client_version / client_platform when known and a
|
|
923
|
-
* sent_at timestamp — the backend ignores what it does not know. */
|
|
924
|
-
export function buildEnvelopePayload(
|
|
925
|
-
state: Pick<SessionState, "id" | "segment" | "resumed"> & {
|
|
926
|
-
sequence?: number;
|
|
927
|
-
clientVersion?: string;
|
|
928
|
-
clientPlatform?: string;
|
|
929
|
-
/** Capture time override (change snapshots carry the time the
|
|
930
|
-
* journal records were captured, not the time the queued upload
|
|
931
|
-
* finally built the envelope). */
|
|
932
|
-
capturedAt?: string;
|
|
933
|
-
},
|
|
934
|
-
snapshotType: SnapshotType,
|
|
935
|
-
files: CollectorFile[],
|
|
936
|
-
): Buffer {
|
|
937
|
-
return Buffer.from(JSON.stringify({
|
|
938
|
-
schema_version: 1,
|
|
939
|
-
session_id: state.id,
|
|
940
|
-
session_segment: state.segment,
|
|
941
|
-
session_resumed: state.resumed,
|
|
942
|
-
sequence: (state.sequence ?? 0) + 1,
|
|
943
|
-
snapshot_type: snapshotType,
|
|
944
|
-
...(state.clientVersion ? { client_version: state.clientVersion } : {}),
|
|
945
|
-
...(state.clientPlatform ? { client_platform: state.clientPlatform } : {}),
|
|
946
|
-
sent_at: new Date().toISOString(),
|
|
947
|
-
// Capture time for the backend's per-turn diff attribution (the
|
|
948
|
-
// desktop sends the same field; without it a CLI change snapshot
|
|
949
|
-
// cannot be pinned to the turn that was running).
|
|
950
|
-
captured_at: state.capturedAt ?? new Date().toISOString(),
|
|
951
|
-
files,
|
|
952
|
-
}));
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
export type EnvelopePart = {
|
|
956
|
-
files: CollectorFile[];
|
|
957
|
-
payload: Buffer;
|
|
958
|
-
compressed: Buffer;
|
|
959
|
-
};
|
|
960
|
-
|
|
961
|
-
/**
|
|
962
|
-
* Deterministic part split for a full files array (compat drain of
|
|
963
|
-
* iterateEnvelopeParts, used by the tests): sort files by path,
|
|
964
|
-
* accumulate in order, keep each part <= maxCompressed compressed.
|
|
965
|
-
*/
|
|
966
|
-
export async function buildEnvelopeParts(
|
|
967
|
-
state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
|
|
968
|
-
snapshotType: SnapshotType,
|
|
969
|
-
files: CollectorFile[],
|
|
970
|
-
maxCompressed = MAX_PART_COMPRESSED_BYTES,
|
|
971
|
-
log?: (message: string, attributes?: Record<string, unknown>) => void,
|
|
972
|
-
): Promise<EnvelopePart[]> {
|
|
973
|
-
// Slice any entry whose content exceeds MAX_SLICE_BYTES (manifest +
|
|
974
|
-
// ordered parts) BEFORE packing, so callers feeding raw file lists get
|
|
975
|
-
// the same deterministic slice layout as the tree walker.
|
|
976
|
-
const sorted = files
|
|
977
|
-
.flatMap((file) => buildFileEntries(file))
|
|
978
|
-
.sort((left, right) => comparePaths(left.path, right.path));
|
|
979
|
-
const parts: EnvelopePart[] = [];
|
|
980
|
-
for await (const part of iterateEnvelopeParts(state, snapshotType, sorted, maxCompressed, log)) {
|
|
981
|
-
parts.push(part);
|
|
982
|
-
}
|
|
983
|
-
return parts;
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
/**
|
|
987
|
-
* Streaming packer for the one-shot path: yields compressed parts in
|
|
988
|
-
* order so callers can upload-and-discard instead of holding the whole
|
|
989
|
-
* workspace's parts (payload + compressed buffers) in memory (issue #6).
|
|
990
|
-
* Same greedy algorithm as before: split at the compressed target, a
|
|
991
|
-
* lone file may use the hard rail, lone-over-rail files are skipped
|
|
992
|
-
* loudly. File content is never split across parts.
|
|
993
|
-
*/
|
|
994
|
-
export async function* iterateEnvelopeParts(
|
|
995
|
-
state: Pick<SessionState, "id" | "segment" | "resumed"> & { sequence?: number },
|
|
996
|
-
snapshotType: SnapshotType,
|
|
997
|
-
sortedFiles: Iterable<CollectorFile> | AsyncIterable<CollectorFile>,
|
|
998
|
-
maxCompressed = MAX_PART_COMPRESSED_BYTES,
|
|
999
|
-
log?: (message: string, attributes?: Record<string, unknown>) => void,
|
|
1000
|
-
): AsyncGenerator<EnvelopePart> {
|
|
1001
|
-
const hard = Math.max(maxCompressed, MAX_PART_COMPRESSED_HARD);
|
|
1002
|
-
const measure = async (batch: CollectorFile[], partIndex: number): Promise<EnvelopePart> => {
|
|
1003
|
-
const payload = buildEnvelopePayload({ ...state, sequence: (state.sequence ?? 0) + partIndex }, snapshotType, batch);
|
|
1004
|
-
return { files: batch, payload, compressed: await compressZstd(payload) };
|
|
1005
|
-
};
|
|
1006
|
-
let partsYielded = 0;
|
|
1007
|
-
let batch: CollectorFile[] = [];
|
|
1008
|
-
const fit = async (batchToFit: CollectorFile[], partIndex: number) => {
|
|
1009
|
-
let measured = await measure(batchToFit, partIndex);
|
|
1010
|
-
const carry: CollectorFile[] = [];
|
|
1011
|
-
// Split at the compressed target; a lone file may use the body
|
|
1012
|
-
// headroom up to the hard rail, beyond which it is skipped loudly
|
|
1013
|
-
// rather than split mid-content.
|
|
1014
|
-
while (measured.compressed.length > maxCompressed && batchToFit.length > 1) {
|
|
1015
|
-
carry.unshift(batchToFit.pop()!);
|
|
1016
|
-
measured = await measure(batchToFit, partIndex);
|
|
1017
|
-
}
|
|
1018
|
-
if (measured.compressed.length > hard && batchToFit.length === 1) {
|
|
1019
|
-
const lone = batchToFit.pop()!;
|
|
1020
|
-
log?.("OmniRush file exceeds the upload limits and cannot be shipped as a single part; skipped", {
|
|
1021
|
-
path: lone.path,
|
|
1022
|
-
bytes: Buffer.byteLength(lone.content ?? ""),
|
|
1023
|
-
});
|
|
1024
|
-
return { measured: null, carry };
|
|
1025
|
-
}
|
|
1026
|
-
return { measured, carry };
|
|
1027
|
-
};
|
|
1028
|
-
for await (const file of sortedFiles) {
|
|
1029
|
-
batch.push(file);
|
|
1030
|
-
const partIndex = partsYielded;
|
|
1031
|
-
const estimate = batch.reduce((total, f) => total + Buffer.byteLength(f.content), 0);
|
|
1032
|
-
if (estimate < PART_UNCOMPRESSED_STEP) continue;
|
|
1033
|
-
const { measured, carry } = await fit(batch, partIndex);
|
|
1034
|
-
if (measured
|
|
1035
|
-
&& (measured.compressed.length >= PART_READY_COMPRESSED_BYTES || estimate >= PART_BATCH_UNCOMPRESSED_MAX)) {
|
|
1036
|
-
partsYielded += 1;
|
|
1037
|
-
yield measured;
|
|
1038
|
-
batch = carry;
|
|
1039
|
-
} else if (measured) {
|
|
1040
|
-
batch = [...batch, ...carry];
|
|
1041
|
-
} else {
|
|
1042
|
-
batch = carry;
|
|
1043
|
-
}
|
|
1044
|
-
}
|
|
1045
|
-
while (batch.length > 0) {
|
|
1046
|
-
const { measured, carry } = await fit(batch, partsYielded);
|
|
1047
|
-
if (measured) {
|
|
1048
|
-
partsYielded += 1;
|
|
1049
|
-
yield measured;
|
|
1050
|
-
}
|
|
1051
|
-
if (carry.length === 0) break;
|
|
1052
|
-
batch = carry;
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
export class WorkspaceCollector {
|
|
1057
|
-
/**
|
|
1058
|
-
* Bounded first-chunk read of an error response body for diagnostics:
|
|
1059
|
-
* at most 512 bytes, decoded and whitespace-collapsed. Never throws.
|
|
1060
|
-
* Error bodies are small JSON from the manager; the bound exists so a
|
|
1061
|
-
* pathological response cannot balloon a log line (or memory).
|
|
1062
|
-
*/
|
|
1063
|
-
private static async errorBodySnippet(response: any): Promise<string> {
|
|
1064
|
-
try {
|
|
1065
|
-
const reader = response?.body?.getReader?.();
|
|
1066
|
-
if (!reader) return "";
|
|
1067
|
-
const { value } = await reader.read();
|
|
1068
|
-
reader.cancel().catch(() => undefined);
|
|
1069
|
-
if (!value) return "";
|
|
1070
|
-
const text = Buffer.from(value).toString("utf8").replace(/\s+/g, " ").trim();
|
|
1071
|
-
return text.length > 200 ? `${text.slice(0, 200)}…` : text;
|
|
1072
|
-
} catch {
|
|
1073
|
-
return "";
|
|
1074
|
-
}
|
|
1075
|
-
}
|
|
1076
|
-
|
|
1077
|
-
private readonly collectUrl: string | null;
|
|
1078
|
-
private token: string;
|
|
1079
|
-
private readonly fetcher: typeof fetch;
|
|
1080
|
-
private readonly refresh?: CollectorOptions["refresh"];
|
|
1081
|
-
private readonly uploader?: CollectorOptions["upload"];
|
|
1082
|
-
private readonly log: NonNullable<CollectorOptions["log"]>;
|
|
1083
|
-
private readonly ledgerPath: string | null;
|
|
1084
|
-
private readonly stateDir: string | null;
|
|
1085
|
-
private readonly agentDir: string | null;
|
|
1086
|
-
private readonly identityProvider?: CollectorOptions["identityProvider"];
|
|
1087
|
-
private readonly clientId?: string;
|
|
1088
|
-
private identity: { userId: string | null } | null = null;
|
|
1089
|
-
private identityTried = false;
|
|
1090
|
-
private readonly ledgerReady: Promise<void>;
|
|
1091
|
-
private ledger: SessionLedger = { version: 1, sessions: {} };
|
|
1092
|
-
private ledgerWriteTail: Promise<void> = Promise.resolve();
|
|
1093
|
-
private readonly sessions = new Map<string, SessionState>();
|
|
1094
|
-
/** Gate for states built without one (direct uploadEnvelope callers,
|
|
1095
|
-
* tests) — per-session gates are created in startSession. */
|
|
1096
|
-
private readonly defaultGate = new PriorityGate();
|
|
1097
|
-
private readonly changeDebounceMs: number;
|
|
1098
|
-
private readonly fallbackScanMs: number;
|
|
1099
|
-
private readonly sessionBudgetBytes: number;
|
|
1100
|
-
private readonly clientVersion: string;
|
|
1101
|
-
private readonly clientPlatform: string;
|
|
1102
|
-
private readonly linkedParentSession: string | null;
|
|
1103
|
-
private readonly partCompressedBytes: number;
|
|
1104
|
-
private readonly partCheckBytes: number;
|
|
1105
|
-
|
|
1106
|
-
constructor(options: CollectorOptions = {}) {
|
|
1107
|
-
this.collectUrl = resolveCollectUrl(options.gatewayUrl ?? process.env.OMNIRUSH_GATEWAY_URL);
|
|
1108
|
-
this.token = (options.accessToken ?? process.env.OMNIRUSH_ACCESS_TOKEN ?? "").trim();
|
|
1109
|
-
this.fetcher = options.fetch ?? ((input: any, init?: any) => fetch(input, init));
|
|
1110
|
-
this.refresh = options.refresh;
|
|
1111
|
-
this.uploader = options.upload;
|
|
1112
|
-
this.log = options.log ?? (() => undefined);
|
|
1113
|
-
this.stateDir = options.stateDir ? resolve(options.stateDir) : null;
|
|
1114
|
-
this.agentDir = options.agentDir ? resolve(options.agentDir) : null;
|
|
1115
|
-
this.identityProvider = options.identityProvider;
|
|
1116
|
-
this.clientId = options.clientId;
|
|
1117
|
-
this.ledgerPath = options.stateDir ? join(resolve(options.stateDir), SESSION_LEDGER_FILE) : null;
|
|
1118
|
-
this.ledgerReady = this.loadLedger();
|
|
1119
|
-
this.changeDebounceMs = options.changeDebounceMs ?? CHANGE_DEBOUNCE_MS;
|
|
1120
|
-
this.fallbackScanMs = options.fallbackScanMs ?? FALLBACK_SCAN_MS;
|
|
1121
|
-
this.sessionBudgetBytes = Math.max(1024, options.sessionBudgetBytes ?? MAX_SESSION_BYTES);
|
|
1122
|
-
this.clientVersion = options.clientVersion ?? detectClientVersion();
|
|
1123
|
-
this.clientPlatform = options.clientPlatform ?? detectClientPlatform();
|
|
1124
|
-
this.linkedParentSession = (options.linkedParentSession ?? "").trim() || null;
|
|
1125
|
-
this.partCompressedBytes = Math.min(
|
|
1126
|
-
Math.max(1024, options.partCompressedBytes ?? MAX_PART_COMPRESSED_BYTES),
|
|
1127
|
-
MAX_PART_COMPRESSED_HARD,
|
|
1128
|
-
);
|
|
1129
|
-
this.partCheckBytes = Math.max(1024, options.partCheckBytes ?? PART_UNCOMPRESSED_STEP);
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
private async loadLedger(): Promise<void> {
|
|
1133
|
-
this.ledger = await readSessionLedger(this.ledgerPath);
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
private async saveLedger(): Promise<void> {
|
|
1137
|
-
if (!this.ledgerPath) return;
|
|
1138
|
-
await this.ledgerReady;
|
|
1139
|
-
const entries = Object.entries(this.ledger.sessions)
|
|
1140
|
-
.sort(([, left], [, right]) => right.lastSeenAt.localeCompare(left.lastSeenAt))
|
|
1141
|
-
.slice(0, MAX_SESSION_LEDGER_ENTRIES);
|
|
1142
|
-
this.ledger.sessions = Object.fromEntries(entries);
|
|
1143
|
-
const snapshot = JSON.stringify(this.ledger);
|
|
1144
|
-
this.ledgerWriteTail = this.ledgerWriteTail
|
|
1145
|
-
.catch(() => undefined)
|
|
1146
|
-
.then(async () => {
|
|
1147
|
-
await mkdir(dirname(this.ledgerPath!), { recursive: true, mode: 0o700 });
|
|
1148
|
-
await writeFile(this.ledgerPath!, snapshot, { encoding: "utf8", mode: 0o600 });
|
|
1149
|
-
});
|
|
1150
|
-
await this.ledgerWriteTail;
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
/** The state fields every envelope builder needs, plus the additive
|
|
1154
|
-
* client metadata. `sent_at`/`captured_at` are stamped by
|
|
1155
|
-
* buildEnvelopePayload at build time. */
|
|
1156
|
-
private sequenceState(
|
|
1157
|
-
state: SessionState,
|
|
1158
|
-
capturedAt?: string,
|
|
1159
|
-
): {
|
|
1160
|
-
id: string;
|
|
1161
|
-
segment: number;
|
|
1162
|
-
resumed: boolean;
|
|
1163
|
-
sequence: number;
|
|
1164
|
-
clientVersion?: string;
|
|
1165
|
-
clientPlatform?: string;
|
|
1166
|
-
capturedAt?: string;
|
|
1167
|
-
} {
|
|
1168
|
-
return {
|
|
1169
|
-
id: state.id,
|
|
1170
|
-
segment: state.segment,
|
|
1171
|
-
resumed: state.resumed,
|
|
1172
|
-
sequence: state.sequence,
|
|
1173
|
-
...(this.clientVersion ? { clientVersion: this.clientVersion } : {}),
|
|
1174
|
-
...(this.clientPlatform ? { clientPlatform: this.clientPlatform } : {}),
|
|
1175
|
-
// A change part is pinned to the capture time of its OLDEST
|
|
1176
|
-
// journal record: the upload may queue behind the baseline, but
|
|
1177
|
-
// the turn attribution must reflect when the edit was seen.
|
|
1178
|
-
...(capturedAt ? { capturedAt } : {}),
|
|
1179
|
-
};
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
private async prepareSession(state: SessionState): Promise<void> {
|
|
1183
|
-
await this.ledgerReady;
|
|
1184
|
-
const previous = this.ledger.sessions[state.id];
|
|
1185
|
-
state.segment = (previous?.segment ?? 0) + 1;
|
|
1186
|
-
state.sequence = previous?.nextSequence ?? 0;
|
|
1187
|
-
state.sentBytes = previous?.sentBytes ?? 0;
|
|
1188
|
-
state.lastMessageId = previous?.lastMessageId;
|
|
1189
|
-
state.baselineUploaded = previous?.baselineUploaded === true;
|
|
1190
|
-
// An interrupted baseline resumes from its cursor; a completed one
|
|
1191
|
-
// (or a fresh session) has none.
|
|
1192
|
-
state.baselineResumePath =
|
|
1193
|
-
previous && !state.baselineUploaded
|
|
1194
|
-
? typeof previous.baselineResumePath === "string"
|
|
1195
|
-
? previous.baselineResumePath
|
|
1196
|
-
: null
|
|
1197
|
-
: null;
|
|
1198
|
-
state.resumed = Boolean(previous);
|
|
1199
|
-
if (previous?.traces && typeof previous.traces === "object") {
|
|
1200
|
-
state.transcripts = { ...previous.traces };
|
|
1201
|
-
}
|
|
1202
|
-
if (state.resumed) {
|
|
1203
|
-
state.trace.push({
|
|
1204
|
-
at: new Date().toISOString(),
|
|
1205
|
-
type: "session.resumed",
|
|
1206
|
-
data: { session_segment: state.segment, previous_segment: previous?.segment ?? null },
|
|
1207
|
-
});
|
|
1208
|
-
}
|
|
1209
|
-
this.ledger.sessions[state.id] = {
|
|
1210
|
-
segment: state.segment,
|
|
1211
|
-
nextSequence: state.sequence,
|
|
1212
|
-
sentBytes: state.sentBytes,
|
|
1213
|
-
...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
|
|
1214
|
-
...(state.baselineUploaded ? { baselineUploaded: true } : {}),
|
|
1215
|
-
...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
|
|
1216
|
-
lastSeenAt: new Date().toISOString(),
|
|
1217
|
-
};
|
|
1218
|
-
await this.saveLedger();
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
private async persistSession(state: SessionState): Promise<void> {
|
|
1222
|
-
await this.ledgerReady;
|
|
1223
|
-
this.ledger.sessions[state.id] = {
|
|
1224
|
-
segment: state.segment,
|
|
1225
|
-
nextSequence: state.sequence,
|
|
1226
|
-
sentBytes: state.sentBytes,
|
|
1227
|
-
...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
|
|
1228
|
-
...(state.baselineUploaded ? { baselineUploaded: true } : {}),
|
|
1229
|
-
...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
|
|
1230
|
-
...(Object.keys(state.transcripts).length > 0 ? { traces: state.transcripts } : {}),
|
|
1231
|
-
lastSeenAt: new Date().toISOString(),
|
|
1232
|
-
};
|
|
1233
|
-
await this.saveLedger();
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
async sessionCheckpoint(sessionId: string): Promise<{ resumed: boolean; segment?: number; lastMessageId?: string }> {
|
|
1237
|
-
const state = this.sessions.get(sessionId);
|
|
1238
|
-
if (!state) return { resumed: false };
|
|
1239
|
-
await state.ready;
|
|
1240
|
-
return { resumed: state.resumed, segment: state.segment, lastMessageId: state.lastMessageId };
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
async setSessionCheckpoint(sessionId: string, messageId: string): Promise<void> {
|
|
1244
|
-
const state = this.sessions.get(sessionId);
|
|
1245
|
-
if (!state || state.finished) return;
|
|
1246
|
-
await state.ready;
|
|
1247
|
-
state.lastMessageId = messageId;
|
|
1248
|
-
await this.persistSession(state);
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
get enabled(): boolean {
|
|
1252
|
-
return Boolean(this.uploader || (this.collectUrl && this.token));
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
startSession(sessionId: string, workspaceId: string, root: string): void {
|
|
1256
|
-
if (!this.enabled || this.sessions.has(sessionId) || !/^[A-Za-z0-9._:-]{8,128}$/.test(sessionId)) return;
|
|
1257
|
-
const state: SessionState = {
|
|
1258
|
-
id: sessionId,
|
|
1259
|
-
root,
|
|
1260
|
-
workspaceId,
|
|
1261
|
-
sentBytes: 0,
|
|
1262
|
-
segment: 1,
|
|
1263
|
-
sequence: 0,
|
|
1264
|
-
resumed: false,
|
|
1265
|
-
lastSignature: "",
|
|
1266
|
-
started: false,
|
|
1267
|
-
finished: false,
|
|
1268
|
-
budgetExhausted: false,
|
|
1269
|
-
baselineUploaded: false,
|
|
1270
|
-
baselineResumePath: null,
|
|
1271
|
-
changeTimer: null,
|
|
1272
|
-
scanTimer: null,
|
|
1273
|
-
knownStats: new Map(),
|
|
1274
|
-
scanning: false,
|
|
1275
|
-
baselineSeeded: false,
|
|
1276
|
-
knownPaths: new Set(),
|
|
1277
|
-
trace: [],
|
|
1278
|
-
journalPath: this.stateDir ? join(resolve(this.stateDir), `journal-${sessionId}.ndjson`) : null,
|
|
1279
|
-
journalAckOffset: 0,
|
|
1280
|
-
journalAppendedBytes: 0,
|
|
1281
|
-
journalAppendTail: Promise.resolve(),
|
|
1282
|
-
journalUploadEof: null,
|
|
1283
|
-
journalCaptureStopped: false,
|
|
1284
|
-
journalLastByPath: new Map(),
|
|
1285
|
-
transcripts: {},
|
|
1286
|
-
changeCaptureTail: Promise.resolve(),
|
|
1287
|
-
ready: Promise.resolve(),
|
|
1288
|
-
tail: Promise.resolve(),
|
|
1289
|
-
traceTail: Promise.resolve(),
|
|
1290
|
-
tracePending: 0,
|
|
1291
|
-
traceDrain: Promise.resolve(),
|
|
1292
|
-
resolveTraceDrain: () => undefined,
|
|
1293
|
-
gate: new PriorityGate(),
|
|
1294
|
-
};
|
|
1295
|
-
state.ready = this.prepareSession(state);
|
|
1296
|
-
this.sessions.set(sessionId, state);
|
|
1297
|
-
if (this.linkedParentSession) {
|
|
1298
|
-
// Child of a live parent session: the parent owns the workspace
|
|
1299
|
-
// collection (baseline/changes/end). This collector only traces.
|
|
1300
|
-
state.started = true;
|
|
1301
|
-
return;
|
|
1302
|
-
}
|
|
1303
|
-
this.enqueue(state, async () => {
|
|
1304
|
-
await state.ready;
|
|
1305
|
-
const baselinePaths = await listWorkspaceFiles(root);
|
|
1306
|
-
state.knownPaths = new Set(baselinePaths);
|
|
1307
|
-
state.lastSignature = await workspaceSignature(root);
|
|
1308
|
-
// Seed the per-path stat snapshot so the scan loop only captures
|
|
1309
|
-
// files that change AFTER the start snapshot.
|
|
1310
|
-
for (const relPath of baselinePaths) {
|
|
1311
|
-
try {
|
|
1312
|
-
const stats = await lstat(resolve(root, relPath));
|
|
1313
|
-
state.knownStats.set(relPath, { mtimeMs: stats.mtimeMs, size: stats.size });
|
|
1314
|
-
} catch {
|
|
1315
|
-
/* raced away — the next scan re-checks */
|
|
1316
|
-
}
|
|
1317
|
-
}
|
|
1318
|
-
state.baselineSeeded = true;
|
|
1319
|
-
if (state.resumed && state.baselineUploaded) {
|
|
1320
|
-
// Resume of a session whose start baseline already shipped: do
|
|
1321
|
-
// NOT re-upload the full tree. A restart-heavy session would
|
|
1322
|
-
// otherwise re-burn the entire baseline (and starve traces) on
|
|
1323
|
-
// every restart — observed live as 846 start snapshots /
|
|
1324
|
-
// multiple GB for one session. Changes + the end snapshot cover
|
|
1325
|
-
// the deltas; a FAILED baseline (flag unset) still retries here.
|
|
1326
|
-
state.started = true;
|
|
1327
|
-
return;
|
|
1328
|
-
}
|
|
1329
|
-
await this.uploadWorkspace(state, "start");
|
|
1330
|
-
state.baselineUploaded = true;
|
|
1331
|
-
state.baselineResumePath = null;
|
|
1332
|
-
await this.persistSession(state);
|
|
1333
|
-
state.started = true;
|
|
1334
|
-
});
|
|
1335
|
-
// Change detection: signature scan (unref'd, self-serializing). No
|
|
1336
|
-
// fs.watch anywhere — its recursive handle keeps the event loop
|
|
1337
|
-
// alive even after close()/unref(), hanging every session at exit.
|
|
1338
|
-
state.scanTimer = setInterval(() => {
|
|
1339
|
-
void this.scanForChanges(state).catch(() => undefined);
|
|
1340
|
-
}, this.fallbackScanMs);
|
|
1341
|
-
state.scanTimer.unref?.();
|
|
1342
|
-
}
|
|
1343
|
-
|
|
1344
|
-
/**
|
|
1345
|
-
* Periodic change detection: cheap signature check first; on a real
|
|
1346
|
-
* change, lstat-diff every path against knownStats and capture exactly
|
|
1347
|
-
* the new/changed/deleted files into the on-disk journal.
|
|
1348
|
-
*/
|
|
1349
|
-
private async scanForChanges(state: SessionState): Promise<void> {
|
|
1350
|
-
if (state.scanning || state.finished || state.budgetExhausted) return;
|
|
1351
|
-
if (!state.baselineSeeded) return;
|
|
1352
|
-
state.scanning = true;
|
|
1353
|
-
try {
|
|
1354
|
-
const signature = await workspaceSignature(state.root);
|
|
1355
|
-
if (!signature || signature === state.lastSignature) return;
|
|
1356
|
-
const paths = (await listWorkspaceFiles(state.root)).sort(comparePaths);
|
|
1357
|
-
const seen = new Set<string>();
|
|
1358
|
-
for (const relPath of paths) {
|
|
1359
|
-
seen.add(relPath);
|
|
1360
|
-
let stats;
|
|
1361
|
-
try {
|
|
1362
|
-
stats = await lstat(resolve(state.root, relPath));
|
|
1363
|
-
} catch {
|
|
1364
|
-
continue;
|
|
1365
|
-
}
|
|
1366
|
-
if (isCollectorPathDenied(relPath)) continue;
|
|
1367
|
-
const known = state.knownStats.get(relPath);
|
|
1368
|
-
if (known && known.mtimeMs === stats.mtimeMs && known.size === stats.size) continue;
|
|
1369
|
-
state.knownStats.set(relPath, { mtimeMs: stats.mtimeMs, size: stats.size });
|
|
1370
|
-
await this.captureChangedPath(state, relPath, { force: true });
|
|
1371
|
-
}
|
|
1372
|
-
// Deletions: paths we knew about that vanished.
|
|
1373
|
-
for (const [relPath] of state.knownStats) {
|
|
1374
|
-
if (!seen.has(relPath)) {
|
|
1375
|
-
state.knownStats.delete(relPath);
|
|
1376
|
-
await this.captureChangedPath(state, relPath, { force: true });
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
state.lastSignature = signature;
|
|
1380
|
-
this.scheduleChange(state);
|
|
1381
|
-
} finally {
|
|
1382
|
-
state.scanning = false;
|
|
1383
|
-
}
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
|
-
recordTrace(sessionId: string, type: string, data?: unknown): void {
|
|
1387
|
-
const state = this.sessions.get(sessionId);
|
|
1388
|
-
if (!state || state.finished || state.budgetExhausted) return;
|
|
1389
|
-
state.trace.push({ at: new Date().toISOString(), type, ...(data === undefined ? {} : { data }) });
|
|
1390
|
-
if (state.trace.length >= TRACE_RELIEF_EVENTS) {
|
|
1391
|
-
// Memory relief, NOT truncation: the whole batch is uploaded.
|
|
1392
|
-
const batch = state.trace.splice(0);
|
|
1393
|
-
this.enqueueTrace(state, async () => {
|
|
1394
|
-
await state.ready;
|
|
1395
|
-
await this.uploadTrace(state, batch);
|
|
1396
|
-
});
|
|
1397
|
-
}
|
|
1398
|
-
}
|
|
1399
|
-
|
|
1400
|
-
finishSession(sessionId: string, finalTrace?: unknown): void {
|
|
1401
|
-
const state = this.sessions.get(sessionId);
|
|
1402
|
-
if (!state || state.finished) return;
|
|
1403
|
-
const pendingTrace = state.trace.splice(0);
|
|
1404
|
-
if (finalTrace !== undefined) {
|
|
1405
|
-
pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
|
|
1406
|
-
}
|
|
1407
|
-
state.finished = true;
|
|
1408
|
-
if (state.changeTimer) clearTimeout(state.changeTimer);
|
|
1409
|
-
if (state.scanTimer) clearInterval(state.scanTimer);
|
|
1410
|
-
// Final trace rides the PRIORITY lane: an in-flight baseline part
|
|
1411
|
-
// finishes first, then the trace ships — never behind the tail of a
|
|
1412
|
-
// multi-part tree upload.
|
|
1413
|
-
this.enqueueTrace(state, async () => {
|
|
1414
|
-
await state.ready;
|
|
1415
|
-
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
|
|
1416
|
-
await this.uploadTrace(state, pendingTrace);
|
|
1417
|
-
});
|
|
1418
|
-
this.enqueue(state, async () => {
|
|
1419
|
-
await state.ready;
|
|
1420
|
-
await state.changeCaptureTail;
|
|
1421
|
-
await this.teardownJournal(state); // ship unacked change records, then unlink
|
|
1422
|
-
// The end snapshot yields to pending trace work (yieldToTrace in
|
|
1423
|
-
// the packer), so the final trace lands first.
|
|
1424
|
-
if (!this.linkedParentSession) {
|
|
1425
|
-
// A linked child ships no end snapshot: the parent session's
|
|
1426
|
-
// collection covers the workspace tree.
|
|
1427
|
-
await this.uploadWorkspace(state, "end");
|
|
1428
|
-
}
|
|
1429
|
-
this.sessions.delete(sessionId);
|
|
1430
|
-
});
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
flushTrace(sessionId: string, finalTrace?: unknown): void {
|
|
1434
|
-
const state = this.sessions.get(sessionId);
|
|
1435
|
-
if (!state || state.finished) return;
|
|
1436
|
-
const pendingTrace = state.trace.splice(0);
|
|
1437
|
-
if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });
|
|
1438
|
-
if (pendingTrace.length === 0) return;
|
|
1439
|
-
// Per-turn flush: the complete trace artifact (events + transcript
|
|
1440
|
-
// messages-so-far) ships on the priority lane, ahead of any queued
|
|
1441
|
-
// baseline parts.
|
|
1442
|
-
this.enqueueTrace(state, async () => {
|
|
1443
|
-
await state.ready;
|
|
1444
|
-
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
|
|
1445
|
-
await this.uploadTrace(state, pendingTrace);
|
|
1446
|
-
});
|
|
1447
|
-
}
|
|
1448
|
-
|
|
1449
|
-
async stop(): Promise<void> {
|
|
1450
|
-
for (const sessionId of [...this.sessions.keys()]) this.finishSession(sessionId);
|
|
1451
|
-
await Promise.allSettled(
|
|
1452
|
-
[...this.sessions.values()].flatMap((state) => [state.tail, state.traceTail]),
|
|
1453
|
-
);
|
|
1454
|
-
}
|
|
1455
|
-
|
|
1456
|
-
/**
|
|
1457
|
-
* Queue trace work on the priority lane and mark it pending: the
|
|
1458
|
-
* background lane's `yieldToTrace` waits for `tracePending` to drain
|
|
1459
|
-
* before starting another baseline part, so trace work enqueued while
|
|
1460
|
-
* a baseline part was in flight goes next.
|
|
1461
|
-
*/
|
|
1462
|
-
private enqueueTrace(state: SessionState, operation: () => Promise<void>): void {
|
|
1463
|
-
if (state.tracePending === 0) {
|
|
1464
|
-
let resolveDrain!: () => void;
|
|
1465
|
-
state.traceDrain = new Promise<void>((resolvePromise) => {
|
|
1466
|
-
resolveDrain = resolvePromise;
|
|
1467
|
-
});
|
|
1468
|
-
state.resolveTraceDrain = () => resolveDrain();
|
|
1469
|
-
}
|
|
1470
|
-
state.tracePending += 1;
|
|
1471
|
-
state.traceTail = state.traceTail
|
|
1472
|
-
.then(operation)
|
|
1473
|
-
.catch((error: unknown) => {
|
|
1474
|
-
this.log("warn", "OmniRush collection operation failed", {
|
|
1475
|
-
sessionId: state.id,
|
|
1476
|
-
error: error instanceof Error ? error.message : "unknown",
|
|
1477
|
-
});
|
|
1478
|
-
})
|
|
1479
|
-
.finally(() => {
|
|
1480
|
-
state.tracePending -= 1;
|
|
1481
|
-
if (state.tracePending === 0) state.resolveTraceDrain();
|
|
1482
|
-
});
|
|
1483
|
-
}
|
|
1484
|
-
|
|
1485
|
-
/** Wait until no trace work is queued or running (re-arming safe). */
|
|
1486
|
-
private async yieldToTrace(state: SessionState): Promise<void> {
|
|
1487
|
-
while (state.tracePending > 0) {
|
|
1488
|
-
await state.traceDrain;
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
|
|
1492
|
-
private stopWatching(state: SessionState): void {
|
|
1493
|
-
if (state.changeTimer) {
|
|
1494
|
-
clearTimeout(state.changeTimer);
|
|
1495
|
-
state.changeTimer = null;
|
|
1496
|
-
}
|
|
1497
|
-
if (state.scanTimer) {
|
|
1498
|
-
clearInterval(state.scanTimer);
|
|
1499
|
-
state.scanTimer = null;
|
|
1500
|
-
}
|
|
1501
|
-
}
|
|
1502
|
-
|
|
1503
|
-
private exhaustSessionBudget(state: SessionState): void {
|
|
1504
|
-
if (state.budgetExhausted) return;
|
|
1505
|
-
state.budgetExhausted = true;
|
|
1506
|
-
this.stopWatching(state);
|
|
1507
|
-
this.log("warn", "OmniRush session budget exhausted — collection stopped for this session", {
|
|
1508
|
-
sessionId: state.id,
|
|
1509
|
-
budgetBytes: this.sessionBudgetBytes,
|
|
1510
|
-
sentBytes: state.sentBytes,
|
|
1511
|
-
});
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
/** Run the debounced change-snapshot upload immediately. */
|
|
1515
|
-
private flushChangeSnapshot(state: SessionState): void {
|
|
1516
|
-
if (state.finished || state.budgetExhausted) return;
|
|
1517
|
-
if (state.changeTimer) {
|
|
1518
|
-
clearTimeout(state.changeTimer);
|
|
1519
|
-
state.changeTimer = null;
|
|
1520
|
-
}
|
|
1521
|
-
this.enqueue(state, async () => {
|
|
1522
|
-
if (state.budgetExhausted || state.finished) return;
|
|
1523
|
-
// The JOURNAL is what holds unshipped change records: a scan
|
|
1524
|
-
// captures edits and advances lastSignature in the same breath,
|
|
1525
|
-
// so by the time this debounced flush runs (it can queue behind a
|
|
1526
|
-
// long multi-part baseline) the workspace signature often equals
|
|
1527
|
-
// lastSignature again and a signature-only guard would skip
|
|
1528
|
-
// forever, stranding the journal until session end. Ship whenever
|
|
1529
|
-
// unacked journal records exist; the signature is only the
|
|
1530
|
-
// trigger for captures that have not been journalled yet.
|
|
1531
|
-
const unackedJournal = state.journalAppendedBytes > state.journalAckOffset;
|
|
1532
|
-
const signature = await workspaceSignature(state.root);
|
|
1533
|
-
if (!unackedJournal && (!signature || signature === state.lastSignature)) return;
|
|
1534
|
-
state.lastSignature = signature;
|
|
1535
|
-
await this.uploadWorkspace(state, "change");
|
|
1536
|
-
});
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
private scheduleChange(state: SessionState): void {
|
|
1540
|
-
if (state.finished || state.budgetExhausted) return;
|
|
1541
|
-
if (state.changeTimer) clearTimeout(state.changeTimer);
|
|
1542
|
-
state.changeTimer = setTimeout(() => {
|
|
1543
|
-
state.changeTimer = null;
|
|
1544
|
-
this.flushChangeSnapshot(state);
|
|
1545
|
-
}, this.changeDebounceMs);
|
|
1546
|
-
state.changeTimer.unref?.();
|
|
1547
|
-
}
|
|
1548
|
-
|
|
1549
|
-
private async captureChangedPath(
|
|
1550
|
-
state: SessionState,
|
|
1551
|
-
filename: string,
|
|
1552
|
-
options?: { force?: boolean },
|
|
1553
|
-
): Promise<void> {
|
|
1554
|
-
const absolute = resolve(state.root, filename);
|
|
1555
|
-
const path = portablePath(state.root, absolute);
|
|
1556
|
-
if (!path || path.startsWith("../") || isCollectorPathDenied(path)) return;
|
|
1557
|
-
const records: ChangeJournalEntry[] = [];
|
|
1558
|
-
let entry: ChangeJournalEntry;
|
|
1559
|
-
try {
|
|
1560
|
-
const file = await lstat(absolute);
|
|
1561
|
-
state.knownPaths.add(path);
|
|
1562
|
-
if (file.isSymbolicLink()) {
|
|
1563
|
-
// Symlink record: the target as text content with the symlink
|
|
1564
|
-
// encoding — replay recreates the actual link from it.
|
|
1565
|
-
const target = await readlink(absolute);
|
|
1566
|
-
entry = {
|
|
1567
|
-
path,
|
|
1568
|
-
at: new Date().toISOString(),
|
|
1569
|
-
status: "present",
|
|
1570
|
-
content: target,
|
|
1571
|
-
encoding: "symlink",
|
|
1572
|
-
};
|
|
1573
|
-
} else if (!file.isFile()) {
|
|
1574
|
-
return;
|
|
1575
|
-
} else {
|
|
1576
|
-
const at = new Date().toISOString();
|
|
1577
|
-
const mode = file.mode & 0o777;
|
|
1578
|
-
const modeField = mode ? { mode: mode.toString(8) } : {};
|
|
1579
|
-
// STREAMED capture: each chunk becomes a journal record the
|
|
1580
|
-
// moment it exists — a big file is never held in memory whole
|
|
1581
|
-
// (issue #6). One record per <= MAX_SLICE_BYTES slice; slice
|
|
1582
|
-
// metadata rides the records only when reassembly is needed.
|
|
1583
|
-
let sliceCount = 0;
|
|
1584
|
-
for await (const chunk of iterateContentChunks(absolute, file.size)) {
|
|
1585
|
-
for (const content of sliceString(chunk.content)) {
|
|
1586
|
-
sliceCount += 1;
|
|
1587
|
-
records.push({
|
|
1588
|
-
path,
|
|
1589
|
-
at,
|
|
1590
|
-
status: "present",
|
|
1591
|
-
content,
|
|
1592
|
-
...(chunk.encoding ? { encoding: chunk.encoding } : {}),
|
|
1593
|
-
...modeField,
|
|
1594
|
-
});
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
if (sliceCount > 1) {
|
|
1598
|
-
records.forEach((record, index) => {
|
|
1599
|
-
record.slice = {
|
|
1600
|
-
index,
|
|
1601
|
-
total: sliceCount,
|
|
1602
|
-
bytes: Buffer.byteLength(record.content ?? ""),
|
|
1603
|
-
sha256: sha256Hex(record.content ?? ""),
|
|
1604
|
-
};
|
|
1605
|
-
});
|
|
1606
|
-
}
|
|
1607
|
-
if (sliceCount === 1) {
|
|
1608
|
-
entry = records.pop()!;
|
|
1609
|
-
} else {
|
|
1610
|
-
await this.appendJournalRecords(state, records);
|
|
1611
|
-
return;
|
|
1612
|
-
}
|
|
1613
|
-
}
|
|
1614
|
-
} catch (error) {
|
|
1615
|
-
// Only a real disappearance is a deletion. Any other lstat failure
|
|
1616
|
-
// (EPERM/EBUSY/... under heavy watcher load) must not fabricate a
|
|
1617
|
-
// deleted record for a file that still exists — the fallback scan
|
|
1618
|
-
// and the end snapshot re-check those paths.
|
|
1619
|
-
if ((error as any)?.code !== "ENOENT") return;
|
|
1620
|
-
// Only paths the baseline (or a captured change) saw can be
|
|
1621
|
-
// deleted inside the workspace; FSEvents sibling/parent leaks
|
|
1622
|
-
// reference foreign paths that must not fabricate deletions.
|
|
1623
|
-
if (!state.knownPaths.has(path)) return;
|
|
1624
|
-
entry = { path, at: new Date().toISOString(), status: "deleted" };
|
|
1625
|
-
}
|
|
1626
|
-
// Consecutive-duplicate suppression: macOS FSEvents can deliver a
|
|
1627
|
-
// late event for a path that was also captured directly, and an
|
|
1628
|
-
// identical re-capture adds upload fat without a state change. The
|
|
1629
|
-
// replay result is identical either way. The last signature per
|
|
1630
|
-
// path lives in a tiny in-memory map (path strings only — content
|
|
1631
|
-
// is already on disk in the journal).
|
|
1632
|
-
const journalSignature = (record: ChangeJournalEntry) =>
|
|
1633
|
-
JSON.stringify([record.status, record.content, record.encoding, record.mode]);
|
|
1634
|
-
const pendingRecords = [entry, ...records];
|
|
1635
|
-
const accepted: ChangeJournalEntry[] = [];
|
|
1636
|
-
for (const record of pendingRecords) {
|
|
1637
|
-
const signature = journalSignature(record);
|
|
1638
|
-
if (state.journalLastByPath.get(record.path) === signature) continue;
|
|
1639
|
-
state.journalLastByPath.set(record.path, signature);
|
|
1640
|
-
accepted.push(record);
|
|
1641
|
-
}
|
|
1642
|
-
await this.appendJournalRecords(state, accepted);
|
|
1643
|
-
}
|
|
1644
|
-
|
|
1645
|
-
/**
|
|
1646
|
-
* APPEND records to the on-disk journal (ndjson — one JSON record per
|
|
1647
|
-
* line, 0600). Returns immediately when the capture hit the soft disk
|
|
1648
|
-
* cap (a backend down for a very long time must stop CAPTURING loudly
|
|
1649
|
-
* rather than fill the disk or RAM — the end snapshot still ships the
|
|
1650
|
-
* final tree). Serialized through the per-session append chain.
|
|
1651
|
-
*/
|
|
1652
|
-
private appendJournalRecords(state: SessionState, records: ChangeJournalEntry[]): Promise<void> {
|
|
1653
|
-
if (!state.journalPath || records.length === 0) return Promise.resolve();
|
|
1654
|
-
if (state.journalCaptureStopped) return Promise.resolve();
|
|
1655
|
-
const unacked = state.journalAppendedBytes - state.journalAckOffset;
|
|
1656
|
-
const capMb = Number(process.env.OMNIRUSH_JOURNAL_DISK_CAP_MB);
|
|
1657
|
-
const capBytes = Number.isFinite(capMb) && capMb > 0 ? capMb * 1024 * 1024 : JOURNAL_DISK_CAP_BYTES;
|
|
1658
|
-
if (unacked > capBytes) {
|
|
1659
|
-
state.journalCaptureStopped = true;
|
|
1660
|
-
this.log("warn", "OmniRush change journal hit the disk cap — change capture stopped for this session", {
|
|
1661
|
-
sessionId: state.id,
|
|
1662
|
-
capBytes: capBytes,
|
|
1663
|
-
unackedBytes: unacked,
|
|
1664
|
-
});
|
|
1665
|
-
state.trace.push({
|
|
1666
|
-
at: new Date().toISOString(),
|
|
1667
|
-
type: "journal.capture_stopped",
|
|
1668
|
-
data: { cap_bytes: capBytes, unacked_bytes: unacked },
|
|
1669
|
-
});
|
|
1670
|
-
return Promise.resolve();
|
|
1671
|
-
}
|
|
1672
|
-
const lines = records.map((record) => JSON.stringify(record) + "\n").join("");
|
|
1673
|
-
const bytes = Buffer.byteLength(lines);
|
|
1674
|
-
state.journalAppendTail = state.journalAppendTail
|
|
1675
|
-
.catch(() => undefined)
|
|
1676
|
-
.then(async () => {
|
|
1677
|
-
const { appendFile, mkdir } = await import("node:fs/promises");
|
|
1678
|
-
await mkdir(dirname(state.journalPath!), { recursive: true, mode: 0o700 });
|
|
1679
|
-
await appendFile(state.journalPath!, lines, { encoding: "utf8", mode: 0o600 });
|
|
1680
|
-
state.journalAppendedBytes += bytes;
|
|
1681
|
-
});
|
|
1682
|
-
return state.journalAppendTail;
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
/**
|
|
1686
|
-
* Stream the journal's unacked records (from journalAckOffset to the
|
|
1687
|
-
* upload-start EOF) as parsed entries, each tagged with its raw line
|
|
1688
|
-
* bytes via a NON-enumerable property so the changes.json sidecar
|
|
1689
|
-
* serialization never sees it. Records appended while the upload runs
|
|
1690
|
-
* land beyond the read window and simply ship with the next snapshot.
|
|
1691
|
-
*/
|
|
1692
|
-
private async *readJournalRecords(state: SessionState): AsyncGenerator<ChangeJournalEntry> {
|
|
1693
|
-
if (!state.journalPath || state.journalAckOffset >= state.journalUploadEof) return;
|
|
1694
|
-
const { createReadStream } = await import("node:fs");
|
|
1695
|
-
const readline = await import("node:readline");
|
|
1696
|
-
const stream = createReadStream(state.journalPath, {
|
|
1697
|
-
start: state.journalAckOffset,
|
|
1698
|
-
end: state.journalUploadEof! - 1,
|
|
1699
|
-
encoding: "utf8",
|
|
1700
|
-
});
|
|
1701
|
-
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
1702
|
-
for await (const line of rl) {
|
|
1703
|
-
if (!line.trim()) continue;
|
|
1704
|
-
try {
|
|
1705
|
-
const record = JSON.parse(line) as ChangeJournalEntry;
|
|
1706
|
-
Object.defineProperty(record, RAW_BYTES_FIELD, {
|
|
1707
|
-
value: Buffer.byteLength(line) + 1,
|
|
1708
|
-
enumerable: false,
|
|
1709
|
-
configurable: true,
|
|
1710
|
-
});
|
|
1711
|
-
yield record;
|
|
1712
|
-
} catch {
|
|
1713
|
-
this.log("warn", "OmniRush journal line was unreadable — skipped", { sessionId: state.id });
|
|
1714
|
-
}
|
|
1715
|
-
}
|
|
1716
|
-
}
|
|
1717
|
-
|
|
1718
|
-
/**
|
|
1719
|
-
* Upload the journal's unacked records as change snapshots, then
|
|
1720
|
-
* compact: everything acked is removed from the file; records appended
|
|
1721
|
-
* during the upload (past journalUploadEof) survive untouched.
|
|
1722
|
-
*/
|
|
1723
|
-
private async uploadChangeJournal(state: SessionState): Promise<void> {
|
|
1724
|
-
if (!state.journalPath) return;
|
|
1725
|
-
let fileSize = 0;
|
|
1726
|
-
try {
|
|
1727
|
-
fileSize = (await lstat(state.journalPath)).size;
|
|
1728
|
-
} catch {
|
|
1729
|
-
return; // no journal yet — nothing captured
|
|
1730
|
-
}
|
|
1731
|
-
if (fileSize <= state.journalAckOffset) return;
|
|
1732
|
-
const ackStart = state.journalAckOffset;
|
|
1733
|
-
state.journalUploadEof = fileSize;
|
|
1734
|
-
try {
|
|
1735
|
-
await this.uploadChangeParts(state, this.readJournalRecords(state), {
|
|
1736
|
-
onAck: (batch) => {
|
|
1737
|
-
for (const record of batch) {
|
|
1738
|
-
const raw = (record as Record<symbol, unknown>)[RAW_BYTES_FIELD];
|
|
1739
|
-
if (typeof raw === "number") state.journalAckOffset += raw;
|
|
1740
|
-
}
|
|
1741
|
-
},
|
|
1742
|
-
});
|
|
1743
|
-
} finally {
|
|
1744
|
-
state.journalUploadEof = null;
|
|
1745
|
-
}
|
|
1746
|
-
if (state.journalAckOffset <= ackStart) return; // nothing acked — keep everything
|
|
1747
|
-
await this.compactJournal(state, state.journalAckOffset);
|
|
1748
|
-
state.journalAckOffset = 0;
|
|
1749
|
-
}
|
|
1750
|
-
|
|
1751
|
-
/** Rewrite the journal without its first `dropBytes` bytes. */
|
|
1752
|
-
private async compactJournal(state: SessionState, dropBytes: number): Promise<void> {
|
|
1753
|
-
if (!state.journalPath) return;
|
|
1754
|
-
const { createReadStream, createWriteStream } = await import("node:fs");
|
|
1755
|
-
const { rename } = await import("node:fs/promises");
|
|
1756
|
-
const tmpPath = `${state.journalPath}.compact`;
|
|
1757
|
-
await new Promise<void>((resolvePromise, rejectPromise) => {
|
|
1758
|
-
const input = createReadStream(state.journalPath!, { start: dropBytes, encoding: "utf8" });
|
|
1759
|
-
const output = createWriteStream(tmpPath, { encoding: "utf8", mode: 0o600 });
|
|
1760
|
-
input.on("error", rejectPromise);
|
|
1761
|
-
output.on("error", rejectPromise);
|
|
1762
|
-
output.on("finish", () => resolvePromise());
|
|
1763
|
-
input.pipe(output);
|
|
1764
|
-
});
|
|
1765
|
-
await rename(tmpPath, state.journalPath);
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
/** Upload unacked journal records (best effort) and remove the file. */
|
|
1769
|
-
private async teardownJournal(state: SessionState): Promise<void> {
|
|
1770
|
-
try {
|
|
1771
|
-
await this.uploadChangeJournal(state);
|
|
1772
|
-
} catch {
|
|
1773
|
-
/* best effort — the end snapshot still ships the final tree */
|
|
1774
|
-
}
|
|
1775
|
-
if (state.journalPath) {
|
|
1776
|
-
const { rm } = await import("node:fs/promises");
|
|
1777
|
-
await rm(state.journalPath, { force: true }).catch(() => undefined);
|
|
1778
|
-
}
|
|
1779
|
-
}
|
|
1780
|
-
|
|
1781
|
-
private enqueue(state: SessionState, operation: () => Promise<void>): void {
|
|
1782
|
-
state.tail = state.tail.then(operation).catch((error: unknown) => {
|
|
1783
|
-
this.log("warn", "OmniRush collection operation failed", {
|
|
1784
|
-
sessionId: state.id,
|
|
1785
|
-
error: error instanceof Error ? error.message : "unknown",
|
|
1786
|
-
});
|
|
1787
|
-
});
|
|
1788
|
-
}
|
|
1789
|
-
|
|
1790
|
-
private async uploadWorkspace(state: SessionState, type: "start" | "change" | "end"): Promise<void> {
|
|
1791
|
-
if (state.budgetExhausted) return;
|
|
1792
|
-
if (type === "change") {
|
|
1793
|
-
// Change snapshots carry ONLY the touched files — streamed from
|
|
1794
|
-
// the on-disk journal, acked per part, then compacted. Never a
|
|
1795
|
-
// full re-enumeration of the tree, never held in memory.
|
|
1796
|
-
await this.uploadChangeJournal(state);
|
|
1797
|
-
} else {
|
|
1798
|
-
await this.uploadTreeParts(state, type);
|
|
1799
|
-
}
|
|
1800
|
-
state.lastSignature = await workspaceSignature(state.root);
|
|
1801
|
-
}
|
|
1802
|
-
|
|
1803
|
-
private async workspaceMetadataFile(state: SessionState): Promise<CollectorFile> {
|
|
1804
|
-
return {
|
|
1805
|
-
path: `${AGENT_DIR}/workspace.json`,
|
|
1806
|
-
content: JSON.stringify({
|
|
1807
|
-
workspace_id: state.workspaceId,
|
|
1808
|
-
session_id: state.id,
|
|
1809
|
-
session_segment: state.segment,
|
|
1810
|
-
session_resumed: state.resumed,
|
|
1811
|
-
root_name: state.root.split(sep).filter(Boolean).at(-1) ?? "workspace",
|
|
1812
|
-
git: await gitMetadata(state.root),
|
|
1813
|
-
}),
|
|
1814
|
-
};
|
|
1815
|
-
}
|
|
1816
|
-
|
|
1817
|
-
/** Full-tree snapshot ("start" | "end"), streamed into ordered parts.
|
|
1818
|
-
* A start snapshot resumed after an interruption skips the paths the
|
|
1819
|
-
* previous run already shipped (`baselineResumePath` cursor — the
|
|
1820
|
-
* first unacked part onward); the end snapshot always ships whole. */
|
|
1821
|
-
private async uploadTreeParts(state: SessionState, snapshotType: "start" | "end"): Promise<void> {
|
|
1822
|
-
const paths = (await listWorkspaceFiles(state.root)).sort(comparePaths);
|
|
1823
|
-
const metadata = await this.workspaceMetadataFile(state);
|
|
1824
|
-
let metadataSent = false;
|
|
1825
|
-
const resumeFrom = snapshotType === "start" ? state.baselineResumePath : null;
|
|
1826
|
-
const measure = async (batch: CollectorFile[]) => {
|
|
1827
|
-
const files = metadataSent ? batch : [metadata, ...batch];
|
|
1828
|
-
const payload = buildEnvelopePayload(this.sequenceState(state), snapshotType, files);
|
|
1829
|
-
return { files, payload, compressed: await compressZstd(payload) };
|
|
1830
|
-
};
|
|
1831
|
-
await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths, resumeFrom), {
|
|
1832
|
-
sizeOf: (file) => Buffer.byteLength(file.content ?? "") + file.path.length + 64,
|
|
1833
|
-
measure,
|
|
1834
|
-
upload: async (measured) => {
|
|
1835
|
-
const ok = await this.uploadEnvelope(state, snapshotType, measured.files, measured);
|
|
1836
|
-
if (ok) {
|
|
1837
|
-
metadataSent = true;
|
|
1838
|
-
if (snapshotType === "start") {
|
|
1839
|
-
// Advance the resume cursor to the last path of the acked
|
|
1840
|
-
// part, so an interrupted run resumes at the first
|
|
1841
|
-
// unacked part instead of re-burning the prefix.
|
|
1842
|
-
const lastPath = measured.files[measured.files.length - 1]?.path;
|
|
1843
|
-
if (typeof lastPath === "string") {
|
|
1844
|
-
state.baselineResumePath = lastPath;
|
|
1845
|
-
await this.persistSession(state).catch(() => undefined);
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1848
|
-
}
|
|
1849
|
-
return ok;
|
|
1850
|
-
},
|
|
1851
|
-
drop: (file) => {
|
|
1852
|
-
this.log("warn", "OmniRush file exceeds the upload rail and cannot be shipped as a single part; skipped", {
|
|
1853
|
-
sessionId: state.id,
|
|
1854
|
-
path: file.path,
|
|
1855
|
-
bytes: Buffer.byteLength(file.content ?? ""),
|
|
1856
|
-
});
|
|
1857
|
-
state.trace.push({
|
|
1858
|
-
at: new Date().toISOString(),
|
|
1859
|
-
type: "file.unshippable",
|
|
1860
|
-
data: { path: file.path, bytes: Buffer.byteLength(file.content ?? "") },
|
|
1861
|
-
});
|
|
1862
|
-
},
|
|
1863
|
-
}, { yieldToTraceLane: true });
|
|
1864
|
-
}
|
|
1865
|
-
|
|
1866
|
-
private async *iterTreeFiles(
|
|
1867
|
-
state: SessionState,
|
|
1868
|
-
paths: string[],
|
|
1869
|
-
resumeFrom?: string | null,
|
|
1870
|
-
): AsyncGenerator<CollectorFile> {
|
|
1871
|
-
// The cursor can name a slice or manifest virtual path
|
|
1872
|
-
// (`<path>.__agent_partN`, `<path>.__agent_manifest.json`) when the
|
|
1873
|
-
// acked part ended mid-file. Comparing a virtual cursor against the
|
|
1874
|
-
// real path list would never match and `path <= cursor` would
|
|
1875
|
-
// swallow the whole base file (silent data loss on resume), so
|
|
1876
|
-
// paths before the cursor's base file are skipped outright and the
|
|
1877
|
-
// base file's own entries are filtered against the cursor.
|
|
1878
|
-
const cursor = resumeFrom ?? null;
|
|
1879
|
-
const cursorBase = cursor
|
|
1880
|
-
? cursor.replace(/\.__agent(?:_part\d+|_manifest\.json)$/, "")
|
|
1881
|
-
: null;
|
|
1882
|
-
for (const path of paths) {
|
|
1883
|
-
if (state.budgetExhausted) return;
|
|
1884
|
-
if (cursorBase) {
|
|
1885
|
-
const order = comparePaths(path, cursorBase);
|
|
1886
|
-
if (order < 0) continue;
|
|
1887
|
-
if (order === 0) {
|
|
1888
|
-
for await (const entry of iterateWorkspaceEntries(state.root, path)) {
|
|
1889
|
-
if (entry.path && comparePaths(entry.path, cursor!) <= 0) continue;
|
|
1890
|
-
yield entry;
|
|
1891
|
-
}
|
|
1892
|
-
continue;
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
yield* iterateWorkspaceEntries(state.root, path);
|
|
1896
|
-
}
|
|
1897
|
-
}
|
|
1898
|
-
|
|
1899
|
-
/**
|
|
1900
|
-
* Change snapshot: ordered journal slices. Each part carries the
|
|
1901
|
-
* touched files for its slice (deduped last-wins, sorted by path) plus
|
|
1902
|
-
* the __agent__/changes.json sidecar holding the slice's ordered
|
|
1903
|
-
* records — replaying start + sidecars in sequence order reproduces the
|
|
1904
|
-
* end tree.
|
|
1905
|
-
*/
|
|
1906
|
-
private async uploadChangeParts(
|
|
1907
|
-
state: SessionState,
|
|
1908
|
-
units: Iterable<ChangeJournalEntry> | AsyncIterable<ChangeJournalEntry>,
|
|
1909
|
-
opts: { onAck?: (batch: ChangeJournalEntry[]) => void } = {},
|
|
1910
|
-
): Promise<ChangeJournalEntry[][]> {
|
|
1911
|
-
const uploadedSlices: ChangeJournalEntry[][] = [];
|
|
1912
|
-
const filesForSlice = (entries: ChangeJournalEntry[]): CollectorFile[] => {
|
|
1913
|
-
const lastByPath = new Map<string, CollectorFile>();
|
|
1914
|
-
const slicedByPath = new Map<string, ChangeJournalEntry[]>();
|
|
1915
|
-
for (const entry of entries) {
|
|
1916
|
-
if (entry.status !== "present") continue;
|
|
1917
|
-
if (entry.slice) {
|
|
1918
|
-
const group = slicedByPath.get(entry.path) ?? [];
|
|
1919
|
-
group.push(entry);
|
|
1920
|
-
slicedByPath.set(entry.path, group);
|
|
1921
|
-
} else if (entry.content !== undefined) {
|
|
1922
|
-
lastByPath.set(entry.path, {
|
|
1923
|
-
path: entry.path,
|
|
1924
|
-
content: entry.content,
|
|
1925
|
-
...(entry.encoding ? { encoding: entry.encoding } : {}),
|
|
1926
|
-
...(entry.mode ? { mode: entry.mode } : {}),
|
|
1927
|
-
});
|
|
1928
|
-
}
|
|
1929
|
-
}
|
|
1930
|
-
const files = [...lastByPath.values()];
|
|
1931
|
-
for (const [logicalPath, records] of slicedByPath) {
|
|
1932
|
-
records.sort((left, right) => (left.slice?.index ?? 0) - (right.slice?.index ?? 0));
|
|
1933
|
-
const encoding = records.find((record) => record.encoding)?.encoding;
|
|
1934
|
-
const mode = records.find((record) => record.mode)?.mode;
|
|
1935
|
-
for (const record of records) {
|
|
1936
|
-
files.push({
|
|
1937
|
-
path: slicePartPath(logicalPath, record.slice!.index),
|
|
1938
|
-
content: record.content,
|
|
1939
|
-
...(encoding ? { encoding } : {}),
|
|
1940
|
-
});
|
|
1941
|
-
}
|
|
1942
|
-
files.push({
|
|
1943
|
-
path: sliceManifestPath(logicalPath),
|
|
1944
|
-
content: JSON.stringify(sliceManifest(
|
|
1945
|
-
{
|
|
1946
|
-
path: logicalPath,
|
|
1947
|
-
...(encoding ? { encoding } : {}),
|
|
1948
|
-
...(mode ? { mode } : {}),
|
|
1949
|
-
},
|
|
1950
|
-
records.map((record) => ({
|
|
1951
|
-
index: record.slice!.index,
|
|
1952
|
-
bytes: record.slice!.bytes,
|
|
1953
|
-
sha256: record.slice!.sha256,
|
|
1954
|
-
})),
|
|
1955
|
-
records[0].slice!.total,
|
|
1956
|
-
)),
|
|
1957
|
-
});
|
|
1958
|
-
}
|
|
1959
|
-
files.sort((left, right) => comparePaths(left.path, right.path));
|
|
1960
|
-
files.push({
|
|
1961
|
-
path: `${AGENT_DIR}/changes.json`,
|
|
1962
|
-
content: JSON.stringify({ schema_version: 1, session_id: state.id, entries }),
|
|
1963
|
-
});
|
|
1964
|
-
return files;
|
|
1965
|
-
};
|
|
1966
|
-
const bySequence = (capturedAt?: string) => this.sequenceState(state, capturedAt);
|
|
1967
|
-
await this.packAndUpload(state, "change", units, {
|
|
1968
|
-
sizeOf: (entry) =>
|
|
1969
|
-
Buffer.byteLength(entry.content ?? "") + entry.path.length + 96,
|
|
1970
|
-
measure: async (batch) => {
|
|
1971
|
-
const files = filesForSlice(batch);
|
|
1972
|
-
const capturedAt = batch.find((entry) => typeof entry.at === "string")?.at;
|
|
1973
|
-
const payload = buildEnvelopePayload(bySequence(capturedAt), "change", files);
|
|
1974
|
-
return { files, payload, compressed: await compressZstd(payload) };
|
|
1975
|
-
},
|
|
1976
|
-
upload: async (measured, batch) => {
|
|
1977
|
-
const ok = await this.uploadEnvelope(state, "change", measured.files, measured);
|
|
1978
|
-
if (ok) {
|
|
1979
|
-
uploadedSlices.push(batch);
|
|
1980
|
-
opts.onAck?.(batch);
|
|
1981
|
-
}
|
|
1982
|
-
return ok;
|
|
1983
|
-
},
|
|
1984
|
-
drop: (entry) => {
|
|
1985
|
-
this.log("warn", "OmniRush change record exceeds the upload limits and cannot be shipped; skipped", {
|
|
1986
|
-
sessionId: state.id,
|
|
1987
|
-
path: entry.path,
|
|
1988
|
-
bytes: Buffer.byteLength(entry.content ?? ""),
|
|
1989
|
-
});
|
|
1990
|
-
state.trace.push({
|
|
1991
|
-
at: new Date().toISOString(),
|
|
1992
|
-
type: "file.unshippable",
|
|
1993
|
-
data: { path: entry.path, reason: "change_record_over_limit" },
|
|
1994
|
-
});
|
|
1995
|
-
},
|
|
1996
|
-
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX, yieldToTraceLane: true });
|
|
1997
|
-
return uploadedSlices;
|
|
1998
|
-
}
|
|
1999
|
-
|
|
2000
|
-
/**
|
|
2001
|
-
* Streaming packer shared by tree and change uploads: accumulate units
|
|
2002
|
-
* in order, compress-check every partCheckBytes of uncompressed growth,
|
|
2003
|
-
* flush parts targeting partCompressedBytes, and respect the hard rail
|
|
2004
|
-
* (a lone unit over the rail is skipped via hooks.drop — never split).
|
|
2005
|
-
* fit() mutates the batch it is given: the returned `measured` always
|
|
2006
|
-
* corresponds to the (mutated) batch passed to hooks.upload, and popped
|
|
2007
|
-
* units come back as `carry` to lead the next part. hooks.upload must
|
|
2008
|
-
* persist sequence bookkeeping via uploadEnvelope.
|
|
2009
|
-
*
|
|
2010
|
-
* `yieldToTraceLane` (baseline/change/end packs): before each part
|
|
2011
|
-
* upload, wait while trace work is pending or running — a trace part
|
|
2012
|
-
* then never queues behind more than the one baseline part already in
|
|
2013
|
-
* flight. Trace packs must NOT set it (they ARE the priority lane).
|
|
2014
|
-
*/
|
|
2015
|
-
private async packAndUpload<U>(
|
|
2016
|
-
state: SessionState,
|
|
2017
|
-
snapshotType: SnapshotType,
|
|
2018
|
-
units: Iterable<U> | AsyncIterable<U>,
|
|
2019
|
-
hooks: {
|
|
2020
|
-
sizeOf: (unit: U) => number;
|
|
2021
|
-
measure: (batch: U[]) => Promise<{ files: CollectorFile[]; payload: Buffer; compressed: Buffer }>;
|
|
2022
|
-
upload: (measured: { files: CollectorFile[]; payload: Buffer; compressed: Buffer }, batch: U[]) => Promise<boolean>;
|
|
2023
|
-
drop?: (unit: U) => void;
|
|
2024
|
-
},
|
|
2025
|
-
opts: { checkBytes?: number; flushPlainBytes?: number; yieldToTraceLane?: boolean } = {},
|
|
2026
|
-
): Promise<void> {
|
|
2027
|
-
const checkBytes = Math.max(1024, opts.checkBytes ?? this.partCheckBytes);
|
|
2028
|
-
const flushPlainBytes = Math.max(checkBytes, opts.flushPlainBytes ?? PART_BATCH_UNCOMPRESSED_MAX);
|
|
2029
|
-
const hard = MAX_PART_COMPRESSED_HARD;
|
|
2030
|
-
const fit = async (batch: U[]) => {
|
|
2031
|
-
let measured = await hooks.measure(batch);
|
|
2032
|
-
const carry: U[] = [];
|
|
2033
|
-
// Multi-file parts split at the compressed target. A lone unit
|
|
2034
|
-
// (e.g. one slice entry) may use the body headroom up to the hard
|
|
2035
|
-
// rail; beyond that it is dropped loudly rather than split
|
|
2036
|
-
// mid-content.
|
|
2037
|
-
while (measured.compressed.length > this.partCompressedBytes && batch.length > 1) {
|
|
2038
|
-
carry.unshift(batch.pop()!);
|
|
2039
|
-
measured = await hooks.measure(batch);
|
|
2040
|
-
}
|
|
2041
|
-
if (measured.compressed.length > hard && batch.length === 1) {
|
|
2042
|
-
const lone = batch.pop()!;
|
|
2043
|
-
hooks.drop?.(lone);
|
|
2044
|
-
return { measured: batch.length > 0 ? await hooks.measure(batch) : null, carry };
|
|
2045
|
-
}
|
|
2046
|
-
return { measured, carry };
|
|
2047
|
-
};
|
|
2048
|
-
|
|
2049
|
-
let batch: U[] = [];
|
|
2050
|
-
let batchBytes = 0;
|
|
2051
|
-
let nextCheck = checkBytes;
|
|
2052
|
-
const resetBatch = (unitsToKeep: U[]) => {
|
|
2053
|
-
batch = unitsToKeep;
|
|
2054
|
-
batchBytes = batch.reduce((total, unit) => total + hooks.sizeOf(unit), 0);
|
|
2055
|
-
nextCheck = batchBytes + checkBytes;
|
|
2056
|
-
};
|
|
2057
|
-
const shouldFlush = (measured: { compressed: Buffer }) =>
|
|
2058
|
-
measured.compressed.length >= PART_READY_COMPRESSED_BYTES || batchBytes >= flushPlainBytes;
|
|
2059
|
-
|
|
2060
|
-
for await (const unit of units as AsyncIterable<U>) {
|
|
2061
|
-
if (state.budgetExhausted) return;
|
|
2062
|
-
batch.push(unit);
|
|
2063
|
-
batchBytes += hooks.sizeOf(unit);
|
|
2064
|
-
if (batchBytes < nextCheck) continue;
|
|
2065
|
-
const { measured, carry } = await fit(batch);
|
|
2066
|
-
if (measured && shouldFlush(measured)) {
|
|
2067
|
-
if (opts.yieldToTraceLane) await this.yieldToTrace(state);
|
|
2068
|
-
if (!(await hooks.upload(measured, batch))) return;
|
|
2069
|
-
resetBatch(carry);
|
|
2070
|
-
} else if (measured) {
|
|
2071
|
-
// Under the flush thresholds — keep accumulating (carry is empty
|
|
2072
|
-
// here: popping only happens above the flush levels).
|
|
2073
|
-
batch = [...batch, ...carry];
|
|
2074
|
-
nextCheck = batchBytes + checkBytes;
|
|
2075
|
-
} else {
|
|
2076
|
-
resetBatch(carry);
|
|
2077
|
-
}
|
|
2078
|
-
}
|
|
2079
|
-
while (batch.length > 0 && !state.budgetExhausted) {
|
|
2080
|
-
const { measured, carry } = await fit(batch);
|
|
2081
|
-
if (measured) {
|
|
2082
|
-
if (opts.yieldToTraceLane) await this.yieldToTrace(state);
|
|
2083
|
-
if (!(await hooks.upload(measured, batch))) return;
|
|
2084
|
-
}
|
|
2085
|
-
if (carry.length === 0) break;
|
|
2086
|
-
batch = carry;
|
|
2087
|
-
batchBytes = batch.reduce((total, unit) => total + hooks.sizeOf(unit), 0);
|
|
2088
|
-
}
|
|
2089
|
-
}
|
|
2090
|
-
|
|
2091
|
-
/** Chunked trace upload: event subsets per part, ordered by sequence. */
|
|
2092
|
-
private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {
|
|
2093
|
-
if (state.budgetExhausted || traceEvents.length === 0) {
|
|
2094
|
-
// No lifecycle events this round — transcript artifacts may still
|
|
2095
|
-
// be due (new or changed session files).
|
|
2096
|
-
await this.uploadSessionTranscripts(state);
|
|
2097
|
-
return;
|
|
2098
|
-
}
|
|
2099
|
-
const remaining = this.sessionBudgetBytes - state.sentBytes;
|
|
2100
|
-
if (remaining <= 1024) {
|
|
2101
|
-
this.exhaustSessionBudget(state);
|
|
2102
|
-
return;
|
|
2103
|
-
}
|
|
2104
|
-
// Budget wall: truncate (loudly, trace_truncated=true) only here.
|
|
2105
|
-
const selected = selectTraceEvents(traceEvents, remaining - 1024);
|
|
2106
|
-
const meta = {
|
|
2107
|
-
truncated: selected.truncated,
|
|
2108
|
-
dropped: selected.droppedCount,
|
|
2109
|
-
};
|
|
2110
|
-
if (selected.truncated) {
|
|
2111
|
-
this.log("warn", "OmniRush trace hit the session budget wall; older events dropped", {
|
|
2112
|
-
sessionId: state.id,
|
|
2113
|
-
droppedEventCount: selected.droppedCount,
|
|
2114
|
-
});
|
|
2115
|
-
}
|
|
2116
|
-
let partIndex = 0;
|
|
2117
|
-
const bySequence = () => this.sequenceState(state);
|
|
2118
|
-
await this.packAndUpload(state, "trace", selected.events, {
|
|
2119
|
-
sizeOf: (event) => Buffer.byteLength(JSON.stringify(event)) + 32,
|
|
2120
|
-
measure: async (batch) => {
|
|
2121
|
-
const content = JSON.stringify({
|
|
2122
|
-
schema_version: 1,
|
|
2123
|
-
schema: AGENT_TRACE_SCHEMA,
|
|
2124
|
-
session_id: state.id,
|
|
2125
|
-
session_segment: state.segment,
|
|
2126
|
-
session_resumed: state.resumed,
|
|
2127
|
-
trace_truncated: meta.truncated,
|
|
2128
|
-
dropped_event_count: meta.dropped,
|
|
2129
|
-
trace_part: { index: partIndex },
|
|
2130
|
-
events: batch,
|
|
2131
|
-
});
|
|
2132
|
-
const files = [{ path: `${AGENT_DIR}/trace.json`, content }];
|
|
2133
|
-
const payload = buildEnvelopePayload(bySequence(), "trace", files);
|
|
2134
|
-
return { files, payload, compressed: await compressZstd(payload) };
|
|
2135
|
-
},
|
|
2136
|
-
upload: async (measured) => {
|
|
2137
|
-
const ok = await this.uploadEnvelope(state, "trace", measured.files, measured);
|
|
2138
|
-
if (ok) partIndex += 1;
|
|
2139
|
-
return ok;
|
|
2140
|
-
},
|
|
2141
|
-
drop: (event) => {
|
|
2142
|
-
this.log("warn", "OmniRush trace event exceeds the upload limits; skipped", {
|
|
2143
|
-
sessionId: state.id,
|
|
2144
|
-
type: event.type,
|
|
2145
|
-
bytes: Buffer.byteLength(JSON.stringify(event)),
|
|
2146
|
-
});
|
|
2147
|
-
},
|
|
2148
|
-
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
|
|
2149
|
-
// Full-conversation transcript artifacts (trace format spec) ride
|
|
2150
|
-
// the same trace uploads — lifecycle events stay in trace.json, the
|
|
2151
|
-
// transcripts land as __agent__/traces/<session>.jsonl.
|
|
2152
|
-
await this.uploadSessionTranscripts(state);
|
|
2153
|
-
}
|
|
2154
|
-
|
|
2155
|
-
/**
|
|
2156
|
-
* The pi agent writes the FULL session transcript as JSONL under
|
|
2157
|
-
* <agentDir>/sessions/<cwd-slug>/<ts>_<session-id>.jsonl — one file
|
|
2158
|
-
* per run, SUBAGENTS as their own files in the same dir. Every file
|
|
2159
|
-
* (new or changed since the last capture) becomes its own trace
|
|
2160
|
-
* artifact (header + full conversation, redacted) uploaded as
|
|
2161
|
-
* __agent__/traces/<session>.jsonl and linked to this session via
|
|
2162
|
-
* parent_session_id. Detection state rides the persistent ledger.
|
|
2163
|
-
*/
|
|
2164
|
-
private async uploadSessionTranscripts(state: SessionState): Promise<void> {
|
|
2165
|
-
if (!this.agentDir || !state.id) return;
|
|
2166
|
-
const { readdir, stat } = await import("node:fs/promises");
|
|
2167
|
-
const sessionsRoot = join(this.agentDir, "sessions");
|
|
2168
|
-
let sessionDir: string | null = null;
|
|
2169
|
-
try {
|
|
2170
|
-
for (const entry of await readdir(sessionsRoot, { withFileTypes: true })) {
|
|
2171
|
-
if (!entry.isDirectory()) continue;
|
|
2172
|
-
const dir = join(sessionsRoot, entry.name);
|
|
2173
|
-
const files = await readdir(dir).catch(() => [] as string[]);
|
|
2174
|
-
if (files.some((f) => f.endsWith(`_${state.id}.jsonl`))) {
|
|
2175
|
-
sessionDir = dir;
|
|
2176
|
-
break;
|
|
2177
|
-
}
|
|
2178
|
-
}
|
|
2179
|
-
} catch {
|
|
2180
|
-
return; // no sessions dir — nothing to capture
|
|
2181
|
-
}
|
|
2182
|
-
if (!sessionDir) return;
|
|
2183
|
-
|
|
2184
|
-
const artifacts: Array<{ path: string; content: string; fileKey: string; size: number; mtimeMs: number }> = [];
|
|
2185
|
-
let files: string[] = [];
|
|
2186
|
-
try {
|
|
2187
|
-
files = (await readdir(sessionDir)).filter((f) => f.endsWith(".jsonl")).sort();
|
|
2188
|
-
} catch {
|
|
2189
|
-
return;
|
|
2190
|
-
}
|
|
2191
|
-
for (const file of files) {
|
|
2192
|
-
const filePath = join(sessionDir, file);
|
|
2193
|
-
let size = 0;
|
|
2194
|
-
let mtimeMs = 0;
|
|
2195
|
-
try {
|
|
2196
|
-
const stats = await stat(filePath);
|
|
2197
|
-
size = stats.size;
|
|
2198
|
-
mtimeMs = stats.mtimeMs;
|
|
2199
|
-
} catch {
|
|
2200
|
-
continue;
|
|
2201
|
-
}
|
|
2202
|
-
const known = state.transcripts[filePath];
|
|
2203
|
-
if (known && known.size === size && known.mtimeMs === mtimeMs) continue;
|
|
2204
|
-
let text: string;
|
|
2205
|
-
try {
|
|
2206
|
-
const { readFile } = await import("node:fs/promises");
|
|
2207
|
-
text = await readFile(filePath, "utf8");
|
|
2208
|
-
} catch {
|
|
2209
|
-
continue;
|
|
2210
|
-
}
|
|
2211
|
-
const parsed = parsePiSession(text);
|
|
2212
|
-
const artifactSessionId = parsed.sessionId ?? file.replace(/^\d{4}-\d{2}-\d{2}T[\d-]+Z_/, "").replace(/\.jsonl$/, "");
|
|
2213
|
-
if (!artifactSessionId || parsed.messageRecords.length === 0) {
|
|
2214
|
-
// Still mark empty/HEAD-only files seen so they are not retried.
|
|
2215
|
-
state.transcripts[filePath] = { size, mtimeMs };
|
|
2216
|
-
continue;
|
|
2217
|
-
}
|
|
2218
|
-
if (this.linkedParentSession && artifactSessionId !== state.id) {
|
|
2219
|
-
// Linked child: the parent's collector already ships the other
|
|
2220
|
-
// session files in this directory (linked to the parent) — a
|
|
2221
|
-
// second copy from the child adds nothing.
|
|
2222
|
-
continue;
|
|
2223
|
-
}
|
|
2224
|
-
const identity = await this.resolveIdentity();
|
|
2225
|
-
const artifact = buildTraceArtifact({
|
|
2226
|
-
sessionId: artifactSessionId,
|
|
2227
|
-
parentSessionId: this.linkedParentSession ?? state.id,
|
|
2228
|
-
userId: identity?.userId ?? null,
|
|
2229
|
-
clientId: this.clientId ?? null,
|
|
2230
|
-
agentId: "omnirush-cli",
|
|
2231
|
-
parsed,
|
|
2232
|
-
redact: (value) => redactCollectorText(value).text,
|
|
2233
|
-
});
|
|
2234
|
-
artifacts.push({
|
|
2235
|
-
path: `${AGENT_DIR}/traces/${artifactSessionId}.jsonl`,
|
|
2236
|
-
content: JSON.stringify(artifact),
|
|
2237
|
-
fileKey: filePath,
|
|
2238
|
-
size,
|
|
2239
|
-
mtimeMs,
|
|
2240
|
-
});
|
|
2241
|
-
}
|
|
2242
|
-
if (artifacts.length === 0) {
|
|
2243
|
-
await this.persistSession(state).catch(() => undefined);
|
|
2244
|
-
return;
|
|
2245
|
-
}
|
|
2246
|
-
const bySequence = () => this.sequenceState(state);
|
|
2247
|
-
await this.packAndUpload(state, "trace", artifacts, {
|
|
2248
|
-
sizeOf: (artifact) => Buffer.byteLength(artifact.content) + artifact.path.length + 64,
|
|
2249
|
-
measure: async (batch) => {
|
|
2250
|
-
const payload = buildEnvelopePayload(bySequence(), "trace", batch);
|
|
2251
|
-
return { files: batch, payload, compressed: await compressZstd(payload) };
|
|
2252
|
-
},
|
|
2253
|
-
upload: async (measured) => {
|
|
2254
|
-
// Mark a transcript seen only after its part is accepted — an
|
|
2255
|
-
// ack-before-success let a single 503 permanently drop the
|
|
2256
|
-
// transcript from every later flush (the ledger said captured).
|
|
2257
|
-
const ok = await this.uploadEnvelope(state, "trace", measured.files, measured);
|
|
2258
|
-
if (ok) {
|
|
2259
|
-
for (const artifact of measured.files) {
|
|
2260
|
-
state.transcripts[artifact.fileKey] = {
|
|
2261
|
-
size: artifact.size,
|
|
2262
|
-
mtimeMs: artifact.mtimeMs,
|
|
2263
|
-
};
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
return ok;
|
|
2267
|
-
},
|
|
2268
|
-
drop: (artifact) => {
|
|
2269
|
-
this.log("warn", "OmniRush transcript artifact exceeds the upload limits; skipped", {
|
|
2270
|
-
sessionId: state.id,
|
|
2271
|
-
path: artifact.path,
|
|
2272
|
-
bytes: Buffer.byteLength(artifact.content),
|
|
2273
|
-
});
|
|
2274
|
-
state.transcripts[artifact.fileKey] = {
|
|
2275
|
-
size: artifact.size,
|
|
2276
|
-
mtimeMs: artifact.mtimeMs,
|
|
2277
|
-
};
|
|
2278
|
-
},
|
|
2279
|
-
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
|
|
2280
|
-
await this.persistSession(state).catch(() => undefined);
|
|
2281
|
-
}
|
|
2282
|
-
|
|
2283
|
-
/** Best-effort /device/me identity, fetched once per collector. */
|
|
2284
|
-
private async resolveIdentity(): Promise<{ userId: string | null } | null> {
|
|
2285
|
-
if (this.identityTried) return this.identity;
|
|
2286
|
-
this.identityTried = true;
|
|
2287
|
-
if (!this.identityProvider) return null;
|
|
2288
|
-
try {
|
|
2289
|
-
this.identity = await this.identityProvider();
|
|
2290
|
-
} catch {
|
|
2291
|
-
this.identity = null;
|
|
2292
|
-
}
|
|
2293
|
-
return this.identity;
|
|
2294
|
-
}
|
|
2295
|
-
|
|
2296
|
-
/**
|
|
2297
|
-
* Send one envelope (one part). `precompressed` skips the rebuild +
|
|
2298
|
-
* recompress when the caller already measured this exact batch.
|
|
2299
|
-
*/
|
|
2300
|
-
private async uploadEnvelope(
|
|
2301
|
-
state: SessionState,
|
|
2302
|
-
snapshotType: SnapshotType,
|
|
2303
|
-
files: CollectorFile[],
|
|
2304
|
-
precompressed?: { payload: Buffer; compressed: Buffer },
|
|
2305
|
-
): Promise<boolean> {
|
|
2306
|
-
if (!this.collectUrl && !this.uploader) return false;
|
|
2307
|
-
if (state.budgetExhausted) return false;
|
|
2308
|
-
if (files.length === 0) {
|
|
2309
|
-
// A zero-file part carries no information and would loop forever
|
|
2310
|
-
// if a caller ever fed a packer an empty unit stream.
|
|
2311
|
-
this.log("warn", "OmniRush refused to upload an empty envelope", {
|
|
2312
|
-
sessionId: state.id,
|
|
2313
|
-
snapshotType,
|
|
2314
|
-
});
|
|
2315
|
-
return false;
|
|
2316
|
-
}
|
|
2317
|
-
const payload = precompressed?.payload ?? buildEnvelopePayload(state, snapshotType, files);
|
|
2318
|
-
if (state.sentBytes + payload.length > this.sessionBudgetBytes) {
|
|
2319
|
-
this.exhaustSessionBudget(state);
|
|
2320
|
-
return false;
|
|
2321
|
-
}
|
|
2322
|
-
const compressed = precompressed?.compressed ?? await compressZstd(payload);
|
|
2323
|
-
if (compressed.length > MAX_PART_COMPRESSED_HARD) {
|
|
2324
|
-
this.log("warn", "OmniRush part exceeded the backend body rail; not sent", {
|
|
2325
|
-
sessionId: state.id,
|
|
2326
|
-
snapshotType,
|
|
2327
|
-
compressedBytes: compressed.length,
|
|
2328
|
-
railBytes: MAX_PART_COMPRESSED_HARD,
|
|
2329
|
-
});
|
|
2330
|
-
return false;
|
|
2331
|
-
}
|
|
2332
|
-
// Two lanes (session tail + trace tail) funnel every envelope send
|
|
2333
|
-
// through this gate. Trace sends acquire with priority, so after the
|
|
2334
|
-
// part currently in flight finishes, the trace part goes next; the
|
|
2335
|
-
// gate is held per ATTEMPT, so a retrying baseline part cannot pin
|
|
2336
|
-
// the transport either.
|
|
2337
|
-
const priority = snapshotType === "trace";
|
|
2338
|
-
const gate = state.gate ?? this.defaultGate;
|
|
2339
|
-
const sendOnce = async () => {
|
|
2340
|
-
await gate.acquire(priority);
|
|
2341
|
-
try {
|
|
2342
|
-
return await (this.uploader
|
|
2343
|
-
? this.uploader(state.id, compressed)
|
|
2344
|
-
: this.fetcher(this.collectUrl!, {
|
|
2345
|
-
method: "POST",
|
|
2346
|
-
headers: {
|
|
2347
|
-
Authorization: `Bearer ${this.token}`,
|
|
2348
|
-
"Content-Type": "application/zstd",
|
|
2349
|
-
"X-Omnirush-Session-ID": state.id,
|
|
2350
|
-
},
|
|
2351
|
-
body: compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer,
|
|
2352
|
-
signal: AbortSignal.timeout(120_000),
|
|
2353
|
-
}));
|
|
2354
|
-
} finally {
|
|
2355
|
-
gate.release();
|
|
2356
|
-
}
|
|
2357
|
-
};
|
|
2358
|
-
// Shared retry layer: 429/5xx/network ride out backend deploys with
|
|
2359
|
-
// exponential backoff + jitter. 401 stays a protocol answer handled
|
|
2360
|
-
// inline (single-flight refresh, retry exactly once).
|
|
2361
|
-
const outcome = await withRetries(
|
|
2362
|
-
async () => {
|
|
2363
|
-
let response = await sendOnce();
|
|
2364
|
-
if (response.status === 401 && this.refresh) {
|
|
2365
|
-
// Broker pattern: single-flight refresh, retry exactly once.
|
|
2366
|
-
const rotated = await this.refresh(this.token).catch(() => null);
|
|
2367
|
-
if (rotated) {
|
|
2368
|
-
this.token = rotated;
|
|
2369
|
-
response = await sendOnce();
|
|
2370
|
-
}
|
|
2371
|
-
}
|
|
2372
|
-
return {
|
|
2373
|
-
response,
|
|
2374
|
-
retryAfterSec: Number(response?.headers?.get?.("retry-after")) || undefined,
|
|
2375
|
-
};
|
|
2376
|
-
},
|
|
2377
|
-
{
|
|
2378
|
-
attempts: retryAttempts(),
|
|
2379
|
-
isRetryable: ({ response }) => isRetryableStatus(response?.status),
|
|
2380
|
-
onRetry: ({ attempt, attempts, delayMs }) => {
|
|
2381
|
-
this.log("warn", "OmniRush upload unavailable — retrying", {
|
|
2382
|
-
sessionId: state.id,
|
|
2383
|
-
snapshotType,
|
|
2384
|
-
attempt: `${attempt + 1}/${attempts}`,
|
|
2385
|
-
delayMs,
|
|
2386
|
-
});
|
|
2387
|
-
},
|
|
2388
|
-
},
|
|
2389
|
-
);
|
|
2390
|
-
const response = outcome.result?.response;
|
|
2391
|
-
if (response?.ok) {
|
|
2392
|
-
// Drain/cancel the body so keep-alive sockets return to the pool —
|
|
2393
|
-
// an unconsumed body keeps a ref'd socket and hangs process exit.
|
|
2394
|
-
await response.body?.cancel().catch(() => undefined);
|
|
2395
|
-
}
|
|
2396
|
-
if (!response) {
|
|
2397
|
-
// Every attempt failed at the network layer.
|
|
2398
|
-
throw new Error(
|
|
2399
|
-
`collector upload failed: network error after ${outcome.attempts} attempt${outcome.attempts === 1 ? "" : "s"}` +
|
|
2400
|
-
` — ${outcome.error?.message ?? "unknown"} [ref ${outcome.ref}]` +
|
|
2401
|
-
` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
|
|
2402
|
-
);
|
|
2403
|
-
}
|
|
2404
|
-
if (!response.ok) {
|
|
2405
|
-
// Diagnosable from a tester's screenshot alone: status, context,
|
|
2406
|
-
// and (bounded) the server's error body. A 401 here means the
|
|
2407
|
-
// refresh-retry above also failed — the caller should re-login.
|
|
2408
|
-
const snippet = await WorkspaceCollector.errorBodySnippet(response);
|
|
2409
|
-
throw new Error(
|
|
2410
|
-
`collector upload failed: HTTP ${response.status}` +
|
|
2411
|
-
`${snippet ? ` — ${snippet}` : ""}` +
|
|
2412
|
-
` [ref ${outcome.ref}]` +
|
|
2413
|
-
` (session ${state.id}, ${snapshotType}, sequence ${state.sequence + 1})`,
|
|
2414
|
-
);
|
|
2415
|
-
}
|
|
2416
|
-
state.sentBytes += payload.length;
|
|
2417
|
-
state.sequence += 1;
|
|
2418
|
-
await this.persistSession(state).catch((error: unknown) => {
|
|
2419
|
-
this.log("warn", "OmniRush session ledger update failed", {
|
|
2420
|
-
sessionId: state.id,
|
|
2421
|
-
error: error instanceof Error ? error.message : "unknown",
|
|
2422
|
-
});
|
|
2423
|
-
});
|
|
2424
|
-
this.log("info", "OmniRush collection artifact uploaded", {
|
|
2425
|
-
sessionId: state.id,
|
|
2426
|
-
snapshotType,
|
|
2427
|
-
fileCount: files.length,
|
|
2428
|
-
compressedBytes: compressed.length,
|
|
2429
|
-
sequence: state.sequence,
|
|
2430
|
-
});
|
|
2431
|
-
return true;
|
|
2432
|
-
}
|
|
2433
|
-
}
|