omnirush 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/extensions/omnirush/agents-lib.ts +421 -0
- package/assets/extensions/omnirush/agents.ts +163 -0
- package/assets/extensions/omnirush/collector-lib.ts +323 -37
- package/assets/extensions/omnirush/collector.ts +24 -7
- package/assets/extensions/omnirush/index.ts +6 -0
- package/assets/extensions/omnirush/plan-lib.ts +113 -0
- package/assets/extensions/omnirush/plan.ts +136 -0
- package/assets/extensions/omnirush/trace-format.ts +17 -0
- package/assets/extensions/omnirush/webfetch-lib.ts +241 -0
- package/assets/extensions/omnirush/webfetch.ts +77 -0
- package/package.json +1 -1
- package/scripts/build-all-packages.py +0 -3
|
@@ -47,7 +47,9 @@
|
|
|
47
47
|
import { createHash } from "node:crypto";
|
|
48
48
|
import { execFile } from "node:child_process";
|
|
49
49
|
|
|
50
|
+
import { readFileSync } from "node:fs";
|
|
50
51
|
import { lstat, mkdir, readFile, readdir, readlink, writeFile } from "node:fs/promises";
|
|
52
|
+
import os from "node:os";
|
|
51
53
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
52
54
|
import { promisify } from "node:util";
|
|
53
55
|
import { zstdCompress as zstdCompressCb } from "node:zlib";
|
|
@@ -124,6 +126,73 @@ export const TRACE_RELIEF_EVENTS = 2_000;
|
|
|
124
126
|
export const AGENT_DIR = "__agent__";
|
|
125
127
|
export const AGENT_TRACE_SCHEMA = "agent.trace.v1";
|
|
126
128
|
|
|
129
|
+
let cachedClientVersion: string | null | undefined;
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* The CLI's package.json version, resolved once relative to this module
|
|
133
|
+
* (repo checkout and installed npm layout both keep package.json three
|
|
134
|
+
* levels up). Empty when it cannot be read — the envelope field is
|
|
135
|
+
* optional and the backend tolerates its absence.
|
|
136
|
+
*/
|
|
137
|
+
export function detectClientVersion(): string {
|
|
138
|
+
if (cachedClientVersion !== undefined) return cachedClientVersion ?? "";
|
|
139
|
+
cachedClientVersion = null;
|
|
140
|
+
try {
|
|
141
|
+
const parsed = JSON.parse(
|
|
142
|
+
readFileSync(new URL("../../../package.json", import.meta.url), "utf8"),
|
|
143
|
+
) as { version?: unknown };
|
|
144
|
+
if (parsed && typeof parsed.version === "string") cachedClientVersion = parsed.version;
|
|
145
|
+
} catch {
|
|
146
|
+
cachedClientVersion = null;
|
|
147
|
+
}
|
|
148
|
+
return cachedClientVersion ?? "";
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** "linux/x64"-style platform tag for the envelope metadata. */
|
|
152
|
+
export function detectClientPlatform(): string {
|
|
153
|
+
try {
|
|
154
|
+
return `${os.platform()}/${os.arch()}`;
|
|
155
|
+
} catch {
|
|
156
|
+
return "";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Per-session transport gate: serializes envelope sends while giving
|
|
162
|
+
* trace parts priority over baseline (start/change/end) parts. Two
|
|
163
|
+
* upload lanes (the serial session tail and the trace tail) share the
|
|
164
|
+
* transport through this gate; a release always wakes a waiting trace
|
|
165
|
+
* acquisition before a background one, and the background lane
|
|
166
|
+
* additionally yields while trace work is pending (see
|
|
167
|
+
* `yieldToTrace`), so a trace part is never delayed by more than the
|
|
168
|
+
* one baseline part already in flight.
|
|
169
|
+
*/
|
|
170
|
+
export class PriorityGate {
|
|
171
|
+
private held = false;
|
|
172
|
+
private readonly queue: Array<{ priority: boolean; wake: () => void }> = [];
|
|
173
|
+
|
|
174
|
+
async acquire(priority: boolean): Promise<void> {
|
|
175
|
+
if (!this.held) {
|
|
176
|
+
this.held = true;
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
await new Promise<void>((resolveWake) => {
|
|
180
|
+
this.queue.push({ priority, wake: resolveWake });
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
release(): void {
|
|
185
|
+
const index = this.queue.findIndex((waiter) => waiter.priority);
|
|
186
|
+
const next = index >= 0 ? this.queue.splice(index, 1)[0] : this.queue.shift();
|
|
187
|
+
if (next) next.wake();
|
|
188
|
+
else this.held = false;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
get waiting(): number {
|
|
192
|
+
return this.queue.length;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
127
196
|
export const CHANGE_DEBOUNCE_MS = 2_000;
|
|
128
197
|
export const FALLBACK_SCAN_MS = 10_000;
|
|
129
198
|
export const MAX_SESSION_LEDGER_ENTRIES = 512;
|
|
@@ -172,6 +241,12 @@ export type SessionLedgerRecord = {
|
|
|
172
241
|
* restart. Observed live: 846 start snapshots / multiple GB for one
|
|
173
242
|
* session. */
|
|
174
243
|
baselineUploaded?: boolean;
|
|
244
|
+
/** Part-level baseline resume cursor: every sorted path <= this has
|
|
245
|
+
* shipped in the session's start snapshot. Set after each acked
|
|
246
|
+
* baseline part, cleared once the baseline completes. A resumed
|
|
247
|
+
* session whose baseline never finished skips those paths (the first
|
|
248
|
+
* unacked part onward) instead of re-burning them. */
|
|
249
|
+
baselineResumePath?: string;
|
|
175
250
|
lastSeenAt: string;
|
|
176
251
|
/** Per-session-FILE transcript capture state (issue: trace format spec).
|
|
177
252
|
* Keys are absolute .jsonl paths; a size/mtime change re-captures. */
|
|
@@ -197,6 +272,9 @@ type SessionState = {
|
|
|
197
272
|
finished: boolean;
|
|
198
273
|
budgetExhausted: boolean;
|
|
199
274
|
baselineUploaded: boolean;
|
|
275
|
+
/** Resume cursor for an interrupted start baseline (see
|
|
276
|
+
* SessionLedgerRecord.baselineResumePath); null once it completed. */
|
|
277
|
+
baselineResumePath: string | null;
|
|
200
278
|
changeTimer: ReturnType<typeof setTimeout> | null;
|
|
201
279
|
scanTimer: ReturnType<typeof setInterval> | null;
|
|
202
280
|
/** Known per-path stat snapshot (mtimeMs + size) for change detection.
|
|
@@ -245,7 +323,22 @@ type SessionState = {
|
|
|
245
323
|
transcripts: Record<string, { size: number; mtimeMs: number }>;
|
|
246
324
|
changeCaptureTail: Promise<void>;
|
|
247
325
|
ready: Promise<void>;
|
|
326
|
+
/** Baseline/change/end lane: serialized, background priority. */
|
|
248
327
|
tail: Promise<void>;
|
|
328
|
+
/** Trace lane: serialized within itself, priority over `tail` at the
|
|
329
|
+
* transport gate. Trace parts must never queue behind baseline
|
|
330
|
+
* parts. */
|
|
331
|
+
traceTail: Promise<void>;
|
|
332
|
+
/** Trace operations enqueued but not settled (queued or running). */
|
|
333
|
+
tracePending: number;
|
|
334
|
+
/** Resolves whenever `tracePending` returns to 0; re-armed by
|
|
335
|
+
* `enqueueTrace`. The background lane waits on it before starting
|
|
336
|
+
* another baseline part. */
|
|
337
|
+
traceDrain: Promise<void>;
|
|
338
|
+
resolveTraceDrain: () => void;
|
|
339
|
+
/** Serializes envelope sends across both lanes; trace acquisitions
|
|
340
|
+
* jump ahead of queued background ones. */
|
|
341
|
+
gate: PriorityGate;
|
|
249
342
|
};
|
|
250
343
|
|
|
251
344
|
export type CollectorOptions = {
|
|
@@ -268,6 +361,17 @@ export type CollectorOptions = {
|
|
|
268
361
|
fallbackScanMs?: number;
|
|
269
362
|
/** Per-session collection budget. Default MAX_SESSION_BYTES (100 GiB). */
|
|
270
363
|
sessionBudgetBytes?: number;
|
|
364
|
+
/** Client version reported in every envelope (additive metadata the
|
|
365
|
+
* backend tolerates). Default: detectClientVersion(). */
|
|
366
|
+
clientVersion?: string;
|
|
367
|
+
/** Client platform tag. Default: detectClientPlatform(). */
|
|
368
|
+
clientPlatform?: string;
|
|
369
|
+
/** Linked/child mode: this process is a SUBAGENT of an existing
|
|
370
|
+
* session (spawn_agents sets OMNIRUSH_PARENT_SESSION). The collector
|
|
371
|
+
* then ships only trace events + this session's own transcript
|
|
372
|
+
* artifact, stamped parent_session_id — never a second workspace
|
|
373
|
+
* baseline/change/end collection. */
|
|
374
|
+
linkedParentSession?: string;
|
|
271
375
|
/** Multi-file part compressed target. Default MAX_PART_COMPRESSED_BYTES. */
|
|
272
376
|
partCompressedBytes?: number;
|
|
273
377
|
/** Uncompressed accumulation step between compress checks. */
|
|
@@ -814,9 +918,19 @@ export function compressZstd(payload: Buffer): Promise<Buffer> {
|
|
|
814
918
|
});
|
|
815
919
|
}
|
|
816
920
|
|
|
817
|
-
/** The uncompressed collect envelope (contract: schema_version 1).
|
|
921
|
+
/** The uncompressed collect envelope (contract: schema_version 1).
|
|
922
|
+
* Additive metadata: client_version / client_platform when known and a
|
|
923
|
+
* sent_at timestamp — the backend ignores what it does not know. */
|
|
818
924
|
export function buildEnvelopePayload(
|
|
819
|
-
state: Pick<SessionState, "id" | "segment" | "resumed"> & {
|
|
925
|
+
state: Pick<SessionState, "id" | "segment" | "resumed"> & {
|
|
926
|
+
sequence?: number;
|
|
927
|
+
clientVersion?: string;
|
|
928
|
+
clientPlatform?: string;
|
|
929
|
+
/** Capture time override (change snapshots carry the time the
|
|
930
|
+
* journal records were captured, not the time the queued upload
|
|
931
|
+
* finally built the envelope). */
|
|
932
|
+
capturedAt?: string;
|
|
933
|
+
},
|
|
820
934
|
snapshotType: SnapshotType,
|
|
821
935
|
files: CollectorFile[],
|
|
822
936
|
): Buffer {
|
|
@@ -827,6 +941,13 @@ export function buildEnvelopePayload(
|
|
|
827
941
|
session_resumed: state.resumed,
|
|
828
942
|
sequence: (state.sequence ?? 0) + 1,
|
|
829
943
|
snapshot_type: snapshotType,
|
|
944
|
+
...(state.clientVersion ? { client_version: state.clientVersion } : {}),
|
|
945
|
+
...(state.clientPlatform ? { client_platform: state.clientPlatform } : {}),
|
|
946
|
+
sent_at: new Date().toISOString(),
|
|
947
|
+
// Capture time for the backend's per-turn diff attribution (the
|
|
948
|
+
// desktop sends the same field; without it a CLI change snapshot
|
|
949
|
+
// cannot be pinned to the turn that was running).
|
|
950
|
+
captured_at: state.capturedAt ?? new Date().toISOString(),
|
|
830
951
|
files,
|
|
831
952
|
}));
|
|
832
953
|
}
|
|
@@ -970,9 +1091,15 @@ export class WorkspaceCollector {
|
|
|
970
1091
|
private ledger: SessionLedger = { version: 1, sessions: {} };
|
|
971
1092
|
private ledgerWriteTail: Promise<void> = Promise.resolve();
|
|
972
1093
|
private readonly sessions = new Map<string, SessionState>();
|
|
1094
|
+
/** Gate for states built without one (direct uploadEnvelope callers,
|
|
1095
|
+
* tests) — per-session gates are created in startSession. */
|
|
1096
|
+
private readonly defaultGate = new PriorityGate();
|
|
973
1097
|
private readonly changeDebounceMs: number;
|
|
974
1098
|
private readonly fallbackScanMs: number;
|
|
975
1099
|
private readonly sessionBudgetBytes: number;
|
|
1100
|
+
private readonly clientVersion: string;
|
|
1101
|
+
private readonly clientPlatform: string;
|
|
1102
|
+
private readonly linkedParentSession: string | null;
|
|
976
1103
|
private readonly partCompressedBytes: number;
|
|
977
1104
|
private readonly partCheckBytes: number;
|
|
978
1105
|
|
|
@@ -992,6 +1119,9 @@ export class WorkspaceCollector {
|
|
|
992
1119
|
this.changeDebounceMs = options.changeDebounceMs ?? CHANGE_DEBOUNCE_MS;
|
|
993
1120
|
this.fallbackScanMs = options.fallbackScanMs ?? FALLBACK_SCAN_MS;
|
|
994
1121
|
this.sessionBudgetBytes = Math.max(1024, options.sessionBudgetBytes ?? MAX_SESSION_BYTES);
|
|
1122
|
+
this.clientVersion = options.clientVersion ?? detectClientVersion();
|
|
1123
|
+
this.clientPlatform = options.clientPlatform ?? detectClientPlatform();
|
|
1124
|
+
this.linkedParentSession = (options.linkedParentSession ?? "").trim() || null;
|
|
995
1125
|
this.partCompressedBytes = Math.min(
|
|
996
1126
|
Math.max(1024, options.partCompressedBytes ?? MAX_PART_COMPRESSED_BYTES),
|
|
997
1127
|
MAX_PART_COMPRESSED_HARD,
|
|
@@ -1020,6 +1150,35 @@ export class WorkspaceCollector {
|
|
|
1020
1150
|
await this.ledgerWriteTail;
|
|
1021
1151
|
}
|
|
1022
1152
|
|
|
1153
|
+
/** The state fields every envelope builder needs, plus the additive
|
|
1154
|
+
* client metadata. `sent_at`/`captured_at` are stamped by
|
|
1155
|
+
* buildEnvelopePayload at build time. */
|
|
1156
|
+
private sequenceState(
|
|
1157
|
+
state: SessionState,
|
|
1158
|
+
capturedAt?: string,
|
|
1159
|
+
): {
|
|
1160
|
+
id: string;
|
|
1161
|
+
segment: number;
|
|
1162
|
+
resumed: boolean;
|
|
1163
|
+
sequence: number;
|
|
1164
|
+
clientVersion?: string;
|
|
1165
|
+
clientPlatform?: string;
|
|
1166
|
+
capturedAt?: string;
|
|
1167
|
+
} {
|
|
1168
|
+
return {
|
|
1169
|
+
id: state.id,
|
|
1170
|
+
segment: state.segment,
|
|
1171
|
+
resumed: state.resumed,
|
|
1172
|
+
sequence: state.sequence,
|
|
1173
|
+
...(this.clientVersion ? { clientVersion: this.clientVersion } : {}),
|
|
1174
|
+
...(this.clientPlatform ? { clientPlatform: this.clientPlatform } : {}),
|
|
1175
|
+
// A change part is pinned to the capture time of its OLDEST
|
|
1176
|
+
// journal record: the upload may queue behind the baseline, but
|
|
1177
|
+
// the turn attribution must reflect when the edit was seen.
|
|
1178
|
+
...(capturedAt ? { capturedAt } : {}),
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1023
1182
|
private async prepareSession(state: SessionState): Promise<void> {
|
|
1024
1183
|
await this.ledgerReady;
|
|
1025
1184
|
const previous = this.ledger.sessions[state.id];
|
|
@@ -1028,6 +1187,14 @@ export class WorkspaceCollector {
|
|
|
1028
1187
|
state.sentBytes = previous?.sentBytes ?? 0;
|
|
1029
1188
|
state.lastMessageId = previous?.lastMessageId;
|
|
1030
1189
|
state.baselineUploaded = previous?.baselineUploaded === true;
|
|
1190
|
+
// An interrupted baseline resumes from its cursor; a completed one
|
|
1191
|
+
// (or a fresh session) has none.
|
|
1192
|
+
state.baselineResumePath =
|
|
1193
|
+
previous && !state.baselineUploaded
|
|
1194
|
+
? typeof previous.baselineResumePath === "string"
|
|
1195
|
+
? previous.baselineResumePath
|
|
1196
|
+
: null
|
|
1197
|
+
: null;
|
|
1031
1198
|
state.resumed = Boolean(previous);
|
|
1032
1199
|
if (previous?.traces && typeof previous.traces === "object") {
|
|
1033
1200
|
state.transcripts = { ...previous.traces };
|
|
@@ -1045,6 +1212,7 @@ export class WorkspaceCollector {
|
|
|
1045
1212
|
sentBytes: state.sentBytes,
|
|
1046
1213
|
...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
|
|
1047
1214
|
...(state.baselineUploaded ? { baselineUploaded: true } : {}),
|
|
1215
|
+
...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
|
|
1048
1216
|
lastSeenAt: new Date().toISOString(),
|
|
1049
1217
|
};
|
|
1050
1218
|
await this.saveLedger();
|
|
@@ -1058,6 +1226,7 @@ export class WorkspaceCollector {
|
|
|
1058
1226
|
sentBytes: state.sentBytes,
|
|
1059
1227
|
...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
|
|
1060
1228
|
...(state.baselineUploaded ? { baselineUploaded: true } : {}),
|
|
1229
|
+
...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
|
|
1061
1230
|
...(Object.keys(state.transcripts).length > 0 ? { traces: state.transcripts } : {}),
|
|
1062
1231
|
lastSeenAt: new Date().toISOString(),
|
|
1063
1232
|
};
|
|
@@ -1098,6 +1267,7 @@ export class WorkspaceCollector {
|
|
|
1098
1267
|
finished: false,
|
|
1099
1268
|
budgetExhausted: false,
|
|
1100
1269
|
baselineUploaded: false,
|
|
1270
|
+
baselineResumePath: null,
|
|
1101
1271
|
changeTimer: null,
|
|
1102
1272
|
scanTimer: null,
|
|
1103
1273
|
knownStats: new Map(),
|
|
@@ -1116,9 +1286,20 @@ export class WorkspaceCollector {
|
|
|
1116
1286
|
changeCaptureTail: Promise.resolve(),
|
|
1117
1287
|
ready: Promise.resolve(),
|
|
1118
1288
|
tail: Promise.resolve(),
|
|
1289
|
+
traceTail: Promise.resolve(),
|
|
1290
|
+
tracePending: 0,
|
|
1291
|
+
traceDrain: Promise.resolve(),
|
|
1292
|
+
resolveTraceDrain: () => undefined,
|
|
1293
|
+
gate: new PriorityGate(),
|
|
1119
1294
|
};
|
|
1120
1295
|
state.ready = this.prepareSession(state);
|
|
1121
1296
|
this.sessions.set(sessionId, state);
|
|
1297
|
+
if (this.linkedParentSession) {
|
|
1298
|
+
// Child of a live parent session: the parent owns the workspace
|
|
1299
|
+
// collection (baseline/changes/end). This collector only traces.
|
|
1300
|
+
state.started = true;
|
|
1301
|
+
return;
|
|
1302
|
+
}
|
|
1122
1303
|
this.enqueue(state, async () => {
|
|
1123
1304
|
await state.ready;
|
|
1124
1305
|
const baselinePaths = await listWorkspaceFiles(root);
|
|
@@ -1147,6 +1328,8 @@ export class WorkspaceCollector {
|
|
|
1147
1328
|
}
|
|
1148
1329
|
await this.uploadWorkspace(state, "start");
|
|
1149
1330
|
state.baselineUploaded = true;
|
|
1331
|
+
state.baselineResumePath = null;
|
|
1332
|
+
await this.persistSession(state);
|
|
1150
1333
|
state.started = true;
|
|
1151
1334
|
});
|
|
1152
1335
|
// Change detection: signature scan (unref'd, self-serializing). No
|
|
@@ -1207,7 +1390,7 @@ export class WorkspaceCollector {
|
|
|
1207
1390
|
if (state.trace.length >= TRACE_RELIEF_EVENTS) {
|
|
1208
1391
|
// Memory relief, NOT truncation: the whole batch is uploaded.
|
|
1209
1392
|
const batch = state.trace.splice(0);
|
|
1210
|
-
this.
|
|
1393
|
+
this.enqueueTrace(state, async () => {
|
|
1211
1394
|
await state.ready;
|
|
1212
1395
|
await this.uploadTrace(state, batch);
|
|
1213
1396
|
});
|
|
@@ -1218,17 +1401,31 @@ export class WorkspaceCollector {
|
|
|
1218
1401
|
const state = this.sessions.get(sessionId);
|
|
1219
1402
|
if (!state || state.finished) return;
|
|
1220
1403
|
const pendingTrace = state.trace.splice(0);
|
|
1404
|
+
if (finalTrace !== undefined) {
|
|
1405
|
+
pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
|
|
1406
|
+
}
|
|
1221
1407
|
state.finished = true;
|
|
1222
1408
|
if (state.changeTimer) clearTimeout(state.changeTimer);
|
|
1223
1409
|
if (state.scanTimer) clearInterval(state.scanTimer);
|
|
1224
|
-
|
|
1410
|
+
// Final trace rides the PRIORITY lane: an in-flight baseline part
|
|
1411
|
+
// finishes first, then the trace ships — never behind the tail of a
|
|
1412
|
+
// multi-part tree upload.
|
|
1413
|
+
this.enqueueTrace(state, async () => {
|
|
1225
1414
|
await state.ready;
|
|
1226
1415
|
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
|
|
1416
|
+
await this.uploadTrace(state, pendingTrace);
|
|
1417
|
+
});
|
|
1418
|
+
this.enqueue(state, async () => {
|
|
1419
|
+
await state.ready;
|
|
1227
1420
|
await state.changeCaptureTail;
|
|
1228
|
-
if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
|
|
1229
1421
|
await this.teardownJournal(state); // ship unacked change records, then unlink
|
|
1230
|
-
|
|
1231
|
-
|
|
1422
|
+
// The end snapshot yields to pending trace work (yieldToTrace in
|
|
1423
|
+
// the packer), so the final trace lands first.
|
|
1424
|
+
if (!this.linkedParentSession) {
|
|
1425
|
+
// A linked child ships no end snapshot: the parent session's
|
|
1426
|
+
// collection covers the workspace tree.
|
|
1427
|
+
await this.uploadWorkspace(state, "end");
|
|
1428
|
+
}
|
|
1232
1429
|
this.sessions.delete(sessionId);
|
|
1233
1430
|
});
|
|
1234
1431
|
}
|
|
@@ -1239,7 +1436,10 @@ export class WorkspaceCollector {
|
|
|
1239
1436
|
const pendingTrace = state.trace.splice(0);
|
|
1240
1437
|
if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });
|
|
1241
1438
|
if (pendingTrace.length === 0) return;
|
|
1242
|
-
|
|
1439
|
+
// Per-turn flush: the complete trace artifact (events + transcript
|
|
1440
|
+
// messages-so-far) ships on the priority lane, ahead of any queued
|
|
1441
|
+
// baseline parts.
|
|
1442
|
+
this.enqueueTrace(state, async () => {
|
|
1243
1443
|
await state.ready;
|
|
1244
1444
|
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
|
|
1245
1445
|
await this.uploadTrace(state, pendingTrace);
|
|
@@ -1248,7 +1448,45 @@ export class WorkspaceCollector {
|
|
|
1248
1448
|
|
|
1249
1449
|
async stop(): Promise<void> {
|
|
1250
1450
|
for (const sessionId of [...this.sessions.keys()]) this.finishSession(sessionId);
|
|
1251
|
-
await Promise.allSettled(
|
|
1451
|
+
await Promise.allSettled(
|
|
1452
|
+
[...this.sessions.values()].flatMap((state) => [state.tail, state.traceTail]),
|
|
1453
|
+
);
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
/**
|
|
1457
|
+
* Queue trace work on the priority lane and mark it pending: the
|
|
1458
|
+
* background lane's `yieldToTrace` waits for `tracePending` to drain
|
|
1459
|
+
* before starting another baseline part, so trace work enqueued while
|
|
1460
|
+
* a baseline part was in flight goes next.
|
|
1461
|
+
*/
|
|
1462
|
+
private enqueueTrace(state: SessionState, operation: () => Promise<void>): void {
|
|
1463
|
+
if (state.tracePending === 0) {
|
|
1464
|
+
let resolveDrain!: () => void;
|
|
1465
|
+
state.traceDrain = new Promise<void>((resolvePromise) => {
|
|
1466
|
+
resolveDrain = resolvePromise;
|
|
1467
|
+
});
|
|
1468
|
+
state.resolveTraceDrain = () => resolveDrain();
|
|
1469
|
+
}
|
|
1470
|
+
state.tracePending += 1;
|
|
1471
|
+
state.traceTail = state.traceTail
|
|
1472
|
+
.then(operation)
|
|
1473
|
+
.catch((error: unknown) => {
|
|
1474
|
+
this.log("warn", "OmniRush collection operation failed", {
|
|
1475
|
+
sessionId: state.id,
|
|
1476
|
+
error: error instanceof Error ? error.message : "unknown",
|
|
1477
|
+
});
|
|
1478
|
+
})
|
|
1479
|
+
.finally(() => {
|
|
1480
|
+
state.tracePending -= 1;
|
|
1481
|
+
if (state.tracePending === 0) state.resolveTraceDrain();
|
|
1482
|
+
});
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/** Wait until no trace work is queued or running (re-arming safe). */
|
|
1486
|
+
private async yieldToTrace(state: SessionState): Promise<void> {
|
|
1487
|
+
while (state.tracePending > 0) {
|
|
1488
|
+
await state.traceDrain;
|
|
1489
|
+
}
|
|
1252
1490
|
}
|
|
1253
1491
|
|
|
1254
1492
|
private stopWatching(state: SessionState): void {
|
|
@@ -1567,23 +1805,38 @@ export class WorkspaceCollector {
|
|
|
1567
1805
|
};
|
|
1568
1806
|
}
|
|
1569
1807
|
|
|
1570
|
-
/** Full-tree snapshot ("start" | "end"), streamed into ordered parts.
|
|
1808
|
+
/** Full-tree snapshot ("start" | "end"), streamed into ordered parts.
|
|
1809
|
+
* A start snapshot resumed after an interruption skips the paths the
|
|
1810
|
+
* previous run already shipped (`baselineResumePath` cursor — the
|
|
1811
|
+
* first unacked part onward); the end snapshot always ships whole. */
|
|
1571
1812
|
private async uploadTreeParts(state: SessionState, snapshotType: "start" | "end"): Promise<void> {
|
|
1572
1813
|
const paths = (await listWorkspaceFiles(state.root)).sort(comparePaths);
|
|
1573
1814
|
const metadata = await this.workspaceMetadataFile(state);
|
|
1574
1815
|
let metadataSent = false;
|
|
1575
|
-
const
|
|
1816
|
+
const resumeFrom = snapshotType === "start" ? state.baselineResumePath : null;
|
|
1576
1817
|
const measure = async (batch: CollectorFile[]) => {
|
|
1577
1818
|
const files = metadataSent ? batch : [metadata, ...batch];
|
|
1578
|
-
const payload = buildEnvelopePayload(
|
|
1819
|
+
const payload = buildEnvelopePayload(this.sequenceState(state), snapshotType, files);
|
|
1579
1820
|
return { files, payload, compressed: await compressZstd(payload) };
|
|
1580
1821
|
};
|
|
1581
|
-
await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths), {
|
|
1822
|
+
await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths, resumeFrom), {
|
|
1582
1823
|
sizeOf: (file) => Buffer.byteLength(file.content ?? "") + file.path.length + 64,
|
|
1583
1824
|
measure,
|
|
1584
1825
|
upload: async (measured) => {
|
|
1585
1826
|
const ok = await this.uploadEnvelope(state, snapshotType, measured.files, measured);
|
|
1586
|
-
if (ok)
|
|
1827
|
+
if (ok) {
|
|
1828
|
+
metadataSent = true;
|
|
1829
|
+
if (snapshotType === "start") {
|
|
1830
|
+
// Advance the resume cursor to the last path of the acked
|
|
1831
|
+
// part, so an interrupted run resumes at the first
|
|
1832
|
+
// unacked part instead of re-burning the prefix.
|
|
1833
|
+
const lastPath = measured.files[measured.files.length - 1]?.path;
|
|
1834
|
+
if (typeof lastPath === "string") {
|
|
1835
|
+
state.baselineResumePath = lastPath;
|
|
1836
|
+
await this.persistSession(state).catch(() => undefined);
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1587
1840
|
return ok;
|
|
1588
1841
|
},
|
|
1589
1842
|
drop: (file) => {
|
|
@@ -1598,12 +1851,17 @@ export class WorkspaceCollector {
|
|
|
1598
1851
|
data: { path: file.path, bytes: Buffer.byteLength(file.content ?? "") },
|
|
1599
1852
|
});
|
|
1600
1853
|
},
|
|
1601
|
-
});
|
|
1854
|
+
}, { yieldToTraceLane: true });
|
|
1602
1855
|
}
|
|
1603
1856
|
|
|
1604
|
-
private async *iterTreeFiles(
|
|
1857
|
+
private async *iterTreeFiles(
|
|
1858
|
+
state: SessionState,
|
|
1859
|
+
paths: string[],
|
|
1860
|
+
resumeFrom?: string | null,
|
|
1861
|
+
): AsyncGenerator<CollectorFile> {
|
|
1605
1862
|
for (const path of paths) {
|
|
1606
1863
|
if (state.budgetExhausted) return;
|
|
1864
|
+
if (resumeFrom && comparePaths(path, resumeFrom) <= 0) continue;
|
|
1607
1865
|
for await (const entry of iterateWorkspaceEntries(state.root, path)) {
|
|
1608
1866
|
yield entry;
|
|
1609
1867
|
}
|
|
@@ -1677,13 +1935,14 @@ export class WorkspaceCollector {
|
|
|
1677
1935
|
});
|
|
1678
1936
|
return files;
|
|
1679
1937
|
};
|
|
1680
|
-
const bySequence = () => (
|
|
1938
|
+
const bySequence = (capturedAt?: string) => this.sequenceState(state, capturedAt);
|
|
1681
1939
|
await this.packAndUpload(state, "change", units, {
|
|
1682
1940
|
sizeOf: (entry) =>
|
|
1683
1941
|
Buffer.byteLength(entry.content ?? "") + entry.path.length + 96,
|
|
1684
1942
|
measure: async (batch) => {
|
|
1685
1943
|
const files = filesForSlice(batch);
|
|
1686
|
-
const
|
|
1944
|
+
const capturedAt = batch.find((entry) => typeof entry.at === "string")?.at;
|
|
1945
|
+
const payload = buildEnvelopePayload(bySequence(capturedAt), "change", files);
|
|
1687
1946
|
return { files, payload, compressed: await compressZstd(payload) };
|
|
1688
1947
|
},
|
|
1689
1948
|
upload: async (measured, batch) => {
|
|
@@ -1706,7 +1965,7 @@ export class WorkspaceCollector {
|
|
|
1706
1965
|
data: { path: entry.path, reason: "change_record_over_limit" },
|
|
1707
1966
|
});
|
|
1708
1967
|
},
|
|
1709
|
-
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
|
|
1968
|
+
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX, yieldToTraceLane: true });
|
|
1710
1969
|
return uploadedSlices;
|
|
1711
1970
|
}
|
|
1712
1971
|
|
|
@@ -1719,6 +1978,11 @@ export class WorkspaceCollector {
|
|
|
1719
1978
|
* corresponds to the (mutated) batch passed to hooks.upload, and popped
|
|
1720
1979
|
* units come back as `carry` to lead the next part. hooks.upload must
|
|
1721
1980
|
* persist sequence bookkeeping via uploadEnvelope.
|
|
1981
|
+
*
|
|
1982
|
+
* `yieldToTraceLane` (baseline/change/end packs): before each part
|
|
1983
|
+
* upload, wait while trace work is pending or running — a trace part
|
|
1984
|
+
* then never queues behind more than the one baseline part already in
|
|
1985
|
+
* flight. Trace packs must NOT set it (they ARE the priority lane).
|
|
1722
1986
|
*/
|
|
1723
1987
|
private async packAndUpload<U>(
|
|
1724
1988
|
state: SessionState,
|
|
@@ -1730,7 +1994,7 @@ export class WorkspaceCollector {
|
|
|
1730
1994
|
upload: (measured: { files: CollectorFile[]; payload: Buffer; compressed: Buffer }, batch: U[]) => Promise<boolean>;
|
|
1731
1995
|
drop?: (unit: U) => void;
|
|
1732
1996
|
},
|
|
1733
|
-
opts: { checkBytes?: number; flushPlainBytes?: number } = {},
|
|
1997
|
+
opts: { checkBytes?: number; flushPlainBytes?: number; yieldToTraceLane?: boolean } = {},
|
|
1734
1998
|
): Promise<void> {
|
|
1735
1999
|
const checkBytes = Math.max(1024, opts.checkBytes ?? this.partCheckBytes);
|
|
1736
2000
|
const flushPlainBytes = Math.max(checkBytes, opts.flushPlainBytes ?? PART_BATCH_UNCOMPRESSED_MAX);
|
|
@@ -1772,6 +2036,7 @@ export class WorkspaceCollector {
|
|
|
1772
2036
|
if (batchBytes < nextCheck) continue;
|
|
1773
2037
|
const { measured, carry } = await fit(batch);
|
|
1774
2038
|
if (measured && shouldFlush(measured)) {
|
|
2039
|
+
if (opts.yieldToTraceLane) await this.yieldToTrace(state);
|
|
1775
2040
|
if (!(await hooks.upload(measured, batch))) return;
|
|
1776
2041
|
resetBatch(carry);
|
|
1777
2042
|
} else if (measured) {
|
|
@@ -1786,6 +2051,7 @@ export class WorkspaceCollector {
|
|
|
1786
2051
|
while (batch.length > 0 && !state.budgetExhausted) {
|
|
1787
2052
|
const { measured, carry } = await fit(batch);
|
|
1788
2053
|
if (measured) {
|
|
2054
|
+
if (opts.yieldToTraceLane) await this.yieldToTrace(state);
|
|
1789
2055
|
if (!(await hooks.upload(measured, batch))) return;
|
|
1790
2056
|
}
|
|
1791
2057
|
if (carry.length === 0) break;
|
|
@@ -1820,7 +2086,7 @@ export class WorkspaceCollector {
|
|
|
1820
2086
|
});
|
|
1821
2087
|
}
|
|
1822
2088
|
let partIndex = 0;
|
|
1823
|
-
const bySequence = () => (
|
|
2089
|
+
const bySequence = () => this.sequenceState(state);
|
|
1824
2090
|
await this.packAndUpload(state, "trace", selected.events, {
|
|
1825
2091
|
sizeOf: (event) => Buffer.byteLength(JSON.stringify(event)) + 32,
|
|
1826
2092
|
measure: async (batch) => {
|
|
@@ -1921,10 +2187,16 @@ export class WorkspaceCollector {
|
|
|
1921
2187
|
state.transcripts[filePath] = { size, mtimeMs };
|
|
1922
2188
|
continue;
|
|
1923
2189
|
}
|
|
2190
|
+
if (this.linkedParentSession && artifactSessionId !== state.id) {
|
|
2191
|
+
// Linked child: the parent's collector already ships the other
|
|
2192
|
+
// session files in this directory (linked to the parent) — a
|
|
2193
|
+
// second copy from the child adds nothing.
|
|
2194
|
+
continue;
|
|
2195
|
+
}
|
|
1924
2196
|
const identity = await this.resolveIdentity();
|
|
1925
2197
|
const artifact = buildTraceArtifact({
|
|
1926
2198
|
sessionId: artifactSessionId,
|
|
1927
|
-
parentSessionId: state.id,
|
|
2199
|
+
parentSessionId: this.linkedParentSession ?? state.id,
|
|
1928
2200
|
userId: identity?.userId ?? null,
|
|
1929
2201
|
clientId: this.clientId ?? null,
|
|
1930
2202
|
agentId: "omnirush-cli",
|
|
@@ -1944,7 +2216,7 @@ export class WorkspaceCollector {
|
|
|
1944
2216
|
await this.persistSession(state).catch(() => undefined);
|
|
1945
2217
|
return;
|
|
1946
2218
|
}
|
|
1947
|
-
const bySequence = () => (
|
|
2219
|
+
const bySequence = () => this.sequenceState(state);
|
|
1948
2220
|
await this.packAndUpload(state, "trace", artifacts, {
|
|
1949
2221
|
sizeOf: (artifact) => Buffer.byteLength(artifact.content) + artifact.path.length + 64,
|
|
1950
2222
|
measure: async (batch) => {
|
|
@@ -2012,30 +2284,44 @@ export class WorkspaceCollector {
|
|
|
2012
2284
|
});
|
|
2013
2285
|
return false;
|
|
2014
2286
|
}
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2287
|
+
// Two lanes (session tail + trace tail) funnel every envelope send
|
|
2288
|
+
// through this gate. Trace sends acquire with priority, so after the
|
|
2289
|
+
// part currently in flight finishes, the trace part goes next; the
|
|
2290
|
+
// gate is held per ATTEMPT, so a retrying baseline part cannot pin
|
|
2291
|
+
// the transport either.
|
|
2292
|
+
const priority = snapshotType === "trace";
|
|
2293
|
+
const gate = state.gate ?? this.defaultGate;
|
|
2294
|
+
const sendOnce = async () => {
|
|
2295
|
+
await gate.acquire(priority);
|
|
2296
|
+
try {
|
|
2297
|
+
return await (this.uploader
|
|
2298
|
+
? this.uploader(state.id, compressed)
|
|
2299
|
+
: this.fetcher(this.collectUrl!, {
|
|
2300
|
+
method: "POST",
|
|
2301
|
+
headers: {
|
|
2302
|
+
Authorization: `Bearer ${this.token}`,
|
|
2303
|
+
"Content-Type": "application/zstd",
|
|
2304
|
+
"X-Omnirush-Session-ID": state.id,
|
|
2305
|
+
},
|
|
2306
|
+
body: compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer,
|
|
2307
|
+
signal: AbortSignal.timeout(120_000),
|
|
2308
|
+
}));
|
|
2309
|
+
} finally {
|
|
2310
|
+
gate.release();
|
|
2311
|
+
}
|
|
2312
|
+
};
|
|
2027
2313
|
// Shared retry layer: 429/5xx/network ride out backend deploys with
|
|
2028
2314
|
// exponential backoff + jitter. 401 stays a protocol answer handled
|
|
2029
2315
|
// inline (single-flight refresh, retry exactly once).
|
|
2030
2316
|
const outcome = await withRetries(
|
|
2031
2317
|
async () => {
|
|
2032
|
-
let response = await
|
|
2318
|
+
let response = await sendOnce();
|
|
2033
2319
|
if (response.status === 401 && this.refresh) {
|
|
2034
2320
|
// Broker pattern: single-flight refresh, retry exactly once.
|
|
2035
2321
|
const rotated = await this.refresh(this.token).catch(() => null);
|
|
2036
2322
|
if (rotated) {
|
|
2037
2323
|
this.token = rotated;
|
|
2038
|
-
response = await
|
|
2324
|
+
response = await sendOnce();
|
|
2039
2325
|
}
|
|
2040
2326
|
}
|
|
2041
2327
|
return {
|