omnirush 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/extensions/omnirush/agents-lib.ts +421 -0
- package/assets/extensions/omnirush/agents.ts +163 -0
- package/assets/extensions/omnirush/collector-lib.ts +354 -37
- package/assets/extensions/omnirush/collector.ts +24 -7
- package/assets/extensions/omnirush/index.ts +6 -0
- package/assets/extensions/omnirush/plan-lib.ts +113 -0
- package/assets/extensions/omnirush/plan.ts +136 -0
- package/assets/extensions/omnirush/trace-format.ts +17 -0
- package/assets/extensions/omnirush/webfetch-lib.ts +241 -0
- package/assets/extensions/omnirush/webfetch.ts +77 -0
- package/package.json +1 -1
- package/scripts/build-all-packages.py +0 -3
- package/src/lib.js +34 -0
|
@@ -47,7 +47,9 @@
|
|
|
47
47
|
import { createHash } from "node:crypto";
|
|
48
48
|
import { execFile } from "node:child_process";
|
|
49
49
|
|
|
50
|
+
import { readFileSync } from "node:fs";
|
|
50
51
|
import { lstat, mkdir, readFile, readdir, readlink, writeFile } from "node:fs/promises";
|
|
52
|
+
import os from "node:os";
|
|
51
53
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
52
54
|
import { promisify } from "node:util";
|
|
53
55
|
import { zstdCompress as zstdCompressCb } from "node:zlib";
|
|
@@ -124,6 +126,73 @@ export const TRACE_RELIEF_EVENTS = 2_000;
|
|
|
124
126
|
export const AGENT_DIR = "__agent__";
|
|
125
127
|
export const AGENT_TRACE_SCHEMA = "agent.trace.v1";
|
|
126
128
|
|
|
129
|
+
let cachedClientVersion: string | null | undefined;
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* The CLI's package.json version, resolved once relative to this module
|
|
133
|
+
* (repo checkout and installed npm layout both keep package.json three
|
|
134
|
+
* levels up). Empty when it cannot be read — the envelope field is
|
|
135
|
+
* optional and the backend tolerates its absence.
|
|
136
|
+
*/
|
|
137
|
+
export function detectClientVersion(): string {
|
|
138
|
+
if (cachedClientVersion !== undefined) return cachedClientVersion ?? "";
|
|
139
|
+
cachedClientVersion = null;
|
|
140
|
+
try {
|
|
141
|
+
const parsed = JSON.parse(
|
|
142
|
+
readFileSync(new URL("../../../package.json", import.meta.url), "utf8"),
|
|
143
|
+
) as { version?: unknown };
|
|
144
|
+
if (parsed && typeof parsed.version === "string") cachedClientVersion = parsed.version;
|
|
145
|
+
} catch {
|
|
146
|
+
cachedClientVersion = null;
|
|
147
|
+
}
|
|
148
|
+
return cachedClientVersion ?? "";
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** "linux/x64"-style platform tag for the envelope metadata. */
|
|
152
|
+
export function detectClientPlatform(): string {
|
|
153
|
+
try {
|
|
154
|
+
return `${os.platform()}/${os.arch()}`;
|
|
155
|
+
} catch {
|
|
156
|
+
return "";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Per-session transport gate: serializes envelope sends while giving
|
|
162
|
+
* trace parts priority over baseline (start/change/end) parts. Two
|
|
163
|
+
* upload lanes (the serial session tail and the trace tail) share the
|
|
164
|
+
* transport through this gate; a release always wakes a waiting trace
|
|
165
|
+
* acquisition before a background one, and the background lane
|
|
166
|
+
* additionally yields while trace work is pending (see
|
|
167
|
+
* `yieldToTrace`), so a trace part is never delayed by more than the
|
|
168
|
+
* one baseline part already in flight.
|
|
169
|
+
*/
|
|
170
|
+
export class PriorityGate {
|
|
171
|
+
private held = false;
|
|
172
|
+
private readonly queue: Array<{ priority: boolean; wake: () => void }> = [];
|
|
173
|
+
|
|
174
|
+
async acquire(priority: boolean): Promise<void> {
|
|
175
|
+
if (!this.held) {
|
|
176
|
+
this.held = true;
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
await new Promise<void>((resolveWake) => {
|
|
180
|
+
this.queue.push({ priority, wake: resolveWake });
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
release(): void {
|
|
185
|
+
const index = this.queue.findIndex((waiter) => waiter.priority);
|
|
186
|
+
const next = index >= 0 ? this.queue.splice(index, 1)[0] : this.queue.shift();
|
|
187
|
+
if (next) next.wake();
|
|
188
|
+
else this.held = false;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
get waiting(): number {
|
|
192
|
+
return this.queue.length;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
127
196
|
export const CHANGE_DEBOUNCE_MS = 2_000;
|
|
128
197
|
export const FALLBACK_SCAN_MS = 10_000;
|
|
129
198
|
export const MAX_SESSION_LEDGER_ENTRIES = 512;
|
|
@@ -166,6 +235,18 @@ export type SessionLedgerRecord = {
|
|
|
166
235
|
nextSequence: number;
|
|
167
236
|
sentBytes?: number;
|
|
168
237
|
lastMessageId?: string;
|
|
238
|
+
/** True once the session's full-tree start baseline shipped. A resumed
|
|
239
|
+
* session must NOT re-upload it — a restart-heavy session would
|
|
240
|
+
* otherwise re-burn the entire baseline (and starve traces) on every
|
|
241
|
+
* restart. Observed live: 846 start snapshots / multiple GB for one
|
|
242
|
+
* session. */
|
|
243
|
+
baselineUploaded?: boolean;
|
|
244
|
+
/** Part-level baseline resume cursor: every sorted path <= this has
|
|
245
|
+
* shipped in the session's start snapshot. Set after each acked
|
|
246
|
+
* baseline part, cleared once the baseline completes. A resumed
|
|
247
|
+
* session whose baseline never finished skips those paths (the first
|
|
248
|
+
* unacked part onward) instead of re-burning them. */
|
|
249
|
+
baselineResumePath?: string;
|
|
169
250
|
lastSeenAt: string;
|
|
170
251
|
/** Per-session-FILE transcript capture state (issue: trace format spec).
|
|
171
252
|
* Keys are absolute .jsonl paths; a size/mtime change re-captures. */
|
|
@@ -190,6 +271,10 @@ type SessionState = {
|
|
|
190
271
|
started: boolean;
|
|
191
272
|
finished: boolean;
|
|
192
273
|
budgetExhausted: boolean;
|
|
274
|
+
baselineUploaded: boolean;
|
|
275
|
+
/** Resume cursor for an interrupted start baseline (see
|
|
276
|
+
* SessionLedgerRecord.baselineResumePath); null once it completed. */
|
|
277
|
+
baselineResumePath: string | null;
|
|
193
278
|
changeTimer: ReturnType<typeof setTimeout> | null;
|
|
194
279
|
scanTimer: ReturnType<typeof setInterval> | null;
|
|
195
280
|
/** Known per-path stat snapshot (mtimeMs + size) for change detection.
|
|
@@ -238,7 +323,22 @@ type SessionState = {
|
|
|
238
323
|
transcripts: Record<string, { size: number; mtimeMs: number }>;
|
|
239
324
|
changeCaptureTail: Promise<void>;
|
|
240
325
|
ready: Promise<void>;
|
|
326
|
+
/** Baseline/change/end lane: serialized, background priority. */
|
|
241
327
|
tail: Promise<void>;
|
|
328
|
+
/** Trace lane: serialized within itself, priority over `tail` at the
|
|
329
|
+
* transport gate. Trace parts must never queue behind baseline
|
|
330
|
+
* parts. */
|
|
331
|
+
traceTail: Promise<void>;
|
|
332
|
+
/** Trace operations enqueued but not settled (queued or running). */
|
|
333
|
+
tracePending: number;
|
|
334
|
+
/** Resolves whenever `tracePending` returns to 0; re-armed by
|
|
335
|
+
* `enqueueTrace`. The background lane waits on it before starting
|
|
336
|
+
* another baseline part. */
|
|
337
|
+
traceDrain: Promise<void>;
|
|
338
|
+
resolveTraceDrain: () => void;
|
|
339
|
+
/** Serializes envelope sends across both lanes; trace acquisitions
|
|
340
|
+
* jump ahead of queued background ones. */
|
|
341
|
+
gate: PriorityGate;
|
|
242
342
|
};
|
|
243
343
|
|
|
244
344
|
export type CollectorOptions = {
|
|
@@ -261,6 +361,17 @@ export type CollectorOptions = {
|
|
|
261
361
|
fallbackScanMs?: number;
|
|
262
362
|
/** Per-session collection budget. Default MAX_SESSION_BYTES (100 GiB). */
|
|
263
363
|
sessionBudgetBytes?: number;
|
|
364
|
+
/** Client version reported in every envelope (additive metadata the
|
|
365
|
+
* backend tolerates). Default: detectClientVersion(). */
|
|
366
|
+
clientVersion?: string;
|
|
367
|
+
/** Client platform tag. Default: detectClientPlatform(). */
|
|
368
|
+
clientPlatform?: string;
|
|
369
|
+
/** Linked/child mode: this process is a SUBAGENT of an existing
|
|
370
|
+
* session (spawn_agents sets OMNIRUSH_PARENT_SESSION). The collector
|
|
371
|
+
* then ships only trace events + this session's own transcript
|
|
372
|
+
* artifact, stamped parent_session_id — never a second workspace
|
|
373
|
+
* baseline/change/end collection. */
|
|
374
|
+
linkedParentSession?: string;
|
|
264
375
|
/** Multi-file part compressed target. Default MAX_PART_COMPRESSED_BYTES. */
|
|
265
376
|
partCompressedBytes?: number;
|
|
266
377
|
/** Uncompressed accumulation step between compress checks. */
|
|
@@ -807,9 +918,19 @@ export function compressZstd(payload: Buffer): Promise<Buffer> {
|
|
|
807
918
|
});
|
|
808
919
|
}
|
|
809
920
|
|
|
810
|
-
/** The uncompressed collect envelope (contract: schema_version 1).
|
|
921
|
+
/** The uncompressed collect envelope (contract: schema_version 1).
|
|
922
|
+
* Additive metadata: client_version / client_platform when known and a
|
|
923
|
+
* sent_at timestamp — the backend ignores what it does not know. */
|
|
811
924
|
export function buildEnvelopePayload(
|
|
812
|
-
state: Pick<SessionState, "id" | "segment" | "resumed"> & {
|
|
925
|
+
state: Pick<SessionState, "id" | "segment" | "resumed"> & {
|
|
926
|
+
sequence?: number;
|
|
927
|
+
clientVersion?: string;
|
|
928
|
+
clientPlatform?: string;
|
|
929
|
+
/** Capture time override (change snapshots carry the time the
|
|
930
|
+
* journal records were captured, not the time the queued upload
|
|
931
|
+
* finally built the envelope). */
|
|
932
|
+
capturedAt?: string;
|
|
933
|
+
},
|
|
813
934
|
snapshotType: SnapshotType,
|
|
814
935
|
files: CollectorFile[],
|
|
815
936
|
): Buffer {
|
|
@@ -820,6 +941,13 @@ export function buildEnvelopePayload(
|
|
|
820
941
|
session_resumed: state.resumed,
|
|
821
942
|
sequence: (state.sequence ?? 0) + 1,
|
|
822
943
|
snapshot_type: snapshotType,
|
|
944
|
+
...(state.clientVersion ? { client_version: state.clientVersion } : {}),
|
|
945
|
+
...(state.clientPlatform ? { client_platform: state.clientPlatform } : {}),
|
|
946
|
+
sent_at: new Date().toISOString(),
|
|
947
|
+
// Capture time for the backend's per-turn diff attribution (the
|
|
948
|
+
// desktop sends the same field; without it a CLI change snapshot
|
|
949
|
+
// cannot be pinned to the turn that was running).
|
|
950
|
+
captured_at: state.capturedAt ?? new Date().toISOString(),
|
|
823
951
|
files,
|
|
824
952
|
}));
|
|
825
953
|
}
|
|
@@ -963,9 +1091,15 @@ export class WorkspaceCollector {
|
|
|
963
1091
|
private ledger: SessionLedger = { version: 1, sessions: {} };
|
|
964
1092
|
private ledgerWriteTail: Promise<void> = Promise.resolve();
|
|
965
1093
|
private readonly sessions = new Map<string, SessionState>();
|
|
1094
|
+
/** Gate for states built without one (direct uploadEnvelope callers,
|
|
1095
|
+
* tests) — per-session gates are created in startSession. */
|
|
1096
|
+
private readonly defaultGate = new PriorityGate();
|
|
966
1097
|
private readonly changeDebounceMs: number;
|
|
967
1098
|
private readonly fallbackScanMs: number;
|
|
968
1099
|
private readonly sessionBudgetBytes: number;
|
|
1100
|
+
private readonly clientVersion: string;
|
|
1101
|
+
private readonly clientPlatform: string;
|
|
1102
|
+
private readonly linkedParentSession: string | null;
|
|
969
1103
|
private readonly partCompressedBytes: number;
|
|
970
1104
|
private readonly partCheckBytes: number;
|
|
971
1105
|
|
|
@@ -985,6 +1119,9 @@ export class WorkspaceCollector {
|
|
|
985
1119
|
this.changeDebounceMs = options.changeDebounceMs ?? CHANGE_DEBOUNCE_MS;
|
|
986
1120
|
this.fallbackScanMs = options.fallbackScanMs ?? FALLBACK_SCAN_MS;
|
|
987
1121
|
this.sessionBudgetBytes = Math.max(1024, options.sessionBudgetBytes ?? MAX_SESSION_BYTES);
|
|
1122
|
+
this.clientVersion = options.clientVersion ?? detectClientVersion();
|
|
1123
|
+
this.clientPlatform = options.clientPlatform ?? detectClientPlatform();
|
|
1124
|
+
this.linkedParentSession = (options.linkedParentSession ?? "").trim() || null;
|
|
988
1125
|
this.partCompressedBytes = Math.min(
|
|
989
1126
|
Math.max(1024, options.partCompressedBytes ?? MAX_PART_COMPRESSED_BYTES),
|
|
990
1127
|
MAX_PART_COMPRESSED_HARD,
|
|
@@ -1013,6 +1150,35 @@ export class WorkspaceCollector {
|
|
|
1013
1150
|
await this.ledgerWriteTail;
|
|
1014
1151
|
}
|
|
1015
1152
|
|
|
1153
|
+
/** The state fields every envelope builder needs, plus the additive
|
|
1154
|
+
* client metadata. `sent_at`/`captured_at` are stamped by
|
|
1155
|
+
* buildEnvelopePayload at build time. */
|
|
1156
|
+
private sequenceState(
|
|
1157
|
+
state: SessionState,
|
|
1158
|
+
capturedAt?: string,
|
|
1159
|
+
): {
|
|
1160
|
+
id: string;
|
|
1161
|
+
segment: number;
|
|
1162
|
+
resumed: boolean;
|
|
1163
|
+
sequence: number;
|
|
1164
|
+
clientVersion?: string;
|
|
1165
|
+
clientPlatform?: string;
|
|
1166
|
+
capturedAt?: string;
|
|
1167
|
+
} {
|
|
1168
|
+
return {
|
|
1169
|
+
id: state.id,
|
|
1170
|
+
segment: state.segment,
|
|
1171
|
+
resumed: state.resumed,
|
|
1172
|
+
sequence: state.sequence,
|
|
1173
|
+
...(this.clientVersion ? { clientVersion: this.clientVersion } : {}),
|
|
1174
|
+
...(this.clientPlatform ? { clientPlatform: this.clientPlatform } : {}),
|
|
1175
|
+
// A change part is pinned to the capture time of its OLDEST
|
|
1176
|
+
// journal record: the upload may queue behind the baseline, but
|
|
1177
|
+
// the turn attribution must reflect when the edit was seen.
|
|
1178
|
+
...(capturedAt ? { capturedAt } : {}),
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1016
1182
|
private async prepareSession(state: SessionState): Promise<void> {
|
|
1017
1183
|
await this.ledgerReady;
|
|
1018
1184
|
const previous = this.ledger.sessions[state.id];
|
|
@@ -1020,6 +1186,15 @@ export class WorkspaceCollector {
|
|
|
1020
1186
|
state.sequence = previous?.nextSequence ?? 0;
|
|
1021
1187
|
state.sentBytes = previous?.sentBytes ?? 0;
|
|
1022
1188
|
state.lastMessageId = previous?.lastMessageId;
|
|
1189
|
+
state.baselineUploaded = previous?.baselineUploaded === true;
|
|
1190
|
+
// An interrupted baseline resumes from its cursor; a completed one
|
|
1191
|
+
// (or a fresh session) has none.
|
|
1192
|
+
state.baselineResumePath =
|
|
1193
|
+
previous && !state.baselineUploaded
|
|
1194
|
+
? typeof previous.baselineResumePath === "string"
|
|
1195
|
+
? previous.baselineResumePath
|
|
1196
|
+
: null
|
|
1197
|
+
: null;
|
|
1023
1198
|
state.resumed = Boolean(previous);
|
|
1024
1199
|
if (previous?.traces && typeof previous.traces === "object") {
|
|
1025
1200
|
state.transcripts = { ...previous.traces };
|
|
@@ -1036,6 +1211,8 @@ export class WorkspaceCollector {
|
|
|
1036
1211
|
nextSequence: state.sequence,
|
|
1037
1212
|
sentBytes: state.sentBytes,
|
|
1038
1213
|
...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
|
|
1214
|
+
...(state.baselineUploaded ? { baselineUploaded: true } : {}),
|
|
1215
|
+
...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
|
|
1039
1216
|
lastSeenAt: new Date().toISOString(),
|
|
1040
1217
|
};
|
|
1041
1218
|
await this.saveLedger();
|
|
@@ -1048,6 +1225,8 @@ export class WorkspaceCollector {
|
|
|
1048
1225
|
nextSequence: state.sequence,
|
|
1049
1226
|
sentBytes: state.sentBytes,
|
|
1050
1227
|
...(state.lastMessageId ? { lastMessageId: state.lastMessageId } : {}),
|
|
1228
|
+
...(state.baselineUploaded ? { baselineUploaded: true } : {}),
|
|
1229
|
+
...(state.baselineResumePath ? { baselineResumePath: state.baselineResumePath } : {}),
|
|
1051
1230
|
...(Object.keys(state.transcripts).length > 0 ? { traces: state.transcripts } : {}),
|
|
1052
1231
|
lastSeenAt: new Date().toISOString(),
|
|
1053
1232
|
};
|
|
@@ -1087,6 +1266,8 @@ export class WorkspaceCollector {
|
|
|
1087
1266
|
started: false,
|
|
1088
1267
|
finished: false,
|
|
1089
1268
|
budgetExhausted: false,
|
|
1269
|
+
baselineUploaded: false,
|
|
1270
|
+
baselineResumePath: null,
|
|
1090
1271
|
changeTimer: null,
|
|
1091
1272
|
scanTimer: null,
|
|
1092
1273
|
knownStats: new Map(),
|
|
@@ -1105,9 +1286,20 @@ export class WorkspaceCollector {
|
|
|
1105
1286
|
changeCaptureTail: Promise.resolve(),
|
|
1106
1287
|
ready: Promise.resolve(),
|
|
1107
1288
|
tail: Promise.resolve(),
|
|
1289
|
+
traceTail: Promise.resolve(),
|
|
1290
|
+
tracePending: 0,
|
|
1291
|
+
traceDrain: Promise.resolve(),
|
|
1292
|
+
resolveTraceDrain: () => undefined,
|
|
1293
|
+
gate: new PriorityGate(),
|
|
1108
1294
|
};
|
|
1109
1295
|
state.ready = this.prepareSession(state);
|
|
1110
1296
|
this.sessions.set(sessionId, state);
|
|
1297
|
+
if (this.linkedParentSession) {
|
|
1298
|
+
// Child of a live parent session: the parent owns the workspace
|
|
1299
|
+
// collection (baseline/changes/end). This collector only traces.
|
|
1300
|
+
state.started = true;
|
|
1301
|
+
return;
|
|
1302
|
+
}
|
|
1111
1303
|
this.enqueue(state, async () => {
|
|
1112
1304
|
await state.ready;
|
|
1113
1305
|
const baselinePaths = await listWorkspaceFiles(root);
|
|
@@ -1124,7 +1316,20 @@ export class WorkspaceCollector {
|
|
|
1124
1316
|
}
|
|
1125
1317
|
}
|
|
1126
1318
|
state.baselineSeeded = true;
|
|
1319
|
+
if (state.resumed && state.baselineUploaded) {
|
|
1320
|
+
// Resume of a session whose start baseline already shipped: do
|
|
1321
|
+
// NOT re-upload the full tree. A restart-heavy session would
|
|
1322
|
+
// otherwise re-burn the entire baseline (and starve traces) on
|
|
1323
|
+
// every restart — observed live as 846 start snapshots /
|
|
1324
|
+
// multiple GB for one session. Changes + the end snapshot cover
|
|
1325
|
+
// the deltas; a FAILED baseline (flag unset) still retries here.
|
|
1326
|
+
state.started = true;
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1127
1329
|
await this.uploadWorkspace(state, "start");
|
|
1330
|
+
state.baselineUploaded = true;
|
|
1331
|
+
state.baselineResumePath = null;
|
|
1332
|
+
await this.persistSession(state);
|
|
1128
1333
|
state.started = true;
|
|
1129
1334
|
});
|
|
1130
1335
|
// Change detection: signature scan (unref'd, self-serializing). No
|
|
@@ -1185,7 +1390,7 @@ export class WorkspaceCollector {
|
|
|
1185
1390
|
if (state.trace.length >= TRACE_RELIEF_EVENTS) {
|
|
1186
1391
|
// Memory relief, NOT truncation: the whole batch is uploaded.
|
|
1187
1392
|
const batch = state.trace.splice(0);
|
|
1188
|
-
this.
|
|
1393
|
+
this.enqueueTrace(state, async () => {
|
|
1189
1394
|
await state.ready;
|
|
1190
1395
|
await this.uploadTrace(state, batch);
|
|
1191
1396
|
});
|
|
@@ -1196,17 +1401,31 @@ export class WorkspaceCollector {
|
|
|
1196
1401
|
const state = this.sessions.get(sessionId);
|
|
1197
1402
|
if (!state || state.finished) return;
|
|
1198
1403
|
const pendingTrace = state.trace.splice(0);
|
|
1404
|
+
if (finalTrace !== undefined) {
|
|
1405
|
+
pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
|
|
1406
|
+
}
|
|
1199
1407
|
state.finished = true;
|
|
1200
1408
|
if (state.changeTimer) clearTimeout(state.changeTimer);
|
|
1201
1409
|
if (state.scanTimer) clearInterval(state.scanTimer);
|
|
1202
|
-
|
|
1410
|
+
// Final trace rides the PRIORITY lane: an in-flight baseline part
|
|
1411
|
+
// finishes first, then the trace ships — never behind the tail of a
|
|
1412
|
+
// multi-part tree upload.
|
|
1413
|
+
this.enqueueTrace(state, async () => {
|
|
1203
1414
|
await state.ready;
|
|
1204
1415
|
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
|
|
1416
|
+
await this.uploadTrace(state, pendingTrace);
|
|
1417
|
+
});
|
|
1418
|
+
this.enqueue(state, async () => {
|
|
1419
|
+
await state.ready;
|
|
1205
1420
|
await state.changeCaptureTail;
|
|
1206
|
-
if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "session.completed", data: finalTrace });
|
|
1207
1421
|
await this.teardownJournal(state); // ship unacked change records, then unlink
|
|
1208
|
-
|
|
1209
|
-
|
|
1422
|
+
// The end snapshot yields to pending trace work (yieldToTrace in
|
|
1423
|
+
// the packer), so the final trace lands first.
|
|
1424
|
+
if (!this.linkedParentSession) {
|
|
1425
|
+
// A linked child ships no end snapshot: the parent session's
|
|
1426
|
+
// collection covers the workspace tree.
|
|
1427
|
+
await this.uploadWorkspace(state, "end");
|
|
1428
|
+
}
|
|
1210
1429
|
this.sessions.delete(sessionId);
|
|
1211
1430
|
});
|
|
1212
1431
|
}
|
|
@@ -1217,7 +1436,10 @@ export class WorkspaceCollector {
|
|
|
1217
1436
|
const pendingTrace = state.trace.splice(0);
|
|
1218
1437
|
if (finalTrace !== undefined) pendingTrace.push({ at: new Date().toISOString(), type: "turn.completed", data: finalTrace });
|
|
1219
1438
|
if (pendingTrace.length === 0) return;
|
|
1220
|
-
|
|
1439
|
+
// Per-turn flush: the complete trace artifact (events + transcript
|
|
1440
|
+
// messages-so-far) ships on the priority lane, ahead of any queued
|
|
1441
|
+
// baseline parts.
|
|
1442
|
+
this.enqueueTrace(state, async () => {
|
|
1221
1443
|
await state.ready;
|
|
1222
1444
|
if (state.trace.length > 0) pendingTrace.push(...state.trace.splice(0));
|
|
1223
1445
|
await this.uploadTrace(state, pendingTrace);
|
|
@@ -1226,7 +1448,45 @@ export class WorkspaceCollector {
|
|
|
1226
1448
|
|
|
1227
1449
|
async stop(): Promise<void> {
|
|
1228
1450
|
for (const sessionId of [...this.sessions.keys()]) this.finishSession(sessionId);
|
|
1229
|
-
await Promise.allSettled(
|
|
1451
|
+
await Promise.allSettled(
|
|
1452
|
+
[...this.sessions.values()].flatMap((state) => [state.tail, state.traceTail]),
|
|
1453
|
+
);
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
/**
|
|
1457
|
+
* Queue trace work on the priority lane and mark it pending: the
|
|
1458
|
+
* background lane's `yieldToTrace` waits for `tracePending` to drain
|
|
1459
|
+
* before starting another baseline part, so trace work enqueued while
|
|
1460
|
+
* a baseline part was in flight goes next.
|
|
1461
|
+
*/
|
|
1462
|
+
private enqueueTrace(state: SessionState, operation: () => Promise<void>): void {
|
|
1463
|
+
if (state.tracePending === 0) {
|
|
1464
|
+
let resolveDrain!: () => void;
|
|
1465
|
+
state.traceDrain = new Promise<void>((resolvePromise) => {
|
|
1466
|
+
resolveDrain = resolvePromise;
|
|
1467
|
+
});
|
|
1468
|
+
state.resolveTraceDrain = () => resolveDrain();
|
|
1469
|
+
}
|
|
1470
|
+
state.tracePending += 1;
|
|
1471
|
+
state.traceTail = state.traceTail
|
|
1472
|
+
.then(operation)
|
|
1473
|
+
.catch((error: unknown) => {
|
|
1474
|
+
this.log("warn", "OmniRush collection operation failed", {
|
|
1475
|
+
sessionId: state.id,
|
|
1476
|
+
error: error instanceof Error ? error.message : "unknown",
|
|
1477
|
+
});
|
|
1478
|
+
})
|
|
1479
|
+
.finally(() => {
|
|
1480
|
+
state.tracePending -= 1;
|
|
1481
|
+
if (state.tracePending === 0) state.resolveTraceDrain();
|
|
1482
|
+
});
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/** Wait until no trace work is queued or running (re-arming safe). */
|
|
1486
|
+
private async yieldToTrace(state: SessionState): Promise<void> {
|
|
1487
|
+
while (state.tracePending > 0) {
|
|
1488
|
+
await state.traceDrain;
|
|
1489
|
+
}
|
|
1230
1490
|
}
|
|
1231
1491
|
|
|
1232
1492
|
private stopWatching(state: SessionState): void {
|
|
@@ -1545,23 +1805,38 @@ export class WorkspaceCollector {
|
|
|
1545
1805
|
};
|
|
1546
1806
|
}
|
|
1547
1807
|
|
|
1548
|
-
/** Full-tree snapshot ("start" | "end"), streamed into ordered parts.
|
|
1808
|
+
/** Full-tree snapshot ("start" | "end"), streamed into ordered parts.
|
|
1809
|
+
* A start snapshot resumed after an interruption skips the paths the
|
|
1810
|
+
* previous run already shipped (`baselineResumePath` cursor — the
|
|
1811
|
+
* first unacked part onward); the end snapshot always ships whole. */
|
|
1549
1812
|
private async uploadTreeParts(state: SessionState, snapshotType: "start" | "end"): Promise<void> {
|
|
1550
1813
|
const paths = (await listWorkspaceFiles(state.root)).sort(comparePaths);
|
|
1551
1814
|
const metadata = await this.workspaceMetadataFile(state);
|
|
1552
1815
|
let metadataSent = false;
|
|
1553
|
-
const
|
|
1816
|
+
const resumeFrom = snapshotType === "start" ? state.baselineResumePath : null;
|
|
1554
1817
|
const measure = async (batch: CollectorFile[]) => {
|
|
1555
1818
|
const files = metadataSent ? batch : [metadata, ...batch];
|
|
1556
|
-
const payload = buildEnvelopePayload(
|
|
1819
|
+
const payload = buildEnvelopePayload(this.sequenceState(state), snapshotType, files);
|
|
1557
1820
|
return { files, payload, compressed: await compressZstd(payload) };
|
|
1558
1821
|
};
|
|
1559
|
-
await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths), {
|
|
1822
|
+
await this.packAndUpload(state, snapshotType, this.iterTreeFiles(state, paths, resumeFrom), {
|
|
1560
1823
|
sizeOf: (file) => Buffer.byteLength(file.content ?? "") + file.path.length + 64,
|
|
1561
1824
|
measure,
|
|
1562
1825
|
upload: async (measured) => {
|
|
1563
1826
|
const ok = await this.uploadEnvelope(state, snapshotType, measured.files, measured);
|
|
1564
|
-
if (ok)
|
|
1827
|
+
if (ok) {
|
|
1828
|
+
metadataSent = true;
|
|
1829
|
+
if (snapshotType === "start") {
|
|
1830
|
+
// Advance the resume cursor to the last path of the acked
|
|
1831
|
+
// part, so an interrupted run resumes at the first
|
|
1832
|
+
// unacked part instead of re-burning the prefix.
|
|
1833
|
+
const lastPath = measured.files[measured.files.length - 1]?.path;
|
|
1834
|
+
if (typeof lastPath === "string") {
|
|
1835
|
+
state.baselineResumePath = lastPath;
|
|
1836
|
+
await this.persistSession(state).catch(() => undefined);
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1565
1840
|
return ok;
|
|
1566
1841
|
},
|
|
1567
1842
|
drop: (file) => {
|
|
@@ -1576,12 +1851,17 @@ export class WorkspaceCollector {
|
|
|
1576
1851
|
data: { path: file.path, bytes: Buffer.byteLength(file.content ?? "") },
|
|
1577
1852
|
});
|
|
1578
1853
|
},
|
|
1579
|
-
});
|
|
1854
|
+
}, { yieldToTraceLane: true });
|
|
1580
1855
|
}
|
|
1581
1856
|
|
|
1582
|
-
private async *iterTreeFiles(
|
|
1857
|
+
private async *iterTreeFiles(
|
|
1858
|
+
state: SessionState,
|
|
1859
|
+
paths: string[],
|
|
1860
|
+
resumeFrom?: string | null,
|
|
1861
|
+
): AsyncGenerator<CollectorFile> {
|
|
1583
1862
|
for (const path of paths) {
|
|
1584
1863
|
if (state.budgetExhausted) return;
|
|
1864
|
+
if (resumeFrom && comparePaths(path, resumeFrom) <= 0) continue;
|
|
1585
1865
|
for await (const entry of iterateWorkspaceEntries(state.root, path)) {
|
|
1586
1866
|
yield entry;
|
|
1587
1867
|
}
|
|
@@ -1655,13 +1935,14 @@ export class WorkspaceCollector {
|
|
|
1655
1935
|
});
|
|
1656
1936
|
return files;
|
|
1657
1937
|
};
|
|
1658
|
-
const bySequence = () => (
|
|
1938
|
+
const bySequence = (capturedAt?: string) => this.sequenceState(state, capturedAt);
|
|
1659
1939
|
await this.packAndUpload(state, "change", units, {
|
|
1660
1940
|
sizeOf: (entry) =>
|
|
1661
1941
|
Buffer.byteLength(entry.content ?? "") + entry.path.length + 96,
|
|
1662
1942
|
measure: async (batch) => {
|
|
1663
1943
|
const files = filesForSlice(batch);
|
|
1664
|
-
const
|
|
1944
|
+
const capturedAt = batch.find((entry) => typeof entry.at === "string")?.at;
|
|
1945
|
+
const payload = buildEnvelopePayload(bySequence(capturedAt), "change", files);
|
|
1665
1946
|
return { files, payload, compressed: await compressZstd(payload) };
|
|
1666
1947
|
},
|
|
1667
1948
|
upload: async (measured, batch) => {
|
|
@@ -1684,7 +1965,7 @@ export class WorkspaceCollector {
|
|
|
1684
1965
|
data: { path: entry.path, reason: "change_record_over_limit" },
|
|
1685
1966
|
});
|
|
1686
1967
|
},
|
|
1687
|
-
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX });
|
|
1968
|
+
}, { checkBytes: PART_SIDECAR_PLAIN_MAX, flushPlainBytes: PART_SIDECAR_PLAIN_MAX, yieldToTraceLane: true });
|
|
1688
1969
|
return uploadedSlices;
|
|
1689
1970
|
}
|
|
1690
1971
|
|
|
@@ -1697,6 +1978,11 @@ export class WorkspaceCollector {
|
|
|
1697
1978
|
* corresponds to the (mutated) batch passed to hooks.upload, and popped
|
|
1698
1979
|
* units come back as `carry` to lead the next part. hooks.upload must
|
|
1699
1980
|
* persist sequence bookkeeping via uploadEnvelope.
|
|
1981
|
+
*
|
|
1982
|
+
* `yieldToTraceLane` (baseline/change/end packs): before each part
|
|
1983
|
+
* upload, wait while trace work is pending or running — a trace part
|
|
1984
|
+
* then never queues behind more than the one baseline part already in
|
|
1985
|
+
* flight. Trace packs must NOT set it (they ARE the priority lane).
|
|
1700
1986
|
*/
|
|
1701
1987
|
private async packAndUpload<U>(
|
|
1702
1988
|
state: SessionState,
|
|
@@ -1708,7 +1994,7 @@ export class WorkspaceCollector {
|
|
|
1708
1994
|
upload: (measured: { files: CollectorFile[]; payload: Buffer; compressed: Buffer }, batch: U[]) => Promise<boolean>;
|
|
1709
1995
|
drop?: (unit: U) => void;
|
|
1710
1996
|
},
|
|
1711
|
-
opts: { checkBytes?: number; flushPlainBytes?: number } = {},
|
|
1997
|
+
opts: { checkBytes?: number; flushPlainBytes?: number; yieldToTraceLane?: boolean } = {},
|
|
1712
1998
|
): Promise<void> {
|
|
1713
1999
|
const checkBytes = Math.max(1024, opts.checkBytes ?? this.partCheckBytes);
|
|
1714
2000
|
const flushPlainBytes = Math.max(checkBytes, opts.flushPlainBytes ?? PART_BATCH_UNCOMPRESSED_MAX);
|
|
@@ -1750,6 +2036,7 @@ export class WorkspaceCollector {
|
|
|
1750
2036
|
if (batchBytes < nextCheck) continue;
|
|
1751
2037
|
const { measured, carry } = await fit(batch);
|
|
1752
2038
|
if (measured && shouldFlush(measured)) {
|
|
2039
|
+
if (opts.yieldToTraceLane) await this.yieldToTrace(state);
|
|
1753
2040
|
if (!(await hooks.upload(measured, batch))) return;
|
|
1754
2041
|
resetBatch(carry);
|
|
1755
2042
|
} else if (measured) {
|
|
@@ -1764,6 +2051,7 @@ export class WorkspaceCollector {
|
|
|
1764
2051
|
while (batch.length > 0 && !state.budgetExhausted) {
|
|
1765
2052
|
const { measured, carry } = await fit(batch);
|
|
1766
2053
|
if (measured) {
|
|
2054
|
+
if (opts.yieldToTraceLane) await this.yieldToTrace(state);
|
|
1767
2055
|
if (!(await hooks.upload(measured, batch))) return;
|
|
1768
2056
|
}
|
|
1769
2057
|
if (carry.length === 0) break;
|
|
@@ -1798,7 +2086,7 @@ export class WorkspaceCollector {
|
|
|
1798
2086
|
});
|
|
1799
2087
|
}
|
|
1800
2088
|
let partIndex = 0;
|
|
1801
|
-
const bySequence = () => (
|
|
2089
|
+
const bySequence = () => this.sequenceState(state);
|
|
1802
2090
|
await this.packAndUpload(state, "trace", selected.events, {
|
|
1803
2091
|
sizeOf: (event) => Buffer.byteLength(JSON.stringify(event)) + 32,
|
|
1804
2092
|
measure: async (batch) => {
|
|
@@ -1899,10 +2187,16 @@ export class WorkspaceCollector {
|
|
|
1899
2187
|
state.transcripts[filePath] = { size, mtimeMs };
|
|
1900
2188
|
continue;
|
|
1901
2189
|
}
|
|
2190
|
+
if (this.linkedParentSession && artifactSessionId !== state.id) {
|
|
2191
|
+
// Linked child: the parent's collector already ships the other
|
|
2192
|
+
// session files in this directory (linked to the parent) — a
|
|
2193
|
+
// second copy from the child adds nothing.
|
|
2194
|
+
continue;
|
|
2195
|
+
}
|
|
1902
2196
|
const identity = await this.resolveIdentity();
|
|
1903
2197
|
const artifact = buildTraceArtifact({
|
|
1904
2198
|
sessionId: artifactSessionId,
|
|
1905
|
-
parentSessionId: state.id,
|
|
2199
|
+
parentSessionId: this.linkedParentSession ?? state.id,
|
|
1906
2200
|
userId: identity?.userId ?? null,
|
|
1907
2201
|
clientId: this.clientId ?? null,
|
|
1908
2202
|
agentId: "omnirush-cli",
|
|
@@ -1922,7 +2216,7 @@ export class WorkspaceCollector {
|
|
|
1922
2216
|
await this.persistSession(state).catch(() => undefined);
|
|
1923
2217
|
return;
|
|
1924
2218
|
}
|
|
1925
|
-
const bySequence = () => (
|
|
2219
|
+
const bySequence = () => this.sequenceState(state);
|
|
1926
2220
|
await this.packAndUpload(state, "trace", artifacts, {
|
|
1927
2221
|
sizeOf: (artifact) => Buffer.byteLength(artifact.content) + artifact.path.length + 64,
|
|
1928
2222
|
measure: async (batch) => {
|
|
@@ -1966,6 +2260,15 @@ export class WorkspaceCollector {
|
|
|
1966
2260
|
): Promise<boolean> {
|
|
1967
2261
|
if (!this.collectUrl && !this.uploader) return false;
|
|
1968
2262
|
if (state.budgetExhausted) return false;
|
|
2263
|
+
if (files.length === 0) {
|
|
2264
|
+
// A zero-file part carries no information and would loop forever
|
|
2265
|
+
// if a caller ever fed a packer an empty unit stream.
|
|
2266
|
+
this.log("warn", "OmniRush refused to upload an empty envelope", {
|
|
2267
|
+
sessionId: state.id,
|
|
2268
|
+
snapshotType,
|
|
2269
|
+
});
|
|
2270
|
+
return false;
|
|
2271
|
+
}
|
|
1969
2272
|
const payload = precompressed?.payload ?? buildEnvelopePayload(state, snapshotType, files);
|
|
1970
2273
|
if (state.sentBytes + payload.length > this.sessionBudgetBytes) {
|
|
1971
2274
|
this.exhaustSessionBudget(state);
|
|
@@ -1981,30 +2284,44 @@ export class WorkspaceCollector {
|
|
|
1981
2284
|
});
|
|
1982
2285
|
return false;
|
|
1983
2286
|
}
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
2287
|
+
// Two lanes (session tail + trace tail) funnel every envelope send
|
|
2288
|
+
// through this gate. Trace sends acquire with priority, so after the
|
|
2289
|
+
// part currently in flight finishes, the trace part goes next; the
|
|
2290
|
+
// gate is held per ATTEMPT, so a retrying baseline part cannot pin
|
|
2291
|
+
// the transport either.
|
|
2292
|
+
const priority = snapshotType === "trace";
|
|
2293
|
+
const gate = state.gate ?? this.defaultGate;
|
|
2294
|
+
const sendOnce = async () => {
|
|
2295
|
+
await gate.acquire(priority);
|
|
2296
|
+
try {
|
|
2297
|
+
return await (this.uploader
|
|
2298
|
+
? this.uploader(state.id, compressed)
|
|
2299
|
+
: this.fetcher(this.collectUrl!, {
|
|
2300
|
+
method: "POST",
|
|
2301
|
+
headers: {
|
|
2302
|
+
Authorization: `Bearer ${this.token}`,
|
|
2303
|
+
"Content-Type": "application/zstd",
|
|
2304
|
+
"X-Omnirush-Session-ID": state.id,
|
|
2305
|
+
},
|
|
2306
|
+
body: compressed.buffer.slice(compressed.byteOffset, compressed.byteOffset + compressed.byteLength) as ArrayBuffer,
|
|
2307
|
+
signal: AbortSignal.timeout(120_000),
|
|
2308
|
+
}));
|
|
2309
|
+
} finally {
|
|
2310
|
+
gate.release();
|
|
2311
|
+
}
|
|
2312
|
+
};
|
|
1996
2313
|
// Shared retry layer: 429/5xx/network ride out backend deploys with
|
|
1997
2314
|
// exponential backoff + jitter. 401 stays a protocol answer handled
|
|
1998
2315
|
// inline (single-flight refresh, retry exactly once).
|
|
1999
2316
|
const outcome = await withRetries(
|
|
2000
2317
|
async () => {
|
|
2001
|
-
let response = await
|
|
2318
|
+
let response = await sendOnce();
|
|
2002
2319
|
if (response.status === 401 && this.refresh) {
|
|
2003
2320
|
// Broker pattern: single-flight refresh, retry exactly once.
|
|
2004
2321
|
const rotated = await this.refresh(this.token).catch(() => null);
|
|
2005
2322
|
if (rotated) {
|
|
2006
2323
|
this.token = rotated;
|
|
2007
|
-
response = await
|
|
2324
|
+
response = await sendOnce();
|
|
2008
2325
|
}
|
|
2009
2326
|
}
|
|
2010
2327
|
return {
|