omnirush 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/CHANGELOG.md +35 -0
- package/assets/extensions/omnirush/agents-lib.ts +421 -66
- package/assets/extensions/omnirush/agents.ts +92 -11
- package/assets/extensions/omnirush/bgshell-lib.ts +32 -0
- package/assets/extensions/omnirush/bgshell.ts +9 -1
- package/assets/extensions/omnirush/capture/session-archive/index.ts +5 -12
- package/assets/extensions/omnirush/capture/session-archive/touched.ts +2 -7
- package/assets/extensions/omnirush/capture/workspace-collector.ts +87 -24
- package/assets/extensions/omnirush/collector.ts +203 -32
- package/assets/extensions/omnirush/index.ts +22 -8
- package/assets/extensions/omnirush/mcp.ts +105 -9
- package/assets/extensions/omnirush/memory-lib.ts +516 -0
- package/assets/extensions/omnirush/pi-engine.ts +50 -2
- package/assets/extensions/omnirush/plan-lib.ts +38 -9
- package/assets/extensions/omnirush/plan.ts +85 -30
- package/assets/extensions/omnirush/secret-env.ts +15 -0
- package/assets/extensions/omnirush/subagent-marker.ts +21 -0
- package/assets/extensions/omnirush/subagents-lib.ts +5 -0
- package/package.json +2 -2
- package/scripts/__pycache__/build-all-packages.cpython-38.pyc +0 -0
- package/scripts/__pycache__/smoke-packages.cpython-38.pyc +0 -0
- package/scripts/smoke-packages.py +1 -0
- package/src/bin.js +130 -23
- package/src/lib.js +7 -2
- package/src/sessions.js +222 -0
|
@@ -4,7 +4,11 @@
|
|
|
4
4
|
// the SAME workspace:
|
|
5
5
|
// - role presets: code-searcher (read-only exploration),
|
|
6
6
|
// researcher-web (web_fetch research), general-worker (full tools)
|
|
7
|
-
// - parallel execution, every task at once unless max_parallel throttles
|
|
7
|
+
// - parallel execution, every task at once unless max_parallel throttles;
|
|
8
|
+
// no count cap, but memory-aware admission (memory-lib.ts): a child
|
|
9
|
+
// starts while the machine keeps enough RAM free and the agents' process
|
|
10
|
+
// tree stays under its budget, the rest wait ("queued: waiting for
|
|
11
|
+
// memory") and start as memory frees up
|
|
8
12
|
// - no wall-clock limit: a child runs until it is done. Only an explicit
|
|
9
13
|
// timeout_minutes, the inactivity watchdog (no output at all for
|
|
10
14
|
// OMNIRUSH_AGENT_INACTIVITY_MINUTES, default 30, longer while one of
|
|
@@ -20,7 +24,8 @@
|
|
|
20
24
|
// running children
|
|
21
25
|
//
|
|
22
26
|
// Tracing is free: every child writes its own session JSONL under
|
|
23
|
-
// <
|
|
27
|
+
// <session dir>/subagents/ (children run without --no-session; that
|
|
28
|
+
// folder keeps them out of /resume and --continue)
|
|
24
29
|
// under a session id the parent picks, and the parent session's
|
|
25
30
|
// collector records each child's conversation as a sub-agent of the
|
|
26
31
|
// turn (session.child events with parent and depth, as the desktop
|
|
@@ -29,6 +34,11 @@
|
|
|
29
34
|
// stopped and recorded as interrupted. Children get
|
|
30
35
|
// OMNIRUSH_PARENT_SESSION, so a child process uploads nothing of its own.
|
|
31
36
|
//
|
|
37
|
+
// Nesting: a sub-agent may start sub-agents of its own, up to
|
|
38
|
+
// MAX_SUBAGENT_DEPTH (3) layers below the main session (the desktop's
|
|
39
|
+
// OMNIRUSH_SUBAGENT_DEPTH). Each child gets OMNIRUSH_AGENT_DEPTH (its layer)
|
|
40
|
+
// and a note saying it; layer 3 has no spawn_agents.
|
|
41
|
+
//
|
|
32
42
|
// `/agents` lists this session's subagents (running + finished);
|
|
33
43
|
// `/agents cancel <id|all>` stops running ones.
|
|
34
44
|
|
|
@@ -46,16 +56,25 @@ import {
|
|
|
46
56
|
type AgentRunner,
|
|
47
57
|
type AgentsHook,
|
|
48
58
|
type AgentRecord,
|
|
59
|
+
agentDepth,
|
|
60
|
+
canDelegate,
|
|
49
61
|
childInactivityMs,
|
|
50
62
|
renderAgentStatus,
|
|
63
|
+
subagentLayerNote,
|
|
51
64
|
renderChildResults,
|
|
52
65
|
renderDelivery,
|
|
53
66
|
runChildAgent,
|
|
54
67
|
childModelFallback,
|
|
68
|
+
subagentSessionDir,
|
|
55
69
|
type ChildResult,
|
|
56
70
|
type ChildTask,
|
|
57
71
|
} from "./agents-lib";
|
|
58
72
|
import { SWARM_GUIDELINE } from "./swarm-lib";
|
|
73
|
+
import { omniDir } from "./auth";
|
|
74
|
+
import { MemoryAdmission, readMemorySettings } from "./memory-lib";
|
|
75
|
+
|
|
76
|
+
/** Blocking spawn_agents: progress updates at most this often (ms). */
|
|
77
|
+
const PROGRESS_UPDATE_MS = 500;
|
|
59
78
|
|
|
60
79
|
/** A timeout_minutes below this is taken for a placeholder, not a limit. */
|
|
61
80
|
const MIN_TIMEOUT_MINUTES = 1;
|
|
@@ -109,7 +128,19 @@ function renderSome(records: AgentRecord[], unknown: string[] = []): string {
|
|
|
109
128
|
return parts.join("\n\n") || "No matching sub-agents.";
|
|
110
129
|
}
|
|
111
130
|
|
|
112
|
-
export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {}) {
|
|
131
|
+
export default function (pi: ExtensionAPI, options: { run?: AgentRunner; admission?: MemoryAdmission | null; env?: NodeJS.ProcessEnv } = {}) {
|
|
132
|
+
// Sub-agents nest up to MAX_SUBAGENT_DEPTH layers below the main session,
|
|
133
|
+
// as in the desktop app: every sub-agent is told its layer, and the last
|
|
134
|
+
// layer gets no spawn_agents (nor the tools that manage sub-agents).
|
|
135
|
+
const depth = agentDepth(options.env ?? process.env);
|
|
136
|
+
if (depth > 0) {
|
|
137
|
+
pi.on("before_agent_start", async (event: any) => {
|
|
138
|
+
const base = typeof event?.systemPrompt === "string" ? event.systemPrompt : "";
|
|
139
|
+
return { systemPrompt: `${base}\n\n${subagentLayerNote(depth)}` };
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
if (!canDelegate(depth)) return;
|
|
143
|
+
|
|
113
144
|
const watchdogMinutes = Math.round(childInactivityMs() / 60_000);
|
|
114
145
|
const hub = deliveryHub(pi);
|
|
115
146
|
/** The last context seen (the TUI status bar for running agents). */
|
|
@@ -130,18 +161,49 @@ export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {})
|
|
|
130
161
|
statusTimer.unref?.();
|
|
131
162
|
}
|
|
132
163
|
|
|
164
|
+
// Memory-aware admission of children; thresholds from the environment or
|
|
165
|
+
// settings.json "subagentMemory" (memory-lib.ts). Built on first use.
|
|
166
|
+
let admission: MemoryAdmission | null = null;
|
|
167
|
+
const admissionOf = (): MemoryAdmission => {
|
|
168
|
+
admission ??= options.admission ?? new MemoryAdmission({ settings: readMemorySettings(omniDir()) });
|
|
169
|
+
return admission;
|
|
170
|
+
};
|
|
171
|
+
const lazyAdmission = {
|
|
172
|
+
check: () => admissionOf().check(),
|
|
173
|
+
started: (key: object, pid?: number | null) => admissionOf().started(key, pid ?? null),
|
|
174
|
+
spawned: (key: object, pid: number | null | undefined) => admissionOf().spawned(key, pid),
|
|
175
|
+
finished: (key: object) => admissionOf().finished(key),
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
/** Tell the user when children wait for memory (a warning when it is critically low). */
|
|
179
|
+
function memoryWait(info: { critical: boolean; detail: string; queued: number } | null): void {
|
|
180
|
+
scheduleStatus();
|
|
181
|
+
if (!info) return;
|
|
182
|
+
const text = info.critical
|
|
183
|
+
? `omnirush: ${info.detail}. Not starting more sub-agents until memory frees up (${info.queued} queued); running ones continue.`
|
|
184
|
+
: `omnirush: ${info.queued} sub-agent${info.queued === 1 ? "" : "s"} queued: waiting for memory (${info.detail}).`;
|
|
185
|
+
if (lastContext?.hasUI && lastContext?.ui?.notify) lastContext.ui.notify(text, info.critical ? "warning" : "info");
|
|
186
|
+
// Headless: the top agent says it on stderr (a sub-agent's stderr only reaches its parent).
|
|
187
|
+
else if (info.critical || !String(process.env.OMNIRUSH_PARENT_SESSION ?? "").trim()) process.stderr.write(`${text}\n`);
|
|
188
|
+
}
|
|
189
|
+
|
|
133
190
|
const manager = new AgentManager({
|
|
134
191
|
run: options.run ?? ((task, options) => runChildAgent(task, {
|
|
135
192
|
cwd: options.cwd ?? hub.cwd,
|
|
136
193
|
parentSessionId: options.parentSessionId,
|
|
137
194
|
sessionId: options.sessionId,
|
|
195
|
+
...(options.sessionDir ? { sessionDir: options.sessionDir } : {}),
|
|
138
196
|
signal: options.signal,
|
|
139
197
|
onActivity: options.onActivity,
|
|
198
|
+
...(options.onSpawn ? { onSpawn: options.onSpawn } : {}),
|
|
199
|
+
extraEnv: admissionOf().childEnv(),
|
|
140
200
|
...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
|
|
141
201
|
})),
|
|
142
202
|
// After the settle's own bookkeeping (waiters took theirs).
|
|
143
203
|
onDeliverable: () => hub.notify(),
|
|
144
204
|
onChanged: scheduleStatus,
|
|
205
|
+
admission: options.admission === null ? null : lazyAdmission,
|
|
206
|
+
onMemoryWait: memoryWait,
|
|
145
207
|
});
|
|
146
208
|
|
|
147
209
|
hub.addSource({
|
|
@@ -231,6 +293,8 @@ export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {})
|
|
|
231
293
|
? Math.floor(params.max_parallel)
|
|
232
294
|
: undefined;
|
|
233
295
|
const background = params.wait === false;
|
|
296
|
+
let progressTimer: ReturnType<typeof setTimeout> | null = null;
|
|
297
|
+
let finished = false;
|
|
234
298
|
|
|
235
299
|
const dispatched = manager.dispatch(parentSessionId, resolvedTasks, {
|
|
236
300
|
background,
|
|
@@ -245,14 +309,24 @@ export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {})
|
|
|
245
309
|
});
|
|
246
310
|
},
|
|
247
311
|
onChanged: () => {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
312
|
+
// Every sign of life of every child lands here: the progress update
|
|
313
|
+
// (which carries the finished results) goes out at most every
|
|
314
|
+
// PROGRESS_UPDATE_MS, not once per output chunk.
|
|
315
|
+
if (background || progressTimer) return;
|
|
316
|
+
progressTimer = setTimeout(() => {
|
|
317
|
+
progressTimer = null;
|
|
318
|
+
if (finished) return;
|
|
319
|
+
onUpdate?.({
|
|
320
|
+
content: [{ type: "text", text: agentStatusSummary(manager.list(parentSessionId)) }],
|
|
321
|
+
details: { results: manager.list(parentSessionId).filter((candidate) => candidate.result).map(resultOf) },
|
|
322
|
+
});
|
|
323
|
+
}, PROGRESS_UPDATE_MS);
|
|
324
|
+
progressTimer.unref?.();
|
|
253
325
|
},
|
|
254
326
|
...(timeoutMs !== undefined ? { timeoutMs } : {}),
|
|
255
327
|
cwd: workdir,
|
|
328
|
+
// Children's sessions stay out of the user's /resume list.
|
|
329
|
+
sessionDir: subagentSessionDir(ctx?.sessionManager?.getSessionDir?.()),
|
|
256
330
|
});
|
|
257
331
|
|
|
258
332
|
if (background) {
|
|
@@ -295,6 +369,8 @@ export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {})
|
|
|
295
369
|
}
|
|
296
370
|
|
|
297
371
|
await dispatched.done;
|
|
372
|
+
finished = true;
|
|
373
|
+
if (progressTimer) clearTimeout(progressTimer);
|
|
298
374
|
const records = dispatched.records;
|
|
299
375
|
return {
|
|
300
376
|
content: [{ type: "text", text: renderChildResults(records.map((record) => record.result as ChildResult), records.map((record) => record.id)) }],
|
|
@@ -404,7 +480,7 @@ export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {})
|
|
|
404
480
|
});
|
|
405
481
|
|
|
406
482
|
pi.registerCommand("agents", {
|
|
407
|
-
description: "List this session's subagents (running and finished); /agents cancel <id|all> stops running ones",
|
|
483
|
+
description: "List this session's subagents (running and finished) and the memory they may use; /agents cancel <id|all> stops running ones; /agents memory shows the memory admission",
|
|
408
484
|
handler: async (args: string, ctx: any) => {
|
|
409
485
|
const session = sessionOf(ctx);
|
|
410
486
|
const words = String(args ?? "").trim().split(/\s+/).filter(Boolean);
|
|
@@ -416,11 +492,16 @@ export default function (pi: ExtensionAPI, options: { run?: AgentRunner } = {})
|
|
|
416
492
|
return;
|
|
417
493
|
}
|
|
418
494
|
const records = manager.list(session);
|
|
495
|
+
const memory = options.admission === null ? "" : `\n${admissionOf().describe()}`;
|
|
496
|
+
if (words[0] === "memory") {
|
|
497
|
+
ctx?.ui?.notify?.(memory.trim() || "Memory admission is off.");
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
419
500
|
if (records.length === 0) {
|
|
420
|
-
ctx?.ui?.notify?.(
|
|
501
|
+
ctx?.ui?.notify?.(`No subagents have been spawned in this session yet.${memory}`);
|
|
421
502
|
return;
|
|
422
503
|
}
|
|
423
|
-
ctx?.ui?.notify?.(renderAgentStatus(records));
|
|
504
|
+
ctx?.ui?.notify?.(`${renderAgentStatus(records)}${memory}`);
|
|
424
505
|
},
|
|
425
506
|
});
|
|
426
507
|
}
|
|
@@ -21,6 +21,12 @@ export const MAX_FOREGROUND_SECONDS = 3_600;
|
|
|
21
21
|
export const JOB_BUFFER_CHARS = 1_000_000;
|
|
22
22
|
/** Output a delivery or a read shows at most (characters): the newest part. */
|
|
23
23
|
export const SHOW_CHARS = 30_000;
|
|
24
|
+
/**
|
|
25
|
+
* Finished jobs whose result the model has seen are kept (for bash_output /
|
|
26
|
+
* status) without their output; beyond this many they are forgotten, oldest
|
|
27
|
+
* first. Every bash call is a job, so a long session has thousands.
|
|
28
|
+
*/
|
|
29
|
+
export const MAX_SETTLED_JOBS = 200;
|
|
24
30
|
/** Longest wait a bash_wait call may ask for (seconds). */
|
|
25
31
|
export const MAX_WAIT_SECONDS = 3_600;
|
|
26
32
|
|
|
@@ -53,6 +59,11 @@ export class OutputBuffer {
|
|
|
53
59
|
if (this.text.length > this.max * 1.25) this.text = this.text.slice(this.text.length - this.max);
|
|
54
60
|
}
|
|
55
61
|
|
|
62
|
+
/** Nothing more will be read: drop the text, keep the counters. */
|
|
63
|
+
release(): void {
|
|
64
|
+
this.text = "";
|
|
65
|
+
}
|
|
66
|
+
|
|
56
67
|
/** Offset of the oldest character still kept. */
|
|
57
68
|
get start(): number {
|
|
58
69
|
return this.total - this.text.length;
|
|
@@ -143,6 +154,7 @@ export class ShellJobManager {
|
|
|
143
154
|
toolCallId?: string | null;
|
|
144
155
|
onData?: (data: Buffer | string) => void;
|
|
145
156
|
}): ShellJob {
|
|
157
|
+
this.prune();
|
|
146
158
|
const controller = new AbortController();
|
|
147
159
|
const job: ShellJob = {
|
|
148
160
|
id: `sh${this.next++}`,
|
|
@@ -209,6 +221,25 @@ export class ShellJobManager {
|
|
|
209
221
|
if (job.background && !job.delivered) this.onDeliverable();
|
|
210
222
|
}
|
|
211
223
|
|
|
224
|
+
/**
|
|
225
|
+
* Memory: finished jobs whose result reached the model keep no output (up
|
|
226
|
+
* to 1.25M characters each otherwise, for the life of the process), and
|
|
227
|
+
* only the newest MAX_SETTLED_JOBS of them are kept at all.
|
|
228
|
+
*/
|
|
229
|
+
prune(): void {
|
|
230
|
+
let settled = 0;
|
|
231
|
+
for (let index = this.jobs.length - 1; index >= 0; index--) {
|
|
232
|
+
const job = this.jobs[index];
|
|
233
|
+
if (job.status === "running" || !job.delivered || (this.awaited.get(job) ?? 0) > 0) continue;
|
|
234
|
+
job.output.release();
|
|
235
|
+
settled += 1;
|
|
236
|
+
if (settled > MAX_SETTLED_JOBS) {
|
|
237
|
+
this.jobs.splice(index, 1);
|
|
238
|
+
this.reminded.delete(job);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
212
243
|
/** Hear the job's (cleaned) output from now on; returns the unsubscribe. */
|
|
213
244
|
listen(job: ShellJob, listener: (text: string) => void): () => void {
|
|
214
245
|
const set = this.listeners.get(job) ?? new Set();
|
|
@@ -304,6 +335,7 @@ export class ShellJobManager {
|
|
|
304
335
|
|
|
305
336
|
/** Finished background jobs whose result the model has not seen (marked delivered). */
|
|
306
337
|
takeDeliverable(session: string): ShellJob[] {
|
|
338
|
+
this.prune();
|
|
307
339
|
const ready = this.list(session).filter((job) => job.background && job.status !== "running" && !job.delivered);
|
|
308
340
|
ready.forEach((job) => { job.delivered = true; });
|
|
309
341
|
return ready;
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
} from "./bgshell-lib";
|
|
43
43
|
import { deliveryHub } from "./deliveries";
|
|
44
44
|
import { BACKGROUND_BASH_TYPE } from "./pi-engine";
|
|
45
|
+
import { sanitizeToolEnvironment } from "./secret-env";
|
|
45
46
|
|
|
46
47
|
/** customType of the message that brings finished background commands back (the trace maps it to bash tool parts). */
|
|
47
48
|
export const BASH_DELIVERY_TYPE = BACKGROUND_BASH_TYPE;
|
|
@@ -221,7 +222,14 @@ export default function (pi: any, options: { exec?: ShellExec; now?: () => numbe
|
|
|
221
222
|
const operations = {
|
|
222
223
|
exec: (command: string, cwd: string, execOptions: any) =>
|
|
223
224
|
new Promise<{ exitCode: number | null }>((resolve, reject) => {
|
|
224
|
-
const current = manager.start({
|
|
225
|
+
const current = manager.start({
|
|
226
|
+
session,
|
|
227
|
+
command,
|
|
228
|
+
cwd,
|
|
229
|
+
env: sanitizeToolEnvironment(execOptions?.env ?? process.env),
|
|
230
|
+
toolCallId,
|
|
231
|
+
onData: execOptions?.onData,
|
|
232
|
+
});
|
|
225
233
|
job = current;
|
|
226
234
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
227
235
|
const onAbort = () => { void manager.kill([current]); };
|
|
@@ -79,10 +79,6 @@ const MAX_JOB_AGE_MS = 7 * 24 * 60 * 60_000;
|
|
|
79
79
|
const JOB_RETRY_BASE_MS = 60_000;
|
|
80
80
|
const JOB_RETRY_MAX_MS = 60 * 60_000;
|
|
81
81
|
const SIGN_OUT_ABORT_TIMEOUT_MS = 5_000;
|
|
82
|
-
/** At app start, sessions active (a base, a prompt, a turn) within this long may get a final archive (startFinalCandidates). */
|
|
83
|
-
const START_FINAL_WINDOW_MS = 7 * 24 * 60 * 60_000;
|
|
84
|
-
/** At most this many sessions are scanned for a final archive at app start. */
|
|
85
|
-
const MAX_START_FINALS = 10;
|
|
86
82
|
/**
|
|
87
83
|
* How long a folder policy answer (4.4: all folders, touched files) is
|
|
88
84
|
* reused for folders without `.git`: sessions started meanwhile send no
|
|
@@ -515,12 +511,9 @@ export class SessionArchiver {
|
|
|
515
511
|
* with a turn captured since its last final archive (the app quit before
|
|
516
512
|
* that final, or crashed), the most recently active session on each
|
|
517
513
|
* other folder (the folder may have changed while the app was closed),
|
|
518
|
-
* and every touched-files session (its files are its own).
|
|
519
|
-
*
|
|
520
|
-
*
|
|
521
|
-
* archive never extends the window, so a chat left alone gets none a week
|
|
522
|
-
* after its last turn, however often the app starts. The most recently
|
|
523
|
-
* active first, at most 10.
|
|
514
|
+
* and every touched-files session (its files are its own). Sessions are
|
|
515
|
+
* recovered until their final archive is settled; there is no age or count
|
|
516
|
+
* cutoff that can silently lose the final workspace state.
|
|
524
517
|
*/
|
|
525
518
|
async startFinalCandidates(): Promise<string[]> {
|
|
526
519
|
try {
|
|
@@ -533,7 +526,7 @@ export class SessionArchiver {
|
|
|
533
526
|
const parsed = sessionStateSchema.safeParse(await readJsonFile(join(this.dirs.sessions, name)));
|
|
534
527
|
if (!parsed.success || name !== `${stateKey(parsed.data.session_id)}.json`) continue;
|
|
535
528
|
const state = parsed.data;
|
|
536
|
-
if (state.stopped || state.ended ||
|
|
529
|
+
if (state.stopped || state.ended || Date.parse(state.last_activity_at) > nowMs) continue;
|
|
537
530
|
// Without a base, only a touched-files session that touched something may get one.
|
|
538
531
|
if (state.next_sequence === 0 && !(state.marker === TOUCHED_MARKER && (await this.touched.has(state.session_id)))) continue;
|
|
539
532
|
sessions.push(state);
|
|
@@ -548,7 +541,7 @@ export class SessionArchiver {
|
|
|
548
541
|
roots.add(folder);
|
|
549
542
|
if (state.final_due || newestOnRoot) picked.push(state.session_id);
|
|
550
543
|
}
|
|
551
|
-
return picked
|
|
544
|
+
return picked;
|
|
552
545
|
} catch (error) {
|
|
553
546
|
this.log("warn", "OmniRush archive could not list the sessions to check at start", { error: errorSummary(error) });
|
|
554
547
|
return [];
|
|
@@ -39,8 +39,8 @@ import {
|
|
|
39
39
|
import { stateKey } from "./files.js";
|
|
40
40
|
import type { ArchiveLog } from "./upload.js";
|
|
41
41
|
|
|
42
|
-
/**
|
|
43
|
-
export const MAX_TOUCHED_PATHS =
|
|
42
|
+
/** Kept as a compatibility export; touched paths are no longer silently capped. */
|
|
43
|
+
export const MAX_TOUCHED_PATHS = Number.MAX_SAFE_INTEGER;
|
|
44
44
|
const MAX_TOUCHED_PATH_CHARS = 4_096;
|
|
45
45
|
/** A touched symlink is followed to a file inside the root through at most this many links. */
|
|
46
46
|
const MAX_LINK_HOPS = 8;
|
|
@@ -387,11 +387,6 @@ export class TouchedPathStore {
|
|
|
387
387
|
}, () => undefined);
|
|
388
388
|
}
|
|
389
389
|
if (session.mode === "ignored" || session.known?.has(path) || session.pending.has(path) || touchedPathParts(path) === null) return;
|
|
390
|
-
if ((session.known?.size ?? 0) + session.pending.size + session.writing.length >= MAX_TOUCHED_PATHS) {
|
|
391
|
-
if (!session.capped) this.options.log("warn", "OmniRush touched-files archive keeps a limited number of paths per session; later ones are not archived", { sessionId, limit: MAX_TOUCHED_PATHS });
|
|
392
|
-
session.capped = true;
|
|
393
|
-
return;
|
|
394
|
-
}
|
|
395
390
|
session.pending.add(path);
|
|
396
391
|
if (session.mode === "tracked") this.schedule();
|
|
397
392
|
}
|
|
@@ -92,7 +92,6 @@ const MAX_SESSION_LEDGER_ENTRIES = 512;
|
|
|
92
92
|
const SESSION_LEDGER_LOCK_TIMEOUT_MS = 5_000;
|
|
93
93
|
const SESSION_LEDGER_LOCK_STALE_MS = 60_000;
|
|
94
94
|
const SESSION_LEDGER_LOCK_RETRY_MS = 25;
|
|
95
|
-
const MAX_TOUCHED_PATHS = 128;
|
|
96
95
|
const MAX_GIT_STATUS_ENTRIES = 500;
|
|
97
96
|
const MAX_GIT_RECENT_COMMITS = 50;
|
|
98
97
|
const MAX_GIT_REMOTES = 10;
|
|
@@ -1689,11 +1688,16 @@ export async function readSessionLedger(path: string | null): Promise<SessionLed
|
|
|
1689
1688
|
}
|
|
1690
1689
|
}
|
|
1691
1690
|
|
|
1691
|
+
/**
|
|
1692
|
+
* The trace document (scrubbed, cut to `maxBytes`) as JSON text, plus the
|
|
1693
|
+
* events it carries (the same values the text encodes): the envelope's
|
|
1694
|
+
* "trace" array is written from them without parsing the text back.
|
|
1695
|
+
*/
|
|
1692
1696
|
function boundedTracePayload(
|
|
1693
1697
|
state: Pick<SessionState, "id" | "workspaceId" | "segment" | "resumed">,
|
|
1694
1698
|
events: TraceEvent[],
|
|
1695
1699
|
maxBytes = MAX_TRACE_BYTES,
|
|
1696
|
-
):
|
|
1700
|
+
): { content: string; events: unknown[] } {
|
|
1697
1701
|
// The trace is scrubbed as a JSON value, never as serialised text: a text
|
|
1698
1702
|
// redaction inside an escaped string (`\n@app.function`) once left a
|
|
1699
1703
|
// dangling backslash that made the whole document unparseable. Header and
|
|
@@ -1707,12 +1711,15 @@ function boundedTracePayload(
|
|
|
1707
1711
|
session_resumed: state.resumed,
|
|
1708
1712
|
events,
|
|
1709
1713
|
}).value as Record<string, unknown> & { events: TraceEvent[] };
|
|
1710
|
-
const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) =>
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1714
|
+
const encode = (selected: TraceEvent[], truncated: boolean, includedCount = selected.length) => {
|
|
1715
|
+
const content = JSON.stringify({
|
|
1716
|
+
...header,
|
|
1717
|
+
trace_truncated: truncated,
|
|
1718
|
+
dropped_event_count: Math.max(0, events.length - includedCount),
|
|
1719
|
+
events: selected,
|
|
1720
|
+
});
|
|
1721
|
+
return { content, events: selected as unknown[], byteLength: Buffer.byteLength(content) };
|
|
1722
|
+
};
|
|
1716
1723
|
|
|
1717
1724
|
// One encode settles the common case. Only an oversized trace pays for the
|
|
1718
1725
|
// search, and that search measures each event once: the payload of a prefix
|
|
@@ -1721,7 +1728,7 @@ function boundedTracePayload(
|
|
|
1721
1728
|
// probe used to be a full serialisation of the prefix, which made a flush
|
|
1722
1729
|
// at the MAX_COLLECTOR_TRACE_EVENTS cap peak at many times the payload.
|
|
1723
1730
|
let payload = encode(scrubbed, false);
|
|
1724
|
-
if (payload.byteLength <= maxBytes) return payload;
|
|
1731
|
+
if (payload.byteLength <= maxBytes) return { content: payload.content, events: payload.events };
|
|
1725
1732
|
const wrapperBytes = (included: number) => Buffer.byteLength(JSON.stringify({
|
|
1726
1733
|
...header,
|
|
1727
1734
|
trace_truncated: true,
|
|
@@ -1749,9 +1756,9 @@ function boundedTracePayload(
|
|
|
1749
1756
|
data: { dropped_event_count: events.length },
|
|
1750
1757
|
};
|
|
1751
1758
|
const marked = encode([marker], true, 0);
|
|
1752
|
-
if (marked.byteLength <= maxBytes) return marked;
|
|
1759
|
+
if (marked.byteLength <= maxBytes) return { content: marked.content, events: marked.events };
|
|
1753
1760
|
}
|
|
1754
|
-
return payload;
|
|
1761
|
+
return { content: payload.content, events: payload.events };
|
|
1755
1762
|
}
|
|
1756
1763
|
|
|
1757
1764
|
/** Eligible files plus the number of non-ignored paths the denylist dropped. */
|
|
@@ -2837,6 +2844,45 @@ const utf8Encoder = new TextEncoder();
|
|
|
2837
2844
|
* entry exists in memory at a time, and writing allocates no buffer per chunk.
|
|
2838
2845
|
* The compressed result is read back from disk only for the upload.
|
|
2839
2846
|
*/
|
|
2847
|
+
/** Strings longer than this are escaped and written a piece at a time. */
|
|
2848
|
+
const JSON_STRING_PIECE_CHARS = 1 << 20;
|
|
2849
|
+
|
|
2850
|
+
/**
|
|
2851
|
+
* Writes `value` as JSON, byte for byte what JSON.stringify(value) gives,
|
|
2852
|
+
* without building the whole text: a plain object's long string values (a
|
|
2853
|
+
* file's content, up to MAX_COLLECTOR_FILE_BYTES; the trace document, up to
|
|
2854
|
+
* MAX_TRACE_BYTES) are escaped in pieces, so no escaped copy of them exists
|
|
2855
|
+
* at once. Anything else goes through JSON.stringify.
|
|
2856
|
+
*/
|
|
2857
|
+
export async function writeJsonStreaming(writer: { write(text: string): Promise<void> }, value: unknown): Promise<void> {
|
|
2858
|
+
if (typeof value === "string" && value.length > JSON_STRING_PIECE_CHARS) {
|
|
2859
|
+
await writer.write('"');
|
|
2860
|
+
for (let at = 0; at < value.length;) {
|
|
2861
|
+
let end = Math.min(value.length, at + JSON_STRING_PIECE_CHARS);
|
|
2862
|
+
// Never split a surrogate pair: each half alone would be escaped.
|
|
2863
|
+
const last = value.charCodeAt(end - 1);
|
|
2864
|
+
if (end < value.length && last >= 0xd800 && last <= 0xdbff) end -= 1;
|
|
2865
|
+
await writer.write(JSON.stringify(value.slice(at, end)).slice(1, -1));
|
|
2866
|
+
at = end;
|
|
2867
|
+
}
|
|
2868
|
+
await writer.write('"');
|
|
2869
|
+
return;
|
|
2870
|
+
}
|
|
2871
|
+
if (value && typeof value === "object" && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype && typeof (value as { toJSON?: unknown }).toJSON !== "function") {
|
|
2872
|
+
await writer.write("{");
|
|
2873
|
+
let first = true;
|
|
2874
|
+
for (const [key, item] of Object.entries(value)) {
|
|
2875
|
+
if (item === undefined || typeof item === "function" || typeof item === "symbol") continue;
|
|
2876
|
+
await writer.write(`${first ? "" : ","}${JSON.stringify(key)}:`);
|
|
2877
|
+
first = false;
|
|
2878
|
+
await writeJsonStreaming(writer, item);
|
|
2879
|
+
}
|
|
2880
|
+
await writer.write("}");
|
|
2881
|
+
return;
|
|
2882
|
+
}
|
|
2883
|
+
await writer.write(JSON.stringify(value) ?? "null");
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2840
2886
|
class EnvelopeWriter {
|
|
2841
2887
|
/** Uncompressed bytes written so far. */
|
|
2842
2888
|
bytes = 0;
|
|
@@ -4105,6 +4151,17 @@ export class WorkspaceCollector {
|
|
|
4105
4151
|
});
|
|
4106
4152
|
}
|
|
4107
4153
|
|
|
4154
|
+
/**
|
|
4155
|
+
* Resolves once the session's queued uploads (traces, snapshots) went out:
|
|
4156
|
+
* a caller recording a lot at once (every sub-agent of a turn) waits here
|
|
4157
|
+
* between flushes, so only one flush's events wait in memory at a time.
|
|
4158
|
+
*/
|
|
4159
|
+
async drained(sessionId: string): Promise<void> {
|
|
4160
|
+
const state = this.sessions.get(sessionId);
|
|
4161
|
+
if (!state) return;
|
|
4162
|
+
await state.tail.catch(() => undefined);
|
|
4163
|
+
}
|
|
4164
|
+
|
|
4108
4165
|
/** Resolves once every queued capture and upload for the session has settled, and its watchers are up. */
|
|
4109
4166
|
async idle(sessionId: string): Promise<void> {
|
|
4110
4167
|
const state = this.sessions.get(sessionId);
|
|
@@ -4377,7 +4434,7 @@ export class WorkspaceCollector {
|
|
|
4377
4434
|
}
|
|
4378
4435
|
|
|
4379
4436
|
private touchedPathsForUpload(state: SessionState): string[] {
|
|
4380
|
-
return [...state.touchedPaths].
|
|
4437
|
+
return [...state.touchedPaths].map(collectorPathForUpload);
|
|
4381
4438
|
}
|
|
4382
4439
|
|
|
4383
4440
|
/**
|
|
@@ -4558,15 +4615,10 @@ export class WorkspaceCollector {
|
|
|
4558
4615
|
|
|
4559
4616
|
private async uploadTrace(state: SessionState, traceEvents: TraceEvent[]): Promise<void> {
|
|
4560
4617
|
if (traceEvents.length === 0) return;
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
const parsed = JSON.parse(content) as { events?: unknown };
|
|
4566
|
-
if (Array.isArray(parsed.events)) events = parsed.events;
|
|
4567
|
-
} catch {
|
|
4568
|
-
events = [];
|
|
4569
|
-
}
|
|
4618
|
+
// The text is uploaded as the trace file and its events as the envelope's
|
|
4619
|
+
// trace (written event by event); neither is copied or parsed back.
|
|
4620
|
+
const { content, events } = boundedTracePayload(state, traceEvents, MAX_TRACE_BYTES);
|
|
4621
|
+
traceEvents.length = 0;
|
|
4570
4622
|
await this.uploadEnvelope(state, "trace", "trace_flush", {
|
|
4571
4623
|
extras: {
|
|
4572
4624
|
workspace: { root_name: workspaceRootName(state.root), git: null },
|
|
@@ -4600,7 +4652,8 @@ export class WorkspaceCollector {
|
|
|
4600
4652
|
"Content-Type": "application/zstd",
|
|
4601
4653
|
"X-OmniRush-Session-ID": sessionId,
|
|
4602
4654
|
},
|
|
4603
|
-
|
|
4655
|
+
// The bytes as they are (a view, not a copy of up to 64 MiB).
|
|
4656
|
+
body: compressed as Uint8Array<ArrayBuffer>,
|
|
4604
4657
|
signal,
|
|
4605
4658
|
});
|
|
4606
4659
|
}
|
|
@@ -4731,7 +4784,8 @@ export class WorkspaceCollector {
|
|
|
4731
4784
|
});
|
|
4732
4785
|
await writer.write(`${head.slice(0, -1)},"files":[`);
|
|
4733
4786
|
await body.writeFiles(async (file) => {
|
|
4734
|
-
|
|
4787
|
+
if (fileCount > 0) await writer.write(",");
|
|
4788
|
+
await writeJsonStreaming(writer, file);
|
|
4735
4789
|
fileCount += 1;
|
|
4736
4790
|
});
|
|
4737
4791
|
await writer.write('],"manifest":[');
|
|
@@ -4741,7 +4795,16 @@ export class WorkspaceCollector {
|
|
|
4741
4795
|
first = false;
|
|
4742
4796
|
}
|
|
4743
4797
|
await writer.write(`],"privacy":${JSON.stringify(body.privacy())}`);
|
|
4744
|
-
if (
|
|
4798
|
+
if (Array.isArray(body.trace)) {
|
|
4799
|
+
// Event by event: the same text as one JSON.stringify of the array.
|
|
4800
|
+
await writer.write(',"trace":[');
|
|
4801
|
+
for (let index = 0; index < body.trace.length; index += 1) {
|
|
4802
|
+
await writer.write(`${index > 0 ? "," : ""}${JSON.stringify(body.trace[index]) ?? "null"}`);
|
|
4803
|
+
}
|
|
4804
|
+
await writer.write("]");
|
|
4805
|
+
} else if (body.trace) {
|
|
4806
|
+
await writer.write(`,"trace":${JSON.stringify(body.trace)}`);
|
|
4807
|
+
}
|
|
4745
4808
|
await writer.write("}");
|
|
4746
4809
|
compressedBytes = await writer.finish();
|
|
4747
4810
|
} catch (error) {
|