@nemus-cli/nemus 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/commands/reflect.js +55 -6
- package/dist/utils/agent-judge.js +68 -25
- package/dist/utils/reflect.js +17 -9
- package/package.json +1 -1
- package/src/commands/reflect.ts +58 -6
- package/src/utils/agent-judge.test.ts +35 -1
- package/src/utils/agent-judge.ts +85 -26
- package/src/utils/reflect.ts +28 -9
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- **`reflect` now shows live progress and no longer looks hung.** The judge call
|
|
13
|
+
ran synchronously (`execFileSync`), which blocked the event loop, and a
|
|
14
|
+
10-workspace run could hit the 180s cap and die with a raw
|
|
15
|
+
`spawnSync pi ETIMEDOUT`. Now: (1) the gather phase prints a **per-workspace
|
|
16
|
+
line** as each session is read (`✓ 1/3 my-workspace 635 turns · 12 prompts ·
|
|
17
|
+
13 failures`); (2) the judge runs **async** behind a live spinner with elapsed
|
|
18
|
+
seconds; (3) the timeout is raised to 300s, overridable via
|
|
19
|
+
`NEMUS_JUDGE_TIMEOUT_MS`, and a timeout now yields an **actionable** message
|
|
20
|
+
("try a smaller --limit, a faster agent, or raise the cap"); (4) the judge
|
|
21
|
+
prompt is leaner (fewer prompts/errors per session) so a local model actually
|
|
22
|
+
finishes.
|
|
23
|
+
|
|
10
24
|
## [0.3.0] - 2026-08-27
|
|
11
25
|
|
|
12
26
|
### Added
|
package/dist/commands/reflect.js
CHANGED
|
@@ -21,11 +21,11 @@ function registerReflectCommand(parent) {
|
|
|
21
21
|
async function handleReflect(opts) {
|
|
22
22
|
const limit = Math.max(1, Number.parseInt(opts.limit ?? '10', 10) || 10);
|
|
23
23
|
try {
|
|
24
|
-
|
|
24
|
+
const showProgress = !opts.json && !opts.dryRun;
|
|
25
|
+
if (showProgress) {
|
|
25
26
|
(0, logger_1.logStep)(`Analyzing your ${(0, colors_1.colorize)(String(limit), 'cyan')} most recent workspaces…`);
|
|
26
|
-
(0, logger_1.logInfo)('Reading sessions and distilling prompts + failures…');
|
|
27
27
|
}
|
|
28
|
-
const corpus = await (0, reflect_1.gatherReflectionCorpus)(limit);
|
|
28
|
+
const corpus = await (0, reflect_1.gatherReflectionCorpus)(limit, showProgress ? printProgress : undefined);
|
|
29
29
|
const withSessions = corpus.workspaces.filter((w) => w.session).length;
|
|
30
30
|
const prompt = (0, reflect_1.buildJudgePrompt)(corpus);
|
|
31
31
|
if (opts.dryRun) {
|
|
@@ -45,9 +45,20 @@ async function handleReflect(opts) {
|
|
|
45
45
|
(0, logger_1.logError)(msg);
|
|
46
46
|
process.exit(1);
|
|
47
47
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
// The judge shells the user's own agent and can take minutes; run it async
|
|
49
|
+
// (non-blocking) so a live spinner shows it's alive, not hung. Timeout is
|
|
50
|
+
// overridable for slow local models.
|
|
51
|
+
const timeoutMs = Number.parseInt(process.env.NEMUS_JUDGE_TIMEOUT_MS ?? '', 10) || undefined;
|
|
52
|
+
const stopSpinner = opts.json
|
|
53
|
+
? () => { }
|
|
54
|
+
: startSpinner(`Judging ${withSessions} session(s) with your configured agent (this can take a minute)…`);
|
|
55
|
+
let parsed;
|
|
56
|
+
try {
|
|
57
|
+
parsed = await (0, agent_judge_1.runAgentJsonAsync)(prompt, { schema: reflect_1.REFLECT_SCHEMA, timeoutMs });
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
stopSpinner();
|
|
61
|
+
}
|
|
51
62
|
const report = (0, reflect_1.parseReflectionReport)(parsed);
|
|
52
63
|
if (opts.json) {
|
|
53
64
|
(0, output_1.outputJson)({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report });
|
|
@@ -75,6 +86,44 @@ const KIND_LABEL = {
|
|
|
75
86
|
workflow: 'Workflow',
|
|
76
87
|
other: 'Other',
|
|
77
88
|
};
|
|
89
|
+
/**
|
|
90
|
+
* A minimal stderr spinner with elapsed seconds. Returns a stop() that clears
|
|
91
|
+
* the line. No-op (single log line) when stderr isn't a TTY (piped/CI), so it
|
|
92
|
+
* never pollutes captured output. Kept local + tiny — no new dependency.
|
|
93
|
+
*/
|
|
94
|
+
function startSpinner(text) {
|
|
95
|
+
if (!process.stderr.isTTY) {
|
|
96
|
+
(0, logger_1.logInfo)(text);
|
|
97
|
+
return () => { };
|
|
98
|
+
}
|
|
99
|
+
const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
100
|
+
const start = Date.now();
|
|
101
|
+
let i = 0;
|
|
102
|
+
const render = () => {
|
|
103
|
+
const secs = Math.floor((Date.now() - start) / 1000);
|
|
104
|
+
process.stderr.write(`\r${(0, colors_1.colorize)(frames[(i = (i + 1) % frames.length)], 'cyan')} ${text} ${(0, colors_1.colorize)(`(${secs}s)`, 'dim')}`);
|
|
105
|
+
};
|
|
106
|
+
render();
|
|
107
|
+
const timer = setInterval(render, 100);
|
|
108
|
+
timer.unref?.(); // never keep the process alive on our account
|
|
109
|
+
return () => {
|
|
110
|
+
clearInterval(timer);
|
|
111
|
+
process.stderr.write('\r' + ' '.repeat(text.length + 24) + '\r');
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/** Live per-workspace line during the gather phase (to stderr — stdout stays
|
|
115
|
+
* reserved for the report / JSON). */
|
|
116
|
+
function printProgress(p) {
|
|
117
|
+
const n = (0, colors_1.colorize)(`${p.index + 1}/${p.total}`, 'dim');
|
|
118
|
+
const d = p.digest.session;
|
|
119
|
+
if (!d) {
|
|
120
|
+
process.stderr.write(` ${(0, colors_1.colorize)('·', 'dim')} ${n} ${p.digest.name} ${(0, colors_1.colorize)('— no session', 'dim')}\n`);
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const failures = `${d.errors.length} ${d.errors.length === 1 ? 'failure' : 'failures'}`;
|
|
124
|
+
const stats = (0, colors_1.colorize)(`${d.turns} turns · ${d.userPrompts.length} prompts · ${failures}`, 'dim');
|
|
125
|
+
process.stderr.write(` ${(0, colors_1.colorize)('✓', 'green')} ${n} ${p.digest.name} ${stats}\n`);
|
|
126
|
+
}
|
|
78
127
|
function priorityBadge(p) {
|
|
79
128
|
if (p === 'high')
|
|
80
129
|
return (0, colors_1.colorize)('● high', 'red');
|
|
@@ -1,11 +1,45 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.agentAttempts = agentAttempts;
|
|
3
4
|
exports.runAgentRaw = runAgentRaw;
|
|
5
|
+
exports.runAgentRawAsync = runAgentRawAsync;
|
|
4
6
|
exports.runAgentJson = runAgentJson;
|
|
7
|
+
exports.runAgentJsonAsync = runAgentJsonAsync;
|
|
5
8
|
exports.parseAgentJson = parseAgentJson;
|
|
6
9
|
const child_process_1 = require("child_process");
|
|
10
|
+
const util_1 = require("util");
|
|
7
11
|
const agent_config_1 = require("./agent-config");
|
|
8
|
-
const
|
|
12
|
+
const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
|
|
13
|
+
/**
|
|
14
|
+
* The ordered invocation attempts for an agent (preferred → fallback), as pure
|
|
15
|
+
* data so both the sync and async runners share ONE flag ladder (and it's
|
|
16
|
+
* unit-testable without spawning anything).
|
|
17
|
+
*/
|
|
18
|
+
function agentAttempts(agentType, prompt, schema) {
|
|
19
|
+
if (agentType === 'claude') {
|
|
20
|
+
const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
|
|
21
|
+
if (schema)
|
|
22
|
+
preferred.push('--json-schema', schema);
|
|
23
|
+
// Older claude may reject the newer flags — fall back to the plainest form.
|
|
24
|
+
return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: ['-p', prompt] }];
|
|
25
|
+
}
|
|
26
|
+
if (agentType === 'opencode') {
|
|
27
|
+
return [{ cmd: 'opencode', args: ['run', prompt] }];
|
|
28
|
+
}
|
|
29
|
+
// pi (and any other): run as lean as possible so a bloated env can't hang it.
|
|
30
|
+
const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
|
|
31
|
+
return [{ cmd: 'pi', args: [...piLean, '-p', prompt] }, { cmd: 'pi', args: ['-p', prompt] }];
|
|
32
|
+
}
|
|
33
|
+
function wrapJudgeError(err, agentType) {
|
|
34
|
+
// A timeout is the common failure (big prompt + slow local model), so make it
|
|
35
|
+
// actionable instead of surfacing a raw `spawn … ETIMEDOUT`.
|
|
36
|
+
if (err?.killed || err?.code === 'ETIMEDOUT' || err?.signal === 'SIGTERM' || /ETIMEDOUT/.test(String(err?.message ?? ''))) {
|
|
37
|
+
return new Error(`agent judge (${agentType}) timed out. Try a smaller --limit, a faster agent, or raise the cap with NEMUS_JUDGE_TIMEOUT_MS.`);
|
|
38
|
+
}
|
|
39
|
+
const detail = (err?.stderr || err?.stdout || err?.message || 'unknown error').toString().trim().slice(0, 500);
|
|
40
|
+
return new Error(`agent judge failed (${agentType}): ${detail}`);
|
|
41
|
+
}
|
|
42
|
+
const DEFAULT_TIMEOUT_MS = 300000; // judging N transcripts is heavier than extraction; a big prompt + slow model can run minutes
|
|
9
43
|
const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024;
|
|
10
44
|
/**
|
|
11
45
|
* Invoke the agent with `prompt` and return the raw stdout. Throws a clear error
|
|
@@ -18,36 +52,40 @@ function runAgentRaw(prompt, opts = {}) {
|
|
|
18
52
|
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
19
53
|
const exec = opts.exec ??
|
|
20
54
|
((cmd, args, o) => (0, child_process_1.execFileSync)(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer }));
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
preferred.push('--json-schema', opts.schema);
|
|
27
|
-
try {
|
|
28
|
-
return attempt('claude', preferred);
|
|
29
|
-
}
|
|
30
|
-
catch {
|
|
31
|
-
// Older claude may reject the newer flags — fall back to the plainest form.
|
|
32
|
-
return attempt('claude', ['-p', prompt]);
|
|
33
|
-
}
|
|
55
|
+
const attempts = agentAttempts(agentType, prompt, opts.schema);
|
|
56
|
+
let lastErr;
|
|
57
|
+
for (const a of attempts) {
|
|
58
|
+
try {
|
|
59
|
+
return exec(a.cmd, a.args, { timeout, maxBuffer });
|
|
34
60
|
}
|
|
35
|
-
|
|
36
|
-
|
|
61
|
+
catch (err) {
|
|
62
|
+
lastErr = err; // try the next (fallback) form
|
|
37
63
|
}
|
|
38
|
-
|
|
39
|
-
|
|
64
|
+
}
|
|
65
|
+
throw wrapJudgeError(lastErr, agentType);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Non-blocking twin of {@link runAgentRaw}. Uses `execFile` (async) so the
|
|
69
|
+
* caller's event loop stays free — letting a spinner/progress UI animate while
|
|
70
|
+
* the judge (which can take minutes) runs. Prefer this in interactive commands.
|
|
71
|
+
*/
|
|
72
|
+
async function runAgentRawAsync(prompt, opts = {}) {
|
|
73
|
+
const agentType = opts.agentType ?? (0, agent_config_1.getPrimaryAgent)().type;
|
|
74
|
+
const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
75
|
+
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
76
|
+
const exec = opts.execAsync ??
|
|
77
|
+
(async (cmd, args, o) => (await execFileAsync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer })).stdout.toString());
|
|
78
|
+
const attempts = agentAttempts(agentType, prompt, opts.schema);
|
|
79
|
+
let lastErr;
|
|
80
|
+
for (const a of attempts) {
|
|
40
81
|
try {
|
|
41
|
-
return
|
|
82
|
+
return await exec(a.cmd, a.args, { timeout, maxBuffer });
|
|
42
83
|
}
|
|
43
|
-
catch {
|
|
44
|
-
|
|
84
|
+
catch (err) {
|
|
85
|
+
lastErr = err; // try the next (fallback) form
|
|
45
86
|
}
|
|
46
87
|
}
|
|
47
|
-
|
|
48
|
-
const detail = (err?.stderr || err?.stdout || err?.message || 'unknown error').toString().trim().slice(0, 500);
|
|
49
|
-
throw new Error(`agent judge failed (${agentType}): ${detail}`);
|
|
50
|
-
}
|
|
88
|
+
throw wrapJudgeError(lastErr, agentType);
|
|
51
89
|
}
|
|
52
90
|
/**
|
|
53
91
|
* Run the agent and parse its reply as JSON, tolerating the shapes different
|
|
@@ -58,6 +96,11 @@ function runAgentJson(prompt, opts = {}) {
|
|
|
58
96
|
const raw = runAgentRaw(prompt, opts);
|
|
59
97
|
return parseAgentJson(raw);
|
|
60
98
|
}
|
|
99
|
+
/** Non-blocking twin of {@link runAgentJson}. */
|
|
100
|
+
async function runAgentJsonAsync(prompt, opts = {}) {
|
|
101
|
+
const raw = await runAgentRawAsync(prompt, opts);
|
|
102
|
+
return parseAgentJson(raw);
|
|
103
|
+
}
|
|
61
104
|
/** Extract a JSON object from an agent's raw stdout. Exported for tests. */
|
|
62
105
|
function parseAgentJson(raw) {
|
|
63
106
|
let text = raw.trim();
|
package/dist/utils/reflect.js
CHANGED
|
@@ -45,10 +45,14 @@ const workspace_meta_1 = require("./workspace-meta");
|
|
|
45
45
|
const agent_config_1 = require("./agent-config");
|
|
46
46
|
const claude_sessions_1 = require("./claude-sessions");
|
|
47
47
|
// --------------------------------------------------------- transcript distill
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
// Kept deliberately lean: the judge runs on the user's own (often local, slow)
|
|
49
|
+
// agent, and a 10-workspace corpus at full verbosity produced a ~200KB prompt
|
|
50
|
+
// that timed pi out. These bounds capture the pattern of a session at a
|
|
51
|
+
// fraction of the tokens (~halving the prompt), so the judge actually finishes.
|
|
52
|
+
const MAX_PROMPTS = 12;
|
|
53
|
+
const MAX_ERRORS = 15;
|
|
54
|
+
const PROMPT_CHARS = 400;
|
|
55
|
+
const ERROR_CHARS = 200;
|
|
52
56
|
/** Flatten a message `content` (string or content-block array) to plain text. */
|
|
53
57
|
function contentToText(content) {
|
|
54
58
|
if (typeof content === 'string')
|
|
@@ -237,9 +241,10 @@ async function contextFilesFor(workspacePath) {
|
|
|
237
241
|
* Build the corpus the judge reasons over: the `limit` most **recently active**
|
|
238
242
|
* workspaces (by their latest agent session, not creation date — a retrospective
|
|
239
243
|
* is about recent *work*), each with its repos, context files, and distilled
|
|
240
|
-
* session, plus the globally-available skills.
|
|
244
|
+
* session, plus the globally-available skills. `onProgress` (optional) fires
|
|
245
|
+
* once per workspace as it finishes, for a live progress display.
|
|
241
246
|
*/
|
|
242
|
-
async function gatherReflectionCorpus(limit) {
|
|
247
|
+
async function gatherReflectionCorpus(limit, onProgress) {
|
|
243
248
|
const [sessions, workspaces, availableSkills] = await Promise.all([
|
|
244
249
|
(0, claude_sessions_1.getWorkspaceSessions)(), // already sorted by last-active, one per workspace
|
|
245
250
|
(0, workspace_meta_1.listWorkspaces)(false),
|
|
@@ -248,15 +253,18 @@ async function gatherReflectionCorpus(limit) {
|
|
|
248
253
|
const metaByName = new Map(workspaces.map((w) => [w.name, w]));
|
|
249
254
|
const recent = sessions.slice(0, limit);
|
|
250
255
|
const digests = [];
|
|
251
|
-
for (
|
|
256
|
+
for (let index = 0; index < recent.length; index++) {
|
|
257
|
+
const s = recent[index];
|
|
252
258
|
const meta = metaByName.get(s.workspaceName);
|
|
253
|
-
|
|
259
|
+
const digest = {
|
|
254
260
|
name: s.workspaceName,
|
|
255
261
|
repoCount: meta?.metadata?.repositories?.length ?? 0,
|
|
256
262
|
repos: (meta?.metadata?.repositories ?? []).map((r) => r.name),
|
|
257
263
|
contextFiles: await contextFilesFor(s.workspacePath),
|
|
258
264
|
session: await readDigestForSession(s),
|
|
259
|
-
}
|
|
265
|
+
};
|
|
266
|
+
digests.push(digest);
|
|
267
|
+
onProgress?.({ index, total: recent.length, digest });
|
|
260
268
|
}
|
|
261
269
|
return { generatedAt: new Date().toISOString(), availableSkills, workspaces: digests };
|
|
262
270
|
}
|
package/package.json
CHANGED
package/src/commands/reflect.ts
CHANGED
|
@@ -8,9 +8,10 @@ import {
|
|
|
8
8
|
parseReflectionReport,
|
|
9
9
|
REFLECT_SCHEMA,
|
|
10
10
|
ReflectionReport,
|
|
11
|
+
ReflectProgress,
|
|
11
12
|
Recommendation,
|
|
12
13
|
} from '../utils/reflect';
|
|
13
|
-
import {
|
|
14
|
+
import { runAgentJsonAsync } from '../utils/agent-judge';
|
|
14
15
|
|
|
15
16
|
export function registerReflectCommand(parent: Command) {
|
|
16
17
|
parent
|
|
@@ -28,12 +29,12 @@ export function registerReflectCommand(parent: Command) {
|
|
|
28
29
|
async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: boolean }) {
|
|
29
30
|
const limit = Math.max(1, Number.parseInt(opts.limit ?? '10', 10) || 10);
|
|
30
31
|
try {
|
|
31
|
-
|
|
32
|
+
const showProgress = !opts.json && !opts.dryRun;
|
|
33
|
+
if (showProgress) {
|
|
32
34
|
logStep(`Analyzing your ${colorize(String(limit), 'cyan')} most recent workspaces…`);
|
|
33
|
-
logInfo('Reading sessions and distilling prompts + failures…');
|
|
34
35
|
}
|
|
35
36
|
|
|
36
|
-
const corpus = await gatherReflectionCorpus(limit);
|
|
37
|
+
const corpus = await gatherReflectionCorpus(limit, showProgress ? printProgress : undefined);
|
|
37
38
|
const withSessions = corpus.workspaces.filter((w) => w.session).length;
|
|
38
39
|
const prompt = buildJudgePrompt(corpus);
|
|
39
40
|
|
|
@@ -53,8 +54,19 @@ async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: bo
|
|
|
53
54
|
process.exit(1);
|
|
54
55
|
}
|
|
55
56
|
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
// The judge shells the user's own agent and can take minutes; run it async
|
|
58
|
+
// (non-blocking) so a live spinner shows it's alive, not hung. Timeout is
|
|
59
|
+
// overridable for slow local models.
|
|
60
|
+
const timeoutMs = Number.parseInt(process.env.NEMUS_JUDGE_TIMEOUT_MS ?? '', 10) || undefined;
|
|
61
|
+
const stopSpinner = opts.json
|
|
62
|
+
? () => {}
|
|
63
|
+
: startSpinner(`Judging ${withSessions} session(s) with your configured agent (this can take a minute)…`);
|
|
64
|
+
let parsed: unknown;
|
|
65
|
+
try {
|
|
66
|
+
parsed = await runAgentJsonAsync(prompt, { schema: REFLECT_SCHEMA, timeoutMs });
|
|
67
|
+
} finally {
|
|
68
|
+
stopSpinner();
|
|
69
|
+
}
|
|
58
70
|
const report = parseReflectionReport(parsed);
|
|
59
71
|
|
|
60
72
|
if (opts.json) {
|
|
@@ -83,6 +95,46 @@ const KIND_LABEL: Record<Recommendation['kind'], string> = {
|
|
|
83
95
|
other: 'Other',
|
|
84
96
|
};
|
|
85
97
|
|
|
98
|
+
/**
|
|
99
|
+
* A minimal stderr spinner with elapsed seconds. Returns a stop() that clears
|
|
100
|
+
* the line. No-op (single log line) when stderr isn't a TTY (piped/CI), so it
|
|
101
|
+
* never pollutes captured output. Kept local + tiny — no new dependency.
|
|
102
|
+
*/
|
|
103
|
+
function startSpinner(text: string): () => void {
|
|
104
|
+
if (!process.stderr.isTTY) {
|
|
105
|
+
logInfo(text);
|
|
106
|
+
return () => {};
|
|
107
|
+
}
|
|
108
|
+
const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
109
|
+
const start = Date.now();
|
|
110
|
+
let i = 0;
|
|
111
|
+
const render = () => {
|
|
112
|
+
const secs = Math.floor((Date.now() - start) / 1000);
|
|
113
|
+
process.stderr.write(`\r${colorize(frames[(i = (i + 1) % frames.length)], 'cyan')} ${text} ${colorize(`(${secs}s)`, 'dim')}`);
|
|
114
|
+
};
|
|
115
|
+
render();
|
|
116
|
+
const timer = setInterval(render, 100);
|
|
117
|
+
timer.unref?.(); // never keep the process alive on our account
|
|
118
|
+
return () => {
|
|
119
|
+
clearInterval(timer);
|
|
120
|
+
process.stderr.write('\r' + ' '.repeat(text.length + 24) + '\r');
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Live per-workspace line during the gather phase (to stderr — stdout stays
|
|
125
|
+
* reserved for the report / JSON). */
|
|
126
|
+
function printProgress(p: ReflectProgress): void {
|
|
127
|
+
const n = colorize(`${p.index + 1}/${p.total}`, 'dim');
|
|
128
|
+
const d = p.digest.session;
|
|
129
|
+
if (!d) {
|
|
130
|
+
process.stderr.write(` ${colorize('·', 'dim')} ${n} ${p.digest.name} ${colorize('— no session', 'dim')}\n`);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
const failures = `${d.errors.length} ${d.errors.length === 1 ? 'failure' : 'failures'}`;
|
|
134
|
+
const stats = colorize(`${d.turns} turns · ${d.userPrompts.length} prompts · ${failures}`, 'dim');
|
|
135
|
+
process.stderr.write(` ${colorize('✓', 'green')} ${n} ${p.digest.name} ${stats}\n`);
|
|
136
|
+
}
|
|
137
|
+
|
|
86
138
|
function priorityBadge(p: Recommendation['priority']): string {
|
|
87
139
|
if (p === 'high') return colorize('● high', 'red');
|
|
88
140
|
if (p === 'medium') return colorize('● med', 'yellow');
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { parseAgentJson, runAgentRaw } from './agent-judge';
|
|
2
|
+
import { parseAgentJson, runAgentRaw, runAgentRawAsync, agentAttempts } from './agent-judge';
|
|
3
3
|
|
|
4
4
|
describe('parseAgentJson', () => {
|
|
5
5
|
it('unwraps the common agent envelopes and shapes', () => {
|
|
@@ -16,6 +16,40 @@ describe('parseAgentJson', () => {
|
|
|
16
16
|
});
|
|
17
17
|
});
|
|
18
18
|
|
|
19
|
+
describe('agentAttempts', () => {
|
|
20
|
+
it('claude: preferred (schema + lean flags) then a plain fallback', () => {
|
|
21
|
+
const a = agentAttempts('claude', 'P', '{"type":"object"}');
|
|
22
|
+
expect(a[0]).toEqual({ cmd: 'claude', args: expect.arrayContaining(['-p', 'P', '--output-format', 'json', '--json-schema', '{"type":"object"}']) });
|
|
23
|
+
expect(a[1]).toEqual({ cmd: 'claude', args: ['-p', 'P'] });
|
|
24
|
+
});
|
|
25
|
+
it('pi: lean then plain; opencode: single run', () => {
|
|
26
|
+
const pi = agentAttempts('pi', 'P');
|
|
27
|
+
expect(pi[0].args).toEqual(expect.arrayContaining(['--no-tools', '--no-skills', '-p', 'P']));
|
|
28
|
+
expect(pi[1]).toEqual({ cmd: 'pi', args: ['-p', 'P'] });
|
|
29
|
+
expect(agentAttempts('opencode', 'P')).toEqual([{ cmd: 'opencode', args: ['run', 'P'] }]);
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe('runAgentRawAsync', () => {
|
|
34
|
+
it('runs the preferred attempt and returns stdout', async () => {
|
|
35
|
+
const calls: string[][] = [];
|
|
36
|
+
const execAsync = async (cmd: string, args: string[]) => {
|
|
37
|
+
calls.push([cmd, ...args]);
|
|
38
|
+
return '{"ok":true}';
|
|
39
|
+
};
|
|
40
|
+
const out = await runAgentRawAsync('P', { agentType: 'pi', execAsync });
|
|
41
|
+
expect(out).toBe('{"ok":true}');
|
|
42
|
+
expect(calls).toHaveLength(1); // first attempt succeeded, no fallback
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('turns a timeout into an actionable error', async () => {
|
|
46
|
+
const execAsync = async () => {
|
|
47
|
+
throw Object.assign(new Error('spawn pi ETIMEDOUT'), { code: 'ETIMEDOUT', killed: true });
|
|
48
|
+
};
|
|
49
|
+
await expect(runAgentRawAsync('P', { agentType: 'pi', execAsync })).rejects.toThrow(/timed out.*NEMUS_JUDGE_TIMEOUT_MS/s);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
|
|
19
53
|
describe('runAgentRaw', () => {
|
|
20
54
|
it('claude: passes the schema + lean flags, falls back on a rejected flag', () => {
|
|
21
55
|
const calls: string[][] = [];
|
package/src/utils/agent-judge.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import { execFileSync } from 'child_process';
|
|
1
|
+
import { execFile, execFileSync } from 'child_process';
|
|
2
|
+
import { promisify } from 'util';
|
|
2
3
|
import { getPrimaryAgent } from './agent-config';
|
|
3
4
|
|
|
5
|
+
const execFileAsync = promisify(execFile);
|
|
6
|
+
|
|
4
7
|
/**
|
|
5
8
|
* Run the user's configured coding agent headlessly as an "LLM-as-a-judge":
|
|
6
9
|
* feed it a prompt, get back a parsed JSON object. This reuses whatever agent
|
|
@@ -16,13 +19,54 @@ export interface JudgeOptions {
|
|
|
16
19
|
schema?: string;
|
|
17
20
|
timeoutMs?: number;
|
|
18
21
|
maxBuffer?: number;
|
|
19
|
-
/** Injected for tests. Defaults to the real child_process runner. */
|
|
22
|
+
/** Injected for tests. Defaults to the real (blocking) child_process runner. */
|
|
20
23
|
exec?: (cmd: string, args: string[], opts: { timeout: number; maxBuffer: number }) => string;
|
|
24
|
+
/** Injected for tests. Async runner used by the non-blocking variants. */
|
|
25
|
+
execAsync?: (cmd: string, args: string[], opts: { timeout: number; maxBuffer: number }) => Promise<string>;
|
|
21
26
|
/** Injected for tests. Defaults to the configured primary agent. */
|
|
22
27
|
agentType?: 'claude' | 'pi' | 'opencode' | 'codex' | 'gemini';
|
|
23
28
|
}
|
|
24
29
|
|
|
25
|
-
|
|
30
|
+
export type JudgeAgentType = NonNullable<JudgeOptions['agentType']>;
|
|
31
|
+
|
|
32
|
+
export interface AgentAttempt {
|
|
33
|
+
cmd: string;
|
|
34
|
+
args: string[];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The ordered invocation attempts for an agent (preferred → fallback), as pure
|
|
39
|
+
* data so both the sync and async runners share ONE flag ladder (and it's
|
|
40
|
+
* unit-testable without spawning anything).
|
|
41
|
+
*/
|
|
42
|
+
export function agentAttempts(agentType: JudgeAgentType, prompt: string, schema?: string): AgentAttempt[] {
|
|
43
|
+
if (agentType === 'claude') {
|
|
44
|
+
const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
|
|
45
|
+
if (schema) preferred.push('--json-schema', schema);
|
|
46
|
+
// Older claude may reject the newer flags — fall back to the plainest form.
|
|
47
|
+
return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: ['-p', prompt] }];
|
|
48
|
+
}
|
|
49
|
+
if (agentType === 'opencode') {
|
|
50
|
+
return [{ cmd: 'opencode', args: ['run', prompt] }];
|
|
51
|
+
}
|
|
52
|
+
// pi (and any other): run as lean as possible so a bloated env can't hang it.
|
|
53
|
+
const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
|
|
54
|
+
return [{ cmd: 'pi', args: [...piLean, '-p', prompt] }, { cmd: 'pi', args: ['-p', prompt] }];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function wrapJudgeError(err: any, agentType: string): Error {
|
|
58
|
+
// A timeout is the common failure (big prompt + slow local model), so make it
|
|
59
|
+
// actionable instead of surfacing a raw `spawn … ETIMEDOUT`.
|
|
60
|
+
if (err?.killed || err?.code === 'ETIMEDOUT' || err?.signal === 'SIGTERM' || /ETIMEDOUT/.test(String(err?.message ?? ''))) {
|
|
61
|
+
return new Error(
|
|
62
|
+
`agent judge (${agentType}) timed out. Try a smaller --limit, a faster agent, or raise the cap with NEMUS_JUDGE_TIMEOUT_MS.`,
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
const detail = (err?.stderr || err?.stdout || err?.message || 'unknown error').toString().trim().slice(0, 500);
|
|
66
|
+
return new Error(`agent judge failed (${agentType}): ${detail}`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const DEFAULT_TIMEOUT_MS = 300_000; // judging N transcripts is heavier than extraction; a big prompt + slow model can run minutes
|
|
26
70
|
const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024;
|
|
27
71
|
|
|
28
72
|
/**
|
|
@@ -38,33 +82,42 @@ export function runAgentRaw(prompt: string, opts: JudgeOptions = {}): string {
|
|
|
38
82
|
opts.exec ??
|
|
39
83
|
((cmd, args, o) => execFileSync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer }));
|
|
40
84
|
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
try
|
|
48
|
-
return attempt('claude', preferred);
|
|
49
|
-
} catch {
|
|
50
|
-
// Older claude may reject the newer flags — fall back to the plainest form.
|
|
51
|
-
return attempt('claude', ['-p', prompt]);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
if (agentType === 'opencode') {
|
|
55
|
-
return attempt('opencode', ['run', prompt]);
|
|
85
|
+
const attempts = agentAttempts(agentType, prompt, opts.schema);
|
|
86
|
+
let lastErr: any;
|
|
87
|
+
for (const a of attempts) {
|
|
88
|
+
try {
|
|
89
|
+
return exec(a.cmd, a.args, { timeout, maxBuffer });
|
|
90
|
+
} catch (err) {
|
|
91
|
+
lastErr = err; // try the next (fallback) form
|
|
56
92
|
}
|
|
57
|
-
|
|
58
|
-
|
|
93
|
+
}
|
|
94
|
+
throw wrapJudgeError(lastErr, agentType);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Non-blocking twin of {@link runAgentRaw}. Uses `execFile` (async) so the
|
|
99
|
+
* caller's event loop stays free — letting a spinner/progress UI animate while
|
|
100
|
+
* the judge (which can take minutes) runs. Prefer this in interactive commands.
|
|
101
|
+
*/
|
|
102
|
+
export async function runAgentRawAsync(prompt: string, opts: JudgeOptions = {}): Promise<string> {
|
|
103
|
+
const agentType = opts.agentType ?? getPrimaryAgent().type;
|
|
104
|
+
const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
105
|
+
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
106
|
+
const exec =
|
|
107
|
+
opts.execAsync ??
|
|
108
|
+
(async (cmd, args, o) =>
|
|
109
|
+
(await execFileAsync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer })).stdout.toString());
|
|
110
|
+
|
|
111
|
+
const attempts = agentAttempts(agentType, prompt, opts.schema);
|
|
112
|
+
let lastErr: any;
|
|
113
|
+
for (const a of attempts) {
|
|
59
114
|
try {
|
|
60
|
-
return
|
|
61
|
-
} catch {
|
|
62
|
-
|
|
115
|
+
return await exec(a.cmd, a.args, { timeout, maxBuffer });
|
|
116
|
+
} catch (err) {
|
|
117
|
+
lastErr = err; // try the next (fallback) form
|
|
63
118
|
}
|
|
64
|
-
} catch (err: any) {
|
|
65
|
-
const detail = (err?.stderr || err?.stdout || err?.message || 'unknown error').toString().trim().slice(0, 500);
|
|
66
|
-
throw new Error(`agent judge failed (${agentType}): ${detail}`);
|
|
67
119
|
}
|
|
120
|
+
throw wrapJudgeError(lastErr, agentType);
|
|
68
121
|
}
|
|
69
122
|
|
|
70
123
|
/**
|
|
@@ -77,6 +130,12 @@ export function runAgentJson(prompt: string, opts: JudgeOptions = {}): unknown {
|
|
|
77
130
|
return parseAgentJson(raw);
|
|
78
131
|
}
|
|
79
132
|
|
|
133
|
+
/** Non-blocking twin of {@link runAgentJson}. */
|
|
134
|
+
export async function runAgentJsonAsync(prompt: string, opts: JudgeOptions = {}): Promise<unknown> {
|
|
135
|
+
const raw = await runAgentRawAsync(prompt, opts);
|
|
136
|
+
return parseAgentJson(raw);
|
|
137
|
+
}
|
|
138
|
+
|
|
80
139
|
/** Extract a JSON object from an agent's raw stdout. Exported for tests. */
|
|
81
140
|
export function parseAgentJson(raw: string): unknown {
|
|
82
141
|
let text = raw.trim();
|
package/src/utils/reflect.ts
CHANGED
|
@@ -56,10 +56,14 @@ export interface ReflectionReport {
|
|
|
56
56
|
|
|
57
57
|
// --------------------------------------------------------- transcript distill
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
59
|
+
// Kept deliberately lean: the judge runs on the user's own (often local, slow)
|
|
60
|
+
// agent, and a 10-workspace corpus at full verbosity produced a ~200KB prompt
|
|
61
|
+
// that timed pi out. These bounds capture the pattern of a session at a
|
|
62
|
+
// fraction of the tokens (~halving the prompt), so the judge actually finishes.
|
|
63
|
+
const MAX_PROMPTS = 12;
|
|
64
|
+
const MAX_ERRORS = 15;
|
|
65
|
+
const PROMPT_CHARS = 400;
|
|
66
|
+
const ERROR_CHARS = 200;
|
|
63
67
|
|
|
64
68
|
/** Flatten a message `content` (string or content-block array) to plain text. */
|
|
65
69
|
function contentToText(content: unknown): string {
|
|
@@ -251,13 +255,25 @@ async function contextFilesFor(workspacePath: string): Promise<string[]> {
|
|
|
251
255
|
return present;
|
|
252
256
|
}
|
|
253
257
|
|
|
258
|
+
/** Fired as each workspace is read + distilled, so the CLI can show live,
|
|
259
|
+
* per-workspace progress during the (I/O-bound) gather phase. */
|
|
260
|
+
export interface ReflectProgress {
|
|
261
|
+
index: number;
|
|
262
|
+
total: number;
|
|
263
|
+
digest: WorkspaceDigest;
|
|
264
|
+
}
|
|
265
|
+
|
|
254
266
|
/**
|
|
255
267
|
* Build the corpus the judge reasons over: the `limit` most **recently active**
|
|
256
268
|
* workspaces (by their latest agent session, not creation date — a retrospective
|
|
257
269
|
* is about recent *work*), each with its repos, context files, and distilled
|
|
258
|
-
* session, plus the globally-available skills.
|
|
270
|
+
* session, plus the globally-available skills. `onProgress` (optional) fires
|
|
271
|
+
* once per workspace as it finishes, for a live progress display.
|
|
259
272
|
*/
|
|
260
|
-
export async function gatherReflectionCorpus(
|
|
273
|
+
export async function gatherReflectionCorpus(
|
|
274
|
+
limit: number,
|
|
275
|
+
onProgress?: (p: ReflectProgress) => void,
|
|
276
|
+
): Promise<ReflectionCorpus> {
|
|
261
277
|
const [sessions, workspaces, availableSkills] = await Promise.all([
|
|
262
278
|
getWorkspaceSessions(), // already sorted by last-active, one per workspace
|
|
263
279
|
listWorkspaces(false),
|
|
@@ -267,15 +283,18 @@ export async function gatherReflectionCorpus(limit: number): Promise<ReflectionC
|
|
|
267
283
|
const recent = sessions.slice(0, limit);
|
|
268
284
|
|
|
269
285
|
const digests: WorkspaceDigest[] = [];
|
|
270
|
-
for (
|
|
286
|
+
for (let index = 0; index < recent.length; index++) {
|
|
287
|
+
const s = recent[index];
|
|
271
288
|
const meta = metaByName.get(s.workspaceName);
|
|
272
|
-
|
|
289
|
+
const digest: WorkspaceDigest = {
|
|
273
290
|
name: s.workspaceName,
|
|
274
291
|
repoCount: meta?.metadata?.repositories?.length ?? 0,
|
|
275
292
|
repos: (meta?.metadata?.repositories ?? []).map((r) => r.name),
|
|
276
293
|
contextFiles: await contextFilesFor(s.workspacePath),
|
|
277
294
|
session: await readDigestForSession(s),
|
|
278
|
-
}
|
|
295
|
+
};
|
|
296
|
+
digests.push(digest);
|
|
297
|
+
onProgress?.({ index, total: recent.length, digest });
|
|
279
298
|
}
|
|
280
299
|
|
|
281
300
|
return { generatedAt: new Date().toISOString(), availableSkills, workspaces: digests };
|