@nemus-cli/nemus 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -11
- package/dist/cli/ai-prompt.js +4 -0
- package/dist/commands/reflect.js +41 -8
- package/dist/utils/agent-judge.js +89 -12
- package/dist/utils/reflect-analyze.js +136 -0
- package/dist/utils/reflect.js +99 -42
- package/package.json +1 -1
- package/src/cli/ai-prompt.ts +4 -0
- package/src/commands/reflect.ts +52 -10
- package/src/utils/agent-judge.test.ts +37 -2
- package/src/utils/agent-judge.ts +98 -15
- package/src/utils/reflect-analyze.test.ts +126 -0
- package/src/utils/reflect-analyze.ts +246 -0
- package/src/utils/reflect.test.ts +50 -27
- package/src/utils/reflect.ts +111 -59
package/dist/utils/reflect.js
CHANGED
|
@@ -33,14 +33,17 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.REFLECT_SCHEMA = void 0;
|
|
36
|
+
exports.REFLECT_REPORTS_DIR = exports.REFLECT_SCHEMA = void 0;
|
|
37
|
+
exports.isCorrectionPrompt = isCorrectionPrompt;
|
|
37
38
|
exports.distillTranscript = distillTranscript;
|
|
39
|
+
exports.classifyAgentsMd = classifyAgentsMd;
|
|
38
40
|
exports.findLatestTranscriptFile = findLatestTranscriptFile;
|
|
39
41
|
exports.gatherReflectionCorpus = gatherReflectionCorpus;
|
|
40
|
-
exports.buildJudgePrompt = buildJudgePrompt;
|
|
41
42
|
exports.parseReflectionReport = parseReflectionReport;
|
|
43
|
+
exports.saveReflectionReport = saveReflectionReport;
|
|
42
44
|
const fs = __importStar(require("fs/promises"));
|
|
43
45
|
const path = __importStar(require("path"));
|
|
46
|
+
const config_1 = require("./config");
|
|
44
47
|
const workspace_meta_1 = require("./workspace-meta");
|
|
45
48
|
const agent_config_1 = require("./agent-config");
|
|
46
49
|
const claude_sessions_1 = require("./claude-sessions");
|
|
@@ -51,8 +54,18 @@ const claude_sessions_1 = require("./claude-sessions");
|
|
|
51
54
|
// fraction of the tokens (~halving the prompt), so the judge actually finishes.
|
|
52
55
|
const MAX_PROMPTS = 12;
|
|
53
56
|
const MAX_ERRORS = 15;
|
|
57
|
+
const MAX_RESTEER = 6;
|
|
54
58
|
const PROMPT_CHARS = 400;
|
|
55
59
|
const ERROR_CHARS = 200;
|
|
60
|
+
const RESTEER_CHARS = 240;
|
|
61
|
+
// A conservative “the user corrected / redirected the agent” cue. Soft signal:
|
|
62
|
+
// used to count re-steers and to capture the verbatim message for the judge.
|
|
63
|
+
const CORRECTION_RE = /\b(no|nope|wrong|incorrect|revert|undo|instead|actually|you (missed|forgot|broke|didn'?t)|that'?s? (wrong|not right|incorrect)|not what|don'?t)\b/i;
|
|
64
|
+
/** Whether a single user prompt reads like a correction/redirect. Exported +
|
|
65
|
+
* shared with the analyzer so the two never diverge. */
|
|
66
|
+
function isCorrectionPrompt(prompt) {
|
|
67
|
+
return CORRECTION_RE.test(prompt ?? '');
|
|
68
|
+
}
|
|
56
69
|
/** Flatten a message `content` (string or content-block array) to plain text. */
|
|
57
70
|
function contentToText(content) {
|
|
58
71
|
if (typeof content === 'string')
|
|
@@ -90,6 +103,7 @@ function isToolFailure(flag, text) {
|
|
|
90
103
|
*/
|
|
91
104
|
function distillTranscript(raw, meta) {
|
|
92
105
|
const userPrompts = [];
|
|
106
|
+
const reSteerSamples = [];
|
|
93
107
|
const errors = [];
|
|
94
108
|
const tools = new Set();
|
|
95
109
|
let turns = 0;
|
|
@@ -143,13 +157,57 @@ function distillTranscript(raw, meta) {
|
|
|
143
157
|
const isToolResultOnly = Array.isArray(content) && content.length > 0 && content.every((b) => b?.type === 'tool_result');
|
|
144
158
|
if (!isToolResultOnly) {
|
|
145
159
|
const text = contentToText(content).trim();
|
|
146
|
-
if (text && !text.startsWith('<')
|
|
147
|
-
userPrompts.
|
|
160
|
+
if (text && !text.startsWith('<')) {
|
|
161
|
+
if (userPrompts.length < MAX_PROMPTS)
|
|
162
|
+
userPrompts.push(text.slice(0, PROMPT_CHARS));
|
|
163
|
+
// Capture corrections verbatim (bounded) — the sharpest coaching signal.
|
|
164
|
+
if (reSteerSamples.length < MAX_RESTEER && isCorrectionPrompt(text)) {
|
|
165
|
+
reSteerSamples.push(text.slice(0, RESTEER_CHARS));
|
|
166
|
+
}
|
|
148
167
|
}
|
|
149
168
|
}
|
|
150
169
|
}
|
|
151
170
|
}
|
|
152
|
-
return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, errors, tools: [...tools] };
|
|
171
|
+
return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, reSteerSamples, errors, tools: [...tools] };
|
|
172
|
+
}
|
|
173
|
+
// ------------------------------------------------------- context classification
|
|
174
|
+
// Lines that are structural/boilerplate rather than real, custom guidance.
|
|
175
|
+
const BOILERPLATE_MARKERS = [
|
|
176
|
+
'ws-rules:',
|
|
177
|
+
'this workspace was created with',
|
|
178
|
+
'workspace manager',
|
|
179
|
+
'saved context',
|
|
180
|
+
'add your own notes here',
|
|
181
|
+
'common workflows',
|
|
182
|
+
];
|
|
183
|
+
/**
|
|
184
|
+
* Classify an AGENTS.md/CLAUDE.md by how much *real* guidance it carries, so the
|
|
185
|
+
* judge can tell “no context” from “has a file but it's the generated template.”
|
|
186
|
+
* Heuristic + pure.
|
|
187
|
+
*
|
|
188
|
+
* Deliberately **newline-independent**: it measures the volume of non-boilerplate
|
|
189
|
+
* prose (word count) plus heading count via a whitespace-tolerant regex, rather
|
|
190
|
+
* than splitting on lines. A line-anchored version would misclassify any excerpt
|
|
191
|
+
* whose newlines were collapsed to spaces upstream (a real bug class caught in a
|
|
192
|
+
* sibling implementation) — here even a fully single-lined file classifies the
|
|
193
|
+
* same as its multi-line original.
|
|
194
|
+
*/
|
|
195
|
+
function classifyAgentsMd(content) {
|
|
196
|
+
if (!content || !content.trim())
|
|
197
|
+
return 'missing';
|
|
198
|
+
let s = content.replace(/\r\n/g, '\n').toLowerCase();
|
|
199
|
+
s = s.replace(/```[\s\S]*?```/g, ' '); // drop fenced code
|
|
200
|
+
s = s.replace(/<!--[\s\S]*?-->/g, ' '); // drop HTML comments
|
|
201
|
+
for (const m of BOILERPLATE_MARKERS)
|
|
202
|
+
s = s.split(m).join(' '); // drop generated boilerplate (markers are lowercase)
|
|
203
|
+
// Headings: a `#` run at start OR after any whitespace (so a collapsed,
|
|
204
|
+
// single-line excerpt still counts them), followed by a space.
|
|
205
|
+
const headings = (s.match(/(?:^|\s)#{1,6}\s/g) || []).length;
|
|
206
|
+
// Remaining non-boilerplate words (markdown punctuation stripped).
|
|
207
|
+
const words = s.replace(/[#|>*_`~-]/g, ' ').split(/\s+/).filter((w) => w.length > 1).length;
|
|
208
|
+
if (words < 40)
|
|
209
|
+
return 'boilerplate';
|
|
210
|
+
return headings >= 2 || words >= 60 ? 'substantive' : 'boilerplate';
|
|
153
211
|
}
|
|
154
212
|
// --------------------------------------------------------- corpus gathering
|
|
155
213
|
/** Locate the most recent `.jsonl` transcript for a workspace under an agent. */
|
|
@@ -224,18 +282,25 @@ async function listAvailableSkills() {
|
|
|
224
282
|
}
|
|
225
283
|
return [...names].sort();
|
|
226
284
|
}
|
|
285
|
+
/** Which context files exist at the workspace root, plus how substantive the
|
|
286
|
+
* primary one is (missing/boilerplate/substantive). One read per present file. */
|
|
227
287
|
async function contextFilesFor(workspacePath) {
|
|
228
288
|
const present = [];
|
|
289
|
+
let quality = 'missing';
|
|
229
290
|
for (const name of (0, agent_config_1.getAllKnownContextFileNames)()) {
|
|
230
291
|
try {
|
|
231
|
-
await fs.
|
|
292
|
+
const content = await fs.readFile(path.join(workspacePath, name), 'utf-8');
|
|
232
293
|
present.push(name);
|
|
294
|
+
// Classify the first present file, then keep the best classification seen.
|
|
295
|
+
const c = classifyAgentsMd(content);
|
|
296
|
+
if (quality === 'missing' || (quality === 'boilerplate' && c === 'substantive'))
|
|
297
|
+
quality = c;
|
|
233
298
|
}
|
|
234
299
|
catch {
|
|
235
|
-
/* not present */
|
|
300
|
+
/* not present / unreadable */
|
|
236
301
|
}
|
|
237
302
|
}
|
|
238
|
-
return present;
|
|
303
|
+
return { files: present, quality };
|
|
239
304
|
}
|
|
240
305
|
/**
|
|
241
306
|
* Build the corpus the judge reasons over: the `limit` most **recently active**
|
|
@@ -244,23 +309,28 @@ async function contextFilesFor(workspacePath) {
|
|
|
244
309
|
* session, plus the globally-available skills. `onProgress` (optional) fires
|
|
245
310
|
* once per workspace as it finishes, for a live progress display.
|
|
246
311
|
*/
|
|
247
|
-
async function gatherReflectionCorpus(limit, onProgress) {
|
|
312
|
+
async function gatherReflectionCorpus(limit, onProgress, opts = {}) {
|
|
248
313
|
const [sessions, workspaces, availableSkills] = await Promise.all([
|
|
249
314
|
(0, claude_sessions_1.getWorkspaceSessions)(), // already sorted by last-active, one per workspace
|
|
250
315
|
(0, workspace_meta_1.listWorkspaces)(false),
|
|
251
316
|
listAvailableSkills(),
|
|
252
317
|
]);
|
|
253
318
|
const metaByName = new Map(workspaces.map((w) => [w.name, w]));
|
|
254
|
-
|
|
319
|
+
// A single named workspace (ignores limit), else the N most recently active.
|
|
320
|
+
const recent = opts.workspace
|
|
321
|
+
? sessions.filter((s) => s.workspaceName === opts.workspace)
|
|
322
|
+
: sessions.slice(0, limit);
|
|
255
323
|
const digests = [];
|
|
256
324
|
for (let index = 0; index < recent.length; index++) {
|
|
257
325
|
const s = recent[index];
|
|
258
326
|
const meta = metaByName.get(s.workspaceName);
|
|
327
|
+
const context = await contextFilesFor(s.workspacePath);
|
|
259
328
|
const digest = {
|
|
260
329
|
name: s.workspaceName,
|
|
261
330
|
repoCount: meta?.metadata?.repositories?.length ?? 0,
|
|
262
331
|
repos: (meta?.metadata?.repositories ?? []).map((r) => r.name),
|
|
263
|
-
contextFiles:
|
|
332
|
+
contextFiles: context.files,
|
|
333
|
+
contextQuality: context.quality,
|
|
264
334
|
session: await readDigestForSession(s),
|
|
265
335
|
};
|
|
266
336
|
digests.push(digest);
|
|
@@ -292,37 +362,8 @@ exports.REFLECT_SCHEMA = JSON.stringify({
|
|
|
292
362
|
},
|
|
293
363
|
required: ['summary', 'recommendations'],
|
|
294
364
|
});
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
* is asked to recommend concrete improvements to the user's SETUP (skills,
|
|
298
|
-
* AGENTS.md/context rules, connectivity/tests, prompt habits, workflow) — not to
|
|
299
|
-
* redo the tasks. Output is strict JSON matching REFLECT_SCHEMA.
|
|
300
|
-
*/
|
|
301
|
-
function buildJudgePrompt(corpus) {
|
|
302
|
-
const lines = [];
|
|
303
|
-
lines.push('You are an expert reviewer ("LLM as a judge") analyzing an engineer\'s recent AI coding-agent sessions.', 'Goal: recommend concrete improvements to their SETUP so the agent works better next time —', 'which skills to add and WHERE, which AGENTS.md/context rules are missing, missing connectivity/', 'smoke tests, and prompt habits to change. Judge the setup, do NOT redo the tasks.', '', 'Base every recommendation on evidence in the sessions below (repeated failures, retries, vague', 'prompts, missing context). Prefer a few high-signal, actionable items over many generic ones.', 'When you suggest a skill or an AGENTS.md rule, include a short concrete `example` snippet.', '', `Globally installed skills (don't re-suggest these; suggest genuinely missing ones): ${corpus.availableSkills.join(', ') || '(none)'}`, '', `Recent workspaces (${corpus.workspaces.length}):`);
|
|
304
|
-
for (const ws of corpus.workspaces) {
|
|
305
|
-
lines.push(`\n## ${ws.name}`);
|
|
306
|
-
lines.push(`repos: ${ws.repos.join(', ') || '(none)'} | context files: ${ws.contextFiles.join(', ') || 'NONE'}`);
|
|
307
|
-
if (!ws.session) {
|
|
308
|
-
lines.push('session: (no recent agent session found)');
|
|
309
|
-
continue;
|
|
310
|
-
}
|
|
311
|
-
lines.push(`session: ${ws.session.turns} turns, tools used: ${ws.session.tools.join(', ') || '(none)'}`);
|
|
312
|
-
if (ws.session.userPrompts.length) {
|
|
313
|
-
lines.push('user prompts:');
|
|
314
|
-
for (const p of ws.session.userPrompts)
|
|
315
|
-
lines.push(` - ${p.replace(/\n/g, ' ')}`);
|
|
316
|
-
}
|
|
317
|
-
if (ws.session.errors.length) {
|
|
318
|
-
lines.push('errors/failures observed:');
|
|
319
|
-
for (const e of ws.session.errors)
|
|
320
|
-
lines.push(` - ${e.replace(/\n/g, ' ')}`);
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
lines.push('', 'Respond with ONLY a JSON object of this shape (no prose, no markdown fence):', '{"summary": string, "recommendations": [{"kind":"skill|context|test|prompt|connectivity|workflow|other",', '"title": string, "detail": string, "target": string(optional workspace/repo/path),', '"priority":"high|medium|low", "example": string(optional snippet)}]}');
|
|
324
|
-
return lines.join('\n');
|
|
325
|
-
}
|
|
365
|
+
// The judge prompt is now built from pre-computed FACTS (see reflect-analyze.ts
|
|
366
|
+
// `buildAnalysisPrompt`), not raw transcripts, so the LLM call stays small/fast.
|
|
326
367
|
// ----------------------------------------------------------- response parse
|
|
327
368
|
const KINDS = ['skill', 'context', 'test', 'prompt', 'connectivity', 'workflow', 'other'];
|
|
328
369
|
const PRIORITIES = ['high', 'medium', 'low'];
|
|
@@ -351,3 +392,19 @@ function parseReflectionReport(parsed) {
|
|
|
351
392
|
.filter((r) => r !== null);
|
|
352
393
|
return { summary, recommendations };
|
|
353
394
|
}
|
|
395
|
+
// -------------------------------------------------------------- report saving
|
|
396
|
+
/** Where saved reflection reports live: `~/.nemus/reflect/`. */
|
|
397
|
+
exports.REFLECT_REPORTS_DIR = path.join(config_1.CACHE_DIR, 'reflect');
|
|
398
|
+
/**
|
|
399
|
+
* Persist a reflection report as timestamped JSON under `~/.nemus/reflect/`, so
|
|
400
|
+
* a run can be revisited or diffed over time. Returns the written path. Pure
|
|
401
|
+
* side-effect (mkdir -p + write); callers treat failure as non-fatal.
|
|
402
|
+
*/
|
|
403
|
+
async function saveReflectionReport(report, meta) {
|
|
404
|
+
await fs.mkdir(exports.REFLECT_REPORTS_DIR, { recursive: true });
|
|
405
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
406
|
+
const scope = meta.workspace ? `-${meta.workspace.replace(/[^a-zA-Z0-9_-]+/g, '_')}` : '';
|
|
407
|
+
const file = path.join(exports.REFLECT_REPORTS_DIR, `${stamp}${scope}.json`);
|
|
408
|
+
await fs.writeFile(file, JSON.stringify({ generatedAt: new Date().toISOString(), ...meta, ...report }, null, 2));
|
|
409
|
+
return file;
|
|
410
|
+
}
|
package/package.json
CHANGED
package/src/cli/ai-prompt.ts
CHANGED
|
@@ -140,6 +140,10 @@ function runExtraction(cmd: string, fullArgs: string[], fallbackArgs?: string[])
|
|
|
140
140
|
const exec = (args: string[]) =>
|
|
141
141
|
execFileSync(cmd, args, {
|
|
142
142
|
encoding: 'utf-8',
|
|
143
|
+
// `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
|
|
144
|
+
// can't block this synchronous call forever in a headless/piped context
|
|
145
|
+
// — the same hang the async judge hit, guarded here for `nemus -- "…"`.
|
|
146
|
+
input: '',
|
|
143
147
|
timeout: EXTRACTION_TIMEOUT_MS,
|
|
144
148
|
maxBuffer: 10 * 1024 * 1024,
|
|
145
149
|
});
|
package/src/commands/reflect.ts
CHANGED
|
@@ -4,13 +4,14 @@ import { outputJson, outputJsonError } from '../utils/output';
|
|
|
4
4
|
import { colorize } from '../utils/colors';
|
|
5
5
|
import {
|
|
6
6
|
gatherReflectionCorpus,
|
|
7
|
-
buildJudgePrompt,
|
|
8
7
|
parseReflectionReport,
|
|
8
|
+
saveReflectionReport,
|
|
9
9
|
REFLECT_SCHEMA,
|
|
10
10
|
ReflectionReport,
|
|
11
11
|
ReflectProgress,
|
|
12
12
|
Recommendation,
|
|
13
13
|
} from '../utils/reflect';
|
|
14
|
+
import { analyzeCorpus, buildAnalysisPrompt } from '../utils/reflect-analyze';
|
|
14
15
|
import { runAgentJsonAsync } from '../utils/agent-judge';
|
|
15
16
|
|
|
16
17
|
export function registerReflectCommand(parent: Command) {
|
|
@@ -19,28 +20,50 @@ export function registerReflectCommand(parent: Command) {
|
|
|
19
20
|
.alias('retro')
|
|
20
21
|
.description('Analyze your recent workspace sessions and suggest skill/prompt/context improvements (LLM-as-a-judge)')
|
|
21
22
|
.option('-n, --limit <n>', 'How many recent workspaces to analyze', '10')
|
|
23
|
+
.option('-w, --workspace <name>', 'Analyze a single workspace by name (ignores --limit)')
|
|
24
|
+
.option('--model <model>', 'Judge model override (agent-native pattern/id)')
|
|
25
|
+
.option('--thinking <level>', 'Judge thinking level for pi: off|minimal|low|medium|high|xhigh|max')
|
|
22
26
|
.option('--json', 'Output the report as JSON')
|
|
27
|
+
.option('--no-save', 'Do not save the report to ~/.nemus/reflect/')
|
|
23
28
|
.option('--dry-run', 'Print the assembled corpus + judge prompt without calling the agent')
|
|
24
29
|
.action(async (opts) => {
|
|
25
30
|
await handleReflect(opts);
|
|
26
31
|
});
|
|
27
32
|
}
|
|
28
33
|
|
|
29
|
-
async function handleReflect(opts: {
|
|
34
|
+
async function handleReflect(opts: {
|
|
35
|
+
limit?: string;
|
|
36
|
+
workspace?: string;
|
|
37
|
+
model?: string;
|
|
38
|
+
thinking?: string;
|
|
39
|
+
json?: boolean;
|
|
40
|
+
save?: boolean; // commander sets `save: false` for --no-save
|
|
41
|
+
dryRun?: boolean;
|
|
42
|
+
}) {
|
|
30
43
|
const limit = Math.max(1, Number.parseInt(opts.limit ?? '10', 10) || 10);
|
|
31
44
|
try {
|
|
32
45
|
const showProgress = !opts.json && !opts.dryRun;
|
|
33
46
|
if (showProgress) {
|
|
34
|
-
logStep(
|
|
47
|
+
logStep(
|
|
48
|
+
opts.workspace
|
|
49
|
+
? `Analyzing workspace ${colorize(opts.workspace, 'cyan')}…`
|
|
50
|
+
: `Analyzing your ${colorize(String(limit), 'cyan')} most recent workspaces…`,
|
|
51
|
+
);
|
|
35
52
|
}
|
|
36
53
|
|
|
37
|
-
const corpus = await gatherReflectionCorpus(limit, showProgress ? printProgress : undefined
|
|
54
|
+
const corpus = await gatherReflectionCorpus(limit, showProgress ? printProgress : undefined, {
|
|
55
|
+
workspace: opts.workspace,
|
|
56
|
+
});
|
|
38
57
|
const withSessions = corpus.workspaces.filter((w) => w.session).length;
|
|
39
|
-
|
|
58
|
+
// A script does the heavy analysis (clustering failures, counting tools,
|
|
59
|
+
// spotting correction loops); the LLM only ever sees these compact facts,
|
|
60
|
+
// so the judge call stays small + fast regardless of workspace count.
|
|
61
|
+
const analysis = analyzeCorpus(corpus);
|
|
62
|
+
const prompt = buildAnalysisPrompt(analysis);
|
|
40
63
|
|
|
41
64
|
if (opts.dryRun) {
|
|
42
|
-
// No LLM call — surface exactly what the judge
|
|
43
|
-
if (opts.json) outputJson({
|
|
65
|
+
// No LLM call — surface the computed facts + exactly what the judge sees.
|
|
66
|
+
if (opts.json) outputJson({ analysis, prompt });
|
|
44
67
|
else {
|
|
45
68
|
process.stdout.write(prompt + '\n');
|
|
46
69
|
}
|
|
@@ -48,7 +71,9 @@ async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: bo
|
|
|
48
71
|
}
|
|
49
72
|
|
|
50
73
|
if (withSessions === 0) {
|
|
51
|
-
const msg =
|
|
74
|
+
const msg = opts.workspace
|
|
75
|
+
? `No recent agent session found for workspace "${opts.workspace}" (need a Claude/pi transcript).`
|
|
76
|
+
: 'No recent agent sessions found to analyze (need Claude/pi session transcripts).';
|
|
52
77
|
if (opts.json) outputJsonError(msg);
|
|
53
78
|
else logError(msg);
|
|
54
79
|
process.exit(1);
|
|
@@ -58,22 +83,39 @@ async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: bo
|
|
|
58
83
|
// (non-blocking) so a live spinner shows it's alive, not hung. Timeout is
|
|
59
84
|
// overridable for slow local models.
|
|
60
85
|
const timeoutMs = Number.parseInt(process.env.NEMUS_JUDGE_TIMEOUT_MS ?? '', 10) || undefined;
|
|
86
|
+
const model = opts.model ?? process.env.NEMUS_JUDGE_MODEL ?? undefined;
|
|
87
|
+
const thinking = opts.thinking ?? process.env.NEMUS_JUDGE_THINKING ?? undefined;
|
|
61
88
|
const stopSpinner = opts.json
|
|
62
89
|
? () => {}
|
|
63
90
|
: startSpinner(`Judging ${withSessions} session(s) with your configured agent (this can take a minute)…`);
|
|
64
91
|
let parsed: unknown;
|
|
65
92
|
try {
|
|
66
|
-
parsed = await runAgentJsonAsync(prompt, { schema: REFLECT_SCHEMA, timeoutMs });
|
|
93
|
+
parsed = await runAgentJsonAsync(prompt, { schema: REFLECT_SCHEMA, timeoutMs, model, thinking });
|
|
67
94
|
} finally {
|
|
68
95
|
stopSpinner();
|
|
69
96
|
}
|
|
70
97
|
const report = parseReflectionReport(parsed);
|
|
71
98
|
|
|
99
|
+
// Persist the report (best-effort; never fails the run) unless --no-save.
|
|
100
|
+
let savedTo: string | undefined;
|
|
101
|
+
if (opts.save !== false) {
|
|
102
|
+
try {
|
|
103
|
+
savedTo = await saveReflectionReport(report, {
|
|
104
|
+
analyzed: withSessions,
|
|
105
|
+
workspaces: corpus.workspaces.length,
|
|
106
|
+
workspace: opts.workspace,
|
|
107
|
+
});
|
|
108
|
+
} catch {
|
|
109
|
+
/* saving is a convenience, not the point */
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
72
113
|
if (opts.json) {
|
|
73
|
-
outputJson({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report });
|
|
114
|
+
outputJson({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report, savedTo });
|
|
74
115
|
return;
|
|
75
116
|
}
|
|
76
117
|
printReport(report, corpus.workspaces.length, withSessions);
|
|
118
|
+
if (savedTo) logInfo(`Saved report to ${colorize(savedTo, 'dim')}`);
|
|
77
119
|
} catch (error) {
|
|
78
120
|
const msg = error instanceof Error ? error.message : 'reflect failed';
|
|
79
121
|
if (opts.json) outputJsonError(msg);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { parseAgentJson, runAgentRaw, runAgentRawAsync, agentAttempts } from './agent-judge';
|
|
2
|
+
import { parseAgentJson, runAgentRaw, runAgentRawAsync, agentAttempts, spawnCollect } from './agent-judge';
|
|
3
3
|
|
|
4
4
|
describe('parseAgentJson', () => {
|
|
5
5
|
it('unwraps the common agent envelopes and shapes', () => {
|
|
@@ -16,9 +16,24 @@ describe('parseAgentJson', () => {
|
|
|
16
16
|
});
|
|
17
17
|
});
|
|
18
18
|
|
|
19
|
+
describe('spawnCollect (stdin-hang regression)', () => {
|
|
20
|
+
it('gives the child stdin EOF so a stdin-reading process does NOT hang', async () => {
|
|
21
|
+
// `cat` with no args reads stdin to EOF. If stdin were an open pipe (the old
|
|
22
|
+
// execFile default) this would block until the timeout and reject; with
|
|
23
|
+
// stdin ignored it gets immediate EOF and exits 0 fast. 2s timeout << any hang.
|
|
24
|
+
const out = await spawnCollect('cat', [], { timeout: 2000, maxBuffer: 1 << 20 });
|
|
25
|
+
expect(out).toBe('');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('captures stdout and rejects on non-zero exit', async () => {
|
|
29
|
+
await expect(spawnCollect('node', ['-e', 'process.stdout.write("hi")'], { timeout: 5000, maxBuffer: 1 << 20 })).resolves.toBe('hi');
|
|
30
|
+
await expect(spawnCollect('node', ['-e', 'process.exit(3)'], { timeout: 5000, maxBuffer: 1 << 20 })).rejects.toThrow(/exit 3/);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
19
34
|
describe('agentAttempts', () => {
|
|
20
35
|
it('claude: preferred (schema + lean flags) then a plain fallback', () => {
|
|
21
|
-
const a = agentAttempts('claude', 'P', '{"type":"object"}');
|
|
36
|
+
const a = agentAttempts('claude', 'P', { schema: '{"type":"object"}' });
|
|
22
37
|
expect(a[0]).toEqual({ cmd: 'claude', args: expect.arrayContaining(['-p', 'P', '--output-format', 'json', '--json-schema', '{"type":"object"}']) });
|
|
23
38
|
expect(a[1]).toEqual({ cmd: 'claude', args: ['-p', 'P'] });
|
|
24
39
|
});
|
|
@@ -28,6 +43,26 @@ describe('agentAttempts', () => {
|
|
|
28
43
|
expect(pi[1]).toEqual({ cmd: 'pi', args: ['-p', 'P'] });
|
|
29
44
|
expect(agentAttempts('opencode', 'P')).toEqual([{ cmd: 'opencode', args: ['run', 'P'] }]);
|
|
30
45
|
});
|
|
46
|
+
it('threads --model (all) + --thinking (pi only); fallback keeps model, drops --thinking', () => {
|
|
47
|
+
const pi = agentAttempts('pi', 'P', { model: 'haiku', thinking: 'low' });
|
|
48
|
+
expect(pi[0].args).toEqual(expect.arrayContaining(['--model', 'haiku', '--thinking', 'low', '-p', 'P']));
|
|
49
|
+
// Safety net: model stays (stable flag), --thinking is dropped so an old pi
|
|
50
|
+
// that rejects --thinking still has a working fallback.
|
|
51
|
+
expect(pi[1].args).toEqual(['--model', 'haiku', '-p', 'P']);
|
|
52
|
+
expect(pi[1].args).not.toContain('--thinking');
|
|
53
|
+
const cl = agentAttempts('claude', 'P', { model: 'sonnet' });
|
|
54
|
+
expect(cl[0].args).toEqual(expect.arrayContaining(['--model', 'sonnet']));
|
|
55
|
+
expect(cl[0].args).not.toContain('--thinking'); // thinking is pi-only
|
|
56
|
+
expect(agentAttempts('opencode', 'P', { model: 'gpt' })[0].args).toEqual(['run', 'P', '--model', 'gpt']);
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
describe('runAgentRaw thinking default', () => {
|
|
61
|
+
it('applies the low-thinking default for pi', () => {
|
|
62
|
+
let seen: string[] = [];
|
|
63
|
+
runAgentRaw('P', { agentType: 'pi', exec: (cmd, args) => ((seen = [cmd, ...args]), 'ok') });
|
|
64
|
+
expect(seen).toEqual(expect.arrayContaining(['--thinking', 'low']));
|
|
65
|
+
});
|
|
31
66
|
});
|
|
32
67
|
|
|
33
68
|
describe('runAgentRawAsync', () => {
|
package/src/utils/agent-judge.ts
CHANGED
|
@@ -1,8 +1,51 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { promisify } from 'util';
|
|
1
|
+
import { execFileSync, spawn } from 'child_process';
|
|
3
2
|
import { getPrimaryAgent } from './agent-config';
|
|
4
3
|
|
|
5
|
-
|
|
4
|
+
/**
|
|
5
|
+
* Run a child to completion, capturing stdout, with stdin set to /dev/null.
|
|
6
|
+
*
|
|
7
|
+
* The stdin part is load-bearing: agents like `pi` block waiting on stdin if
|
|
8
|
+
* it's an open pipe (the default for execFile), which made the judge hang until
|
|
9
|
+
* the timeout regardless of prompt size or model speed. `stdio: ['ignore', …]`
|
|
10
|
+
* gives the child an immediate EOF, exactly like a non-interactive shell.
|
|
11
|
+
*/
|
|
12
|
+
export function spawnCollect(
|
|
13
|
+
cmd: string,
|
|
14
|
+
args: string[],
|
|
15
|
+
opts: { timeout: number; maxBuffer: number },
|
|
16
|
+
): Promise<string> {
|
|
17
|
+
return new Promise((resolve, reject) => {
|
|
18
|
+
const child = spawn(cmd, args, {
|
|
19
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
20
|
+
timeout: opts.timeout,
|
|
21
|
+
killSignal: 'SIGKILL',
|
|
22
|
+
});
|
|
23
|
+
// Decode as UTF-8 at the stream boundary so a multi-byte char split across
|
|
24
|
+
// two chunks isn't corrupted (which would break JSON.parse downstream).
|
|
25
|
+
child.stdout.setEncoding('utf8');
|
|
26
|
+
child.stderr.setEncoding('utf8');
|
|
27
|
+
let stdout = '';
|
|
28
|
+
let stderr = '';
|
|
29
|
+
let overflow = false;
|
|
30
|
+
child.stdout.on('data', (d) => {
|
|
31
|
+
stdout += d;
|
|
32
|
+
if (stdout.length > opts.maxBuffer) {
|
|
33
|
+
overflow = true;
|
|
34
|
+
child.kill('SIGKILL');
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
child.stderr.on('data', (d) => {
|
|
38
|
+
stderr += d;
|
|
39
|
+
});
|
|
40
|
+
child.on('error', (e) => reject(e));
|
|
41
|
+
child.on('close', (code, signal) => {
|
|
42
|
+
if (overflow) return reject(Object.assign(new Error('maxBuffer exceeded'), { stdout, stderr }));
|
|
43
|
+
if (signal) return reject(Object.assign(new Error(`killed by ${signal}`), { killed: true, signal, stdout, stderr }));
|
|
44
|
+
if (code !== 0) return reject(Object.assign(new Error(`exit ${code}`), { code, stdout, stderr }));
|
|
45
|
+
resolve(stdout);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
}
|
|
6
49
|
|
|
7
50
|
/**
|
|
8
51
|
* Run the user's configured coding agent headlessly as an "LLM-as-a-judge":
|
|
@@ -17,6 +60,10 @@ const execFileAsync = promisify(execFile);
|
|
|
17
60
|
export interface JudgeOptions {
|
|
18
61
|
/** JSON schema string passed to `claude --json-schema` (ignored by others). */
|
|
19
62
|
schema?: string;
|
|
63
|
+
/** Model override (`--model`), agent-native pattern/id. */
|
|
64
|
+
model?: string;
|
|
65
|
+
/** Thinking level (`--thinking`, pi only): off|minimal|low|medium|high|xhigh|max. */
|
|
66
|
+
thinking?: string;
|
|
20
67
|
timeoutMs?: number;
|
|
21
68
|
maxBuffer?: number;
|
|
22
69
|
/** Injected for tests. Defaults to the real (blocking) child_process runner. */
|
|
@@ -34,24 +81,61 @@ export interface AgentAttempt {
|
|
|
34
81
|
args: string[];
|
|
35
82
|
}
|
|
36
83
|
|
|
84
|
+
export interface AttemptOptions {
|
|
85
|
+
schema?: string;
|
|
86
|
+
model?: string;
|
|
87
|
+
thinking?: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Default thinking level for the judge. The judge is a mechanical transform
|
|
92
|
+
* (facts → recommendations), not deep reasoning, so a heavy default like Opus
|
|
93
|
+
* @ medium thinking just makes it slow. `low` keeps pi fast; override per-run.
|
|
94
|
+
*/
|
|
95
|
+
export const DEFAULT_JUDGE_THINKING = 'low';
|
|
96
|
+
|
|
37
97
|
/**
|
|
38
98
|
* The ordered invocation attempts for an agent (preferred → fallback), as pure
|
|
39
99
|
* data so both the sync and async runners share ONE flag ladder (and it's
|
|
40
|
-
* unit-testable without spawning anything).
|
|
100
|
+
* unit-testable without spawning anything). `--model` applies to all; pi also
|
|
101
|
+
* takes `--thinking` (the speed lever); both are ignored where unsupported.
|
|
41
102
|
*/
|
|
42
|
-
export function agentAttempts(agentType: JudgeAgentType, prompt: string,
|
|
103
|
+
export function agentAttempts(agentType: JudgeAgentType, prompt: string, opts: AttemptOptions = {}): AgentAttempt[] {
|
|
104
|
+
const { schema, model, thinking } = opts;
|
|
43
105
|
if (agentType === 'claude') {
|
|
44
106
|
const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
|
|
107
|
+
if (model) preferred.push('--model', model);
|
|
45
108
|
if (schema) preferred.push('--json-schema', schema);
|
|
46
109
|
// Older claude may reject the newer flags — fall back to the plainest form.
|
|
47
|
-
|
|
110
|
+
const plain = ['-p', prompt];
|
|
111
|
+
if (model) plain.push('--model', model);
|
|
112
|
+
return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: plain }];
|
|
48
113
|
}
|
|
49
114
|
if (agentType === 'opencode') {
|
|
50
|
-
|
|
115
|
+
const args = ['run', prompt];
|
|
116
|
+
if (model) args.push('--model', model);
|
|
117
|
+
return [{ cmd: 'opencode', args }];
|
|
51
118
|
}
|
|
52
119
|
// pi (and any other): run as lean as possible so a bloated env can't hang it.
|
|
53
120
|
const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
|
|
54
|
-
|
|
121
|
+
const modelArgs = model ? ['--model', model] : [];
|
|
122
|
+
const tune = thinking ? [...modelArgs, '--thinking', thinking] : modelArgs;
|
|
123
|
+
// Fallback keeps the stable --model but DROPS --thinking: --thinking is the
|
|
124
|
+
// newest flag and the most likely reason an older pi rejects the first
|
|
125
|
+
// attempt, so the safety net must not carry it (else both attempts fail).
|
|
126
|
+
return [
|
|
127
|
+
{ cmd: 'pi', args: [...piLean, ...tune, '-p', prompt] },
|
|
128
|
+
{ cmd: 'pi', args: [...modelArgs, '-p', prompt] },
|
|
129
|
+
];
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Resolve the attempt options for a run, applying the pi thinking default. */
|
|
133
|
+
function attemptOptions(agentType: JudgeAgentType, opts: JudgeOptions): AttemptOptions {
|
|
134
|
+
return {
|
|
135
|
+
schema: opts.schema,
|
|
136
|
+
model: opts.model,
|
|
137
|
+
thinking: opts.thinking ?? (agentType === 'pi' ? DEFAULT_JUDGE_THINKING : undefined),
|
|
138
|
+
};
|
|
55
139
|
}
|
|
56
140
|
|
|
57
141
|
function wrapJudgeError(err: any, agentType: string): Error {
|
|
@@ -80,9 +164,11 @@ export function runAgentRaw(prompt: string, opts: JudgeOptions = {}): string {
|
|
|
80
164
|
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
81
165
|
const exec =
|
|
82
166
|
opts.exec ??
|
|
83
|
-
|
|
167
|
+
// `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
|
|
168
|
+
// can't hang the synchronous call — the sync twin of spawnCollect's fix.
|
|
169
|
+
((cmd, args, o) => execFileSync(cmd, args, { encoding: 'utf-8', input: '', timeout: o.timeout, maxBuffer: o.maxBuffer }));
|
|
84
170
|
|
|
85
|
-
const attempts = agentAttempts(agentType, prompt, opts
|
|
171
|
+
const attempts = agentAttempts(agentType, prompt, attemptOptions(agentType, opts));
|
|
86
172
|
let lastErr: any;
|
|
87
173
|
for (const a of attempts) {
|
|
88
174
|
try {
|
|
@@ -103,12 +189,9 @@ export async function runAgentRawAsync(prompt: string, opts: JudgeOptions = {}):
|
|
|
103
189
|
const agentType = opts.agentType ?? getPrimaryAgent().type;
|
|
104
190
|
const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
105
191
|
const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
|
|
106
|
-
const exec =
|
|
107
|
-
opts.execAsync ??
|
|
108
|
-
(async (cmd, args, o) =>
|
|
109
|
-
(await execFileAsync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer })).stdout.toString());
|
|
192
|
+
const exec = opts.execAsync ?? ((cmd, args, o) => spawnCollect(cmd, args, o));
|
|
110
193
|
|
|
111
|
-
const attempts = agentAttempts(agentType, prompt, opts
|
|
194
|
+
const attempts = agentAttempts(agentType, prompt, attemptOptions(agentType, opts));
|
|
112
195
|
let lastErr: any;
|
|
113
196
|
for (const a of attempts) {
|
|
114
197
|
try {
|