@nemus-cli/nemus 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -33,14 +33,17 @@ var __importStar = (this && this.__importStar) || (function () {
33
33
  };
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.REFLECT_SCHEMA = void 0;
36
+ exports.REFLECT_REPORTS_DIR = exports.REFLECT_SCHEMA = void 0;
37
+ exports.isCorrectionPrompt = isCorrectionPrompt;
37
38
  exports.distillTranscript = distillTranscript;
39
+ exports.classifyAgentsMd = classifyAgentsMd;
38
40
  exports.findLatestTranscriptFile = findLatestTranscriptFile;
39
41
  exports.gatherReflectionCorpus = gatherReflectionCorpus;
40
- exports.buildJudgePrompt = buildJudgePrompt;
41
42
  exports.parseReflectionReport = parseReflectionReport;
43
+ exports.saveReflectionReport = saveReflectionReport;
42
44
  const fs = __importStar(require("fs/promises"));
43
45
  const path = __importStar(require("path"));
46
+ const config_1 = require("./config");
44
47
  const workspace_meta_1 = require("./workspace-meta");
45
48
  const agent_config_1 = require("./agent-config");
46
49
  const claude_sessions_1 = require("./claude-sessions");
@@ -51,8 +54,18 @@ const claude_sessions_1 = require("./claude-sessions");
51
54
  // fraction of the tokens (~halving the prompt), so the judge actually finishes.
52
55
  const MAX_PROMPTS = 12;
53
56
  const MAX_ERRORS = 15;
57
+ const MAX_RESTEER = 6;
54
58
  const PROMPT_CHARS = 400;
55
59
  const ERROR_CHARS = 200;
60
+ const RESTEER_CHARS = 240;
61
+ // A conservative “the user corrected / redirected the agent” cue. Soft signal:
62
+ // used to count re-steers and to capture the verbatim message for the judge.
63
+ const CORRECTION_RE = /\b(no|nope|wrong|incorrect|revert|undo|instead|actually|you (missed|forgot|broke|didn'?t)|that'?s? (wrong|not right|incorrect)|not what|don'?t)\b/i;
64
+ /** Whether a single user prompt reads like a correction/redirect. Exported +
65
+ * shared with the analyzer so the two never diverge. */
66
+ function isCorrectionPrompt(prompt) {
67
+ return CORRECTION_RE.test(prompt ?? '');
68
+ }
56
69
  /** Flatten a message `content` (string or content-block array) to plain text. */
57
70
  function contentToText(content) {
58
71
  if (typeof content === 'string')
@@ -90,6 +103,7 @@ function isToolFailure(flag, text) {
90
103
  */
91
104
  function distillTranscript(raw, meta) {
92
105
  const userPrompts = [];
106
+ const reSteerSamples = [];
93
107
  const errors = [];
94
108
  const tools = new Set();
95
109
  let turns = 0;
@@ -143,13 +157,57 @@ function distillTranscript(raw, meta) {
143
157
  const isToolResultOnly = Array.isArray(content) && content.length > 0 && content.every((b) => b?.type === 'tool_result');
144
158
  if (!isToolResultOnly) {
145
159
  const text = contentToText(content).trim();
146
- if (text && !text.startsWith('<') && userPrompts.length < MAX_PROMPTS) {
147
- userPrompts.push(text.slice(0, PROMPT_CHARS));
160
+ if (text && !text.startsWith('<')) {
161
+ if (userPrompts.length < MAX_PROMPTS)
162
+ userPrompts.push(text.slice(0, PROMPT_CHARS));
163
+ // Capture corrections verbatim (bounded) — the sharpest coaching signal.
164
+ if (reSteerSamples.length < MAX_RESTEER && isCorrectionPrompt(text)) {
165
+ reSteerSamples.push(text.slice(0, RESTEER_CHARS));
166
+ }
148
167
  }
149
168
  }
150
169
  }
151
170
  }
152
- return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, errors, tools: [...tools] };
171
+ return { sessionId: meta.sessionId, agentType: meta.agentType, turns, userPrompts, reSteerSamples, errors, tools: [...tools] };
172
+ }
173
+ // ------------------------------------------------------- context classification
174
+ // Lines that are structural/boilerplate rather than real, custom guidance.
175
+ const BOILERPLATE_MARKERS = [
176
+ 'ws-rules:',
177
+ 'this workspace was created with',
178
+ 'workspace manager',
179
+ 'saved context',
180
+ 'add your own notes here',
181
+ 'common workflows',
182
+ ];
183
+ /**
184
+ * Classify an AGENTS.md/CLAUDE.md by how much *real* guidance it carries, so the
185
+ * judge can tell “no context” from “has a file but it's the generated template.”
186
+ * Heuristic + pure.
187
+ *
188
+ * Deliberately **newline-independent**: it measures the volume of non-boilerplate
189
+ * prose (word count) plus heading count via a whitespace-tolerant regex, rather
190
+ * than splitting on lines. A line-anchored version would misclassify any excerpt
191
+ * whose newlines were collapsed to spaces upstream (a real bug class caught in a
192
+ * sibling implementation) — here even a fully single-lined file classifies the
193
+ * same as its multi-line original.
194
+ */
195
+ function classifyAgentsMd(content) {
196
+ if (!content || !content.trim())
197
+ return 'missing';
198
+ let s = content.replace(/\r\n/g, '\n').toLowerCase();
199
+ s = s.replace(/```[\s\S]*?```/g, ' '); // drop fenced code
200
+ s = s.replace(/<!--[\s\S]*?-->/g, ' '); // drop HTML comments
201
+ for (const m of BOILERPLATE_MARKERS)
202
+ s = s.split(m).join(' '); // drop generated boilerplate (markers are lowercase)
203
+ // Headings: a `#` run at start OR after any whitespace (so a collapsed,
204
+ // single-line excerpt still counts them), followed by a space.
205
+ const headings = (s.match(/(?:^|\s)#{1,6}\s/g) || []).length;
206
+ // Remaining non-boilerplate words (markdown punctuation stripped).
207
+ const words = s.replace(/[#|>*_`~-]/g, ' ').split(/\s+/).filter((w) => w.length > 1).length;
208
+ if (words < 40)
209
+ return 'boilerplate';
210
+ return headings >= 2 || words >= 60 ? 'substantive' : 'boilerplate';
153
211
  }
154
212
  // --------------------------------------------------------- corpus gathering
155
213
  /** Locate the most recent `.jsonl` transcript for a workspace under an agent. */
@@ -224,18 +282,25 @@ async function listAvailableSkills() {
224
282
  }
225
283
  return [...names].sort();
226
284
  }
285
+ /** Which context files exist at the workspace root, plus how substantive the
286
+ * primary one is (missing/boilerplate/substantive). One read per present file. */
227
287
  async function contextFilesFor(workspacePath) {
228
288
  const present = [];
289
+ let quality = 'missing';
229
290
  for (const name of (0, agent_config_1.getAllKnownContextFileNames)()) {
230
291
  try {
231
- await fs.access(path.join(workspacePath, name));
292
+ const content = await fs.readFile(path.join(workspacePath, name), 'utf-8');
232
293
  present.push(name);
294
+ // Classify the first present file, then keep the best classification seen.
295
+ const c = classifyAgentsMd(content);
296
+ if (quality === 'missing' || (quality === 'boilerplate' && c === 'substantive'))
297
+ quality = c;
233
298
  }
234
299
  catch {
235
- /* not present */
300
+ /* not present / unreadable */
236
301
  }
237
302
  }
238
- return present;
303
+ return { files: present, quality };
239
304
  }
240
305
  /**
241
306
  * Build the corpus the judge reasons over: the `limit` most **recently active**
@@ -244,23 +309,28 @@ async function contextFilesFor(workspacePath) {
244
309
  * session, plus the globally-available skills. `onProgress` (optional) fires
245
310
  * once per workspace as it finishes, for a live progress display.
246
311
  */
247
- async function gatherReflectionCorpus(limit, onProgress) {
312
+ async function gatherReflectionCorpus(limit, onProgress, opts = {}) {
248
313
  const [sessions, workspaces, availableSkills] = await Promise.all([
249
314
  (0, claude_sessions_1.getWorkspaceSessions)(), // already sorted by last-active, one per workspace
250
315
  (0, workspace_meta_1.listWorkspaces)(false),
251
316
  listAvailableSkills(),
252
317
  ]);
253
318
  const metaByName = new Map(workspaces.map((w) => [w.name, w]));
254
- const recent = sessions.slice(0, limit);
319
+ // A single named workspace (ignores limit), else the N most recently active.
320
+ const recent = opts.workspace
321
+ ? sessions.filter((s) => s.workspaceName === opts.workspace)
322
+ : sessions.slice(0, limit);
255
323
  const digests = [];
256
324
  for (let index = 0; index < recent.length; index++) {
257
325
  const s = recent[index];
258
326
  const meta = metaByName.get(s.workspaceName);
327
+ const context = await contextFilesFor(s.workspacePath);
259
328
  const digest = {
260
329
  name: s.workspaceName,
261
330
  repoCount: meta?.metadata?.repositories?.length ?? 0,
262
331
  repos: (meta?.metadata?.repositories ?? []).map((r) => r.name),
263
- contextFiles: await contextFilesFor(s.workspacePath),
332
+ contextFiles: context.files,
333
+ contextQuality: context.quality,
264
334
  session: await readDigestForSession(s),
265
335
  };
266
336
  digests.push(digest);
@@ -292,37 +362,8 @@ exports.REFLECT_SCHEMA = JSON.stringify({
292
362
  },
293
363
  required: ['summary', 'recommendations'],
294
364
  });
295
- /**
296
- * Build the LLM-as-a-judge prompt. The judge sees distilled recent sessions and
297
- * is asked to recommend concrete improvements to the user's SETUP (skills,
298
- * AGENTS.md/context rules, connectivity/tests, prompt habits, workflow) — not to
299
- * redo the tasks. Output is strict JSON matching REFLECT_SCHEMA.
300
- */
301
- function buildJudgePrompt(corpus) {
302
- const lines = [];
303
- lines.push('You are an expert reviewer ("LLM as a judge") analyzing an engineer\'s recent AI coding-agent sessions.', 'Goal: recommend concrete improvements to their SETUP so the agent works better next time —', 'which skills to add and WHERE, which AGENTS.md/context rules are missing, missing connectivity/', 'smoke tests, and prompt habits to change. Judge the setup, do NOT redo the tasks.', '', 'Base every recommendation on evidence in the sessions below (repeated failures, retries, vague', 'prompts, missing context). Prefer a few high-signal, actionable items over many generic ones.', 'When you suggest a skill or an AGENTS.md rule, include a short concrete `example` snippet.', '', `Globally installed skills (don't re-suggest these; suggest genuinely missing ones): ${corpus.availableSkills.join(', ') || '(none)'}`, '', `Recent workspaces (${corpus.workspaces.length}):`);
304
- for (const ws of corpus.workspaces) {
305
- lines.push(`\n## ${ws.name}`);
306
- lines.push(`repos: ${ws.repos.join(', ') || '(none)'} | context files: ${ws.contextFiles.join(', ') || 'NONE'}`);
307
- if (!ws.session) {
308
- lines.push('session: (no recent agent session found)');
309
- continue;
310
- }
311
- lines.push(`session: ${ws.session.turns} turns, tools used: ${ws.session.tools.join(', ') || '(none)'}`);
312
- if (ws.session.userPrompts.length) {
313
- lines.push('user prompts:');
314
- for (const p of ws.session.userPrompts)
315
- lines.push(` - ${p.replace(/\n/g, ' ')}`);
316
- }
317
- if (ws.session.errors.length) {
318
- lines.push('errors/failures observed:');
319
- for (const e of ws.session.errors)
320
- lines.push(` - ${e.replace(/\n/g, ' ')}`);
321
- }
322
- }
323
- lines.push('', 'Respond with ONLY a JSON object of this shape (no prose, no markdown fence):', '{"summary": string, "recommendations": [{"kind":"skill|context|test|prompt|connectivity|workflow|other",', '"title": string, "detail": string, "target": string(optional workspace/repo/path),', '"priority":"high|medium|low", "example": string(optional snippet)}]}');
324
- return lines.join('\n');
325
- }
365
+ // The judge prompt is now built from pre-computed FACTS (see reflect-analyze.ts
366
+ // `buildAnalysisPrompt`), not raw transcripts, so the LLM call stays small/fast.
326
367
  // ----------------------------------------------------------- response parse
327
368
  const KINDS = ['skill', 'context', 'test', 'prompt', 'connectivity', 'workflow', 'other'];
328
369
  const PRIORITIES = ['high', 'medium', 'low'];
@@ -351,3 +392,19 @@ function parseReflectionReport(parsed) {
351
392
  .filter((r) => r !== null);
352
393
  return { summary, recommendations };
353
394
  }
395
+ // -------------------------------------------------------------- report saving
396
+ /** Where saved reflection reports live: `~/.nemus/reflect/`. */
397
+ exports.REFLECT_REPORTS_DIR = path.join(config_1.CACHE_DIR, 'reflect');
398
+ /**
399
+ * Persist a reflection report as timestamped JSON under `~/.nemus/reflect/`, so
400
+ * a run can be revisited or diffed over time. Returns the written path. Pure
401
+ * side-effect (mkdir -p + write); callers treat failure as non-fatal.
402
+ */
403
+ async function saveReflectionReport(report, meta) {
404
+ await fs.mkdir(exports.REFLECT_REPORTS_DIR, { recursive: true });
405
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
406
+ const scope = meta.workspace ? `-${meta.workspace.replace(/[^a-zA-Z0-9_-]+/g, '_')}` : '';
407
+ const file = path.join(exports.REFLECT_REPORTS_DIR, `${stamp}${scope}.json`);
408
+ await fs.writeFile(file, JSON.stringify({ generatedAt: new Date().toISOString(), ...meta, ...report }, null, 2));
409
+ return file;
410
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nemus-cli/nemus",
3
- "version": "0.3.1",
3
+ "version": "0.3.3",
4
4
  "workspaces": [
5
5
  "packages/*"
6
6
  ],
@@ -140,6 +140,10 @@ function runExtraction(cmd: string, fullArgs: string[], fallbackArgs?: string[])
140
140
  const exec = (args: string[]) =>
141
141
  execFileSync(cmd, args, {
142
142
  encoding: 'utf-8',
143
+ // `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
144
+ // can't block this synchronous call forever in a headless/piped context
145
+ // — the same hang the async judge hit, guarded here for `nemus -- "…"`.
146
+ input: '',
143
147
  timeout: EXTRACTION_TIMEOUT_MS,
144
148
  maxBuffer: 10 * 1024 * 1024,
145
149
  });
@@ -4,13 +4,14 @@ import { outputJson, outputJsonError } from '../utils/output';
4
4
  import { colorize } from '../utils/colors';
5
5
  import {
6
6
  gatherReflectionCorpus,
7
- buildJudgePrompt,
8
7
  parseReflectionReport,
8
+ saveReflectionReport,
9
9
  REFLECT_SCHEMA,
10
10
  ReflectionReport,
11
11
  ReflectProgress,
12
12
  Recommendation,
13
13
  } from '../utils/reflect';
14
+ import { analyzeCorpus, buildAnalysisPrompt } from '../utils/reflect-analyze';
14
15
  import { runAgentJsonAsync } from '../utils/agent-judge';
15
16
 
16
17
  export function registerReflectCommand(parent: Command) {
@@ -19,28 +20,50 @@ export function registerReflectCommand(parent: Command) {
19
20
  .alias('retro')
20
21
  .description('Analyze your recent workspace sessions and suggest skill/prompt/context improvements (LLM-as-a-judge)')
21
22
  .option('-n, --limit <n>', 'How many recent workspaces to analyze', '10')
23
+ .option('-w, --workspace <name>', 'Analyze a single workspace by name (ignores --limit)')
24
+ .option('--model <model>', 'Judge model override (agent-native pattern/id)')
25
+ .option('--thinking <level>', 'Judge thinking level for pi: off|minimal|low|medium|high|xhigh|max')
22
26
  .option('--json', 'Output the report as JSON')
27
+ .option('--no-save', 'Do not save the report to ~/.nemus/reflect/')
23
28
  .option('--dry-run', 'Print the assembled corpus + judge prompt without calling the agent')
24
29
  .action(async (opts) => {
25
30
  await handleReflect(opts);
26
31
  });
27
32
  }
28
33
 
29
- async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: boolean }) {
34
+ async function handleReflect(opts: {
35
+ limit?: string;
36
+ workspace?: string;
37
+ model?: string;
38
+ thinking?: string;
39
+ json?: boolean;
40
+ save?: boolean; // commander sets `save: false` for --no-save
41
+ dryRun?: boolean;
42
+ }) {
30
43
  const limit = Math.max(1, Number.parseInt(opts.limit ?? '10', 10) || 10);
31
44
  try {
32
45
  const showProgress = !opts.json && !opts.dryRun;
33
46
  if (showProgress) {
34
- logStep(`Analyzing your ${colorize(String(limit), 'cyan')} most recent workspaces…`);
47
+ logStep(
48
+ opts.workspace
49
+ ? `Analyzing workspace ${colorize(opts.workspace, 'cyan')}…`
50
+ : `Analyzing your ${colorize(String(limit), 'cyan')} most recent workspaces…`,
51
+ );
35
52
  }
36
53
 
37
- const corpus = await gatherReflectionCorpus(limit, showProgress ? printProgress : undefined);
54
+ const corpus = await gatherReflectionCorpus(limit, showProgress ? printProgress : undefined, {
55
+ workspace: opts.workspace,
56
+ });
38
57
  const withSessions = corpus.workspaces.filter((w) => w.session).length;
39
- const prompt = buildJudgePrompt(corpus);
58
+ // A script does the heavy analysis (clustering failures, counting tools,
59
+ // spotting correction loops); the LLM only ever sees these compact facts,
60
+ // so the judge call stays small + fast regardless of workspace count.
61
+ const analysis = analyzeCorpus(corpus);
62
+ const prompt = buildAnalysisPrompt(analysis);
40
63
 
41
64
  if (opts.dryRun) {
42
- // No LLM call — surface exactly what the judge would see.
43
- if (opts.json) outputJson({ corpus, prompt });
65
+ // No LLM call — surface the computed facts + exactly what the judge sees.
66
+ if (opts.json) outputJson({ analysis, prompt });
44
67
  else {
45
68
  process.stdout.write(prompt + '\n');
46
69
  }
@@ -48,7 +71,9 @@ async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: bo
48
71
  }
49
72
 
50
73
  if (withSessions === 0) {
51
- const msg = 'No recent agent sessions found to analyze (need Claude/pi session transcripts).';
74
+ const msg = opts.workspace
75
+ ? `No recent agent session found for workspace "${opts.workspace}" (need a Claude/pi transcript).`
76
+ : 'No recent agent sessions found to analyze (need Claude/pi session transcripts).';
52
77
  if (opts.json) outputJsonError(msg);
53
78
  else logError(msg);
54
79
  process.exit(1);
@@ -58,22 +83,39 @@ async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: bo
58
83
  // (non-blocking) so a live spinner shows it's alive, not hung. Timeout is
59
84
  // overridable for slow local models.
60
85
  const timeoutMs = Number.parseInt(process.env.NEMUS_JUDGE_TIMEOUT_MS ?? '', 10) || undefined;
86
+ const model = opts.model ?? process.env.NEMUS_JUDGE_MODEL ?? undefined;
87
+ const thinking = opts.thinking ?? process.env.NEMUS_JUDGE_THINKING ?? undefined;
61
88
  const stopSpinner = opts.json
62
89
  ? () => {}
63
90
  : startSpinner(`Judging ${withSessions} session(s) with your configured agent (this can take a minute)…`);
64
91
  let parsed: unknown;
65
92
  try {
66
- parsed = await runAgentJsonAsync(prompt, { schema: REFLECT_SCHEMA, timeoutMs });
93
+ parsed = await runAgentJsonAsync(prompt, { schema: REFLECT_SCHEMA, timeoutMs, model, thinking });
67
94
  } finally {
68
95
  stopSpinner();
69
96
  }
70
97
  const report = parseReflectionReport(parsed);
71
98
 
99
+ // Persist the report (best-effort; never fails the run) unless --no-save.
100
+ let savedTo: string | undefined;
101
+ if (opts.save !== false) {
102
+ try {
103
+ savedTo = await saveReflectionReport(report, {
104
+ analyzed: withSessions,
105
+ workspaces: corpus.workspaces.length,
106
+ workspace: opts.workspace,
107
+ });
108
+ } catch {
109
+ /* saving is a convenience, not the point */
110
+ }
111
+ }
112
+
72
113
  if (opts.json) {
73
- outputJson({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report });
114
+ outputJson({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report, savedTo });
74
115
  return;
75
116
  }
76
117
  printReport(report, corpus.workspaces.length, withSessions);
118
+ if (savedTo) logInfo(`Saved report to ${colorize(savedTo, 'dim')}`);
77
119
  } catch (error) {
78
120
  const msg = error instanceof Error ? error.message : 'reflect failed';
79
121
  if (opts.json) outputJsonError(msg);
@@ -1,5 +1,5 @@
1
1
  import { describe, it, expect } from 'vitest';
2
- import { parseAgentJson, runAgentRaw, runAgentRawAsync, agentAttempts } from './agent-judge';
2
+ import { parseAgentJson, runAgentRaw, runAgentRawAsync, agentAttempts, spawnCollect } from './agent-judge';
3
3
 
4
4
  describe('parseAgentJson', () => {
5
5
  it('unwraps the common agent envelopes and shapes', () => {
@@ -16,9 +16,24 @@ describe('parseAgentJson', () => {
16
16
  });
17
17
  });
18
18
 
19
+ describe('spawnCollect (stdin-hang regression)', () => {
20
+ it('gives the child stdin EOF so a stdin-reading process does NOT hang', async () => {
21
+ // `cat` with no args reads stdin to EOF. If stdin were an open pipe (the old
22
+ // execFile default) this would block until the timeout and reject; with
23
+ // stdin ignored it gets immediate EOF and exits 0 fast. 2s timeout << any hang.
24
+ const out = await spawnCollect('cat', [], { timeout: 2000, maxBuffer: 1 << 20 });
25
+ expect(out).toBe('');
26
+ });
27
+
28
+ it('captures stdout and rejects on non-zero exit', async () => {
29
+ await expect(spawnCollect('node', ['-e', 'process.stdout.write("hi")'], { timeout: 5000, maxBuffer: 1 << 20 })).resolves.toBe('hi');
30
+ await expect(spawnCollect('node', ['-e', 'process.exit(3)'], { timeout: 5000, maxBuffer: 1 << 20 })).rejects.toThrow(/exit 3/);
31
+ });
32
+ });
33
+
19
34
  describe('agentAttempts', () => {
20
35
  it('claude: preferred (schema + lean flags) then a plain fallback', () => {
21
- const a = agentAttempts('claude', 'P', '{"type":"object"}');
36
+ const a = agentAttempts('claude', 'P', { schema: '{"type":"object"}' });
22
37
  expect(a[0]).toEqual({ cmd: 'claude', args: expect.arrayContaining(['-p', 'P', '--output-format', 'json', '--json-schema', '{"type":"object"}']) });
23
38
  expect(a[1]).toEqual({ cmd: 'claude', args: ['-p', 'P'] });
24
39
  });
@@ -28,6 +43,26 @@ describe('agentAttempts', () => {
28
43
  expect(pi[1]).toEqual({ cmd: 'pi', args: ['-p', 'P'] });
29
44
  expect(agentAttempts('opencode', 'P')).toEqual([{ cmd: 'opencode', args: ['run', 'P'] }]);
30
45
  });
46
+ it('threads --model (all) + --thinking (pi only); fallback keeps model, drops --thinking', () => {
47
+ const pi = agentAttempts('pi', 'P', { model: 'haiku', thinking: 'low' });
48
+ expect(pi[0].args).toEqual(expect.arrayContaining(['--model', 'haiku', '--thinking', 'low', '-p', 'P']));
49
+ // Safety net: model stays (stable flag), --thinking is dropped so an old pi
50
+ // that rejects --thinking still has a working fallback.
51
+ expect(pi[1].args).toEqual(['--model', 'haiku', '-p', 'P']);
52
+ expect(pi[1].args).not.toContain('--thinking');
53
+ const cl = agentAttempts('claude', 'P', { model: 'sonnet' });
54
+ expect(cl[0].args).toEqual(expect.arrayContaining(['--model', 'sonnet']));
55
+ expect(cl[0].args).not.toContain('--thinking'); // thinking is pi-only
56
+ expect(agentAttempts('opencode', 'P', { model: 'gpt' })[0].args).toEqual(['run', 'P', '--model', 'gpt']);
57
+ });
58
+ });
59
+
60
+ describe('runAgentRaw thinking default', () => {
61
+ it('applies the low-thinking default for pi', () => {
62
+ let seen: string[] = [];
63
+ runAgentRaw('P', { agentType: 'pi', exec: (cmd, args) => ((seen = [cmd, ...args]), 'ok') });
64
+ expect(seen).toEqual(expect.arrayContaining(['--thinking', 'low']));
65
+ });
31
66
  });
32
67
 
33
68
  describe('runAgentRawAsync', () => {
@@ -1,8 +1,51 @@
1
- import { execFile, execFileSync } from 'child_process';
2
- import { promisify } from 'util';
1
+ import { execFileSync, spawn } from 'child_process';
3
2
  import { getPrimaryAgent } from './agent-config';
4
3
 
5
- const execFileAsync = promisify(execFile);
4
+ /**
5
+ * Run a child to completion, capturing stdout, with stdin set to /dev/null.
6
+ *
7
+ * The stdin part is load-bearing: agents like `pi` block waiting on stdin if
8
+ * it's an open pipe (the default for execFile), which made the judge hang until
9
+ * the timeout regardless of prompt size or model speed. `stdio: ['ignore', …]`
10
+ * gives the child an immediate EOF, exactly like a non-interactive shell.
11
+ */
12
+ export function spawnCollect(
13
+ cmd: string,
14
+ args: string[],
15
+ opts: { timeout: number; maxBuffer: number },
16
+ ): Promise<string> {
17
+ return new Promise((resolve, reject) => {
18
+ const child = spawn(cmd, args, {
19
+ stdio: ['ignore', 'pipe', 'pipe'],
20
+ timeout: opts.timeout,
21
+ killSignal: 'SIGKILL',
22
+ });
23
+ // Decode as UTF-8 at the stream boundary so a multi-byte char split across
24
+ // two chunks isn't corrupted (which would break JSON.parse downstream).
25
+ child.stdout.setEncoding('utf8');
26
+ child.stderr.setEncoding('utf8');
27
+ let stdout = '';
28
+ let stderr = '';
29
+ let overflow = false;
30
+ child.stdout.on('data', (d) => {
31
+ stdout += d;
32
+ if (stdout.length > opts.maxBuffer) {
33
+ overflow = true;
34
+ child.kill('SIGKILL');
35
+ }
36
+ });
37
+ child.stderr.on('data', (d) => {
38
+ stderr += d;
39
+ });
40
+ child.on('error', (e) => reject(e));
41
+ child.on('close', (code, signal) => {
42
+ if (overflow) return reject(Object.assign(new Error('maxBuffer exceeded'), { stdout, stderr }));
43
+ if (signal) return reject(Object.assign(new Error(`killed by ${signal}`), { killed: true, signal, stdout, stderr }));
44
+ if (code !== 0) return reject(Object.assign(new Error(`exit ${code}`), { code, stdout, stderr }));
45
+ resolve(stdout);
46
+ });
47
+ });
48
+ }
6
49
 
7
50
  /**
8
51
  * Run the user's configured coding agent headlessly as an "LLM-as-a-judge":
@@ -17,6 +60,10 @@ const execFileAsync = promisify(execFile);
17
60
  export interface JudgeOptions {
18
61
  /** JSON schema string passed to `claude --json-schema` (ignored by others). */
19
62
  schema?: string;
63
+ /** Model override (`--model`), agent-native pattern/id. */
64
+ model?: string;
65
+ /** Thinking level (`--thinking`, pi only): off|minimal|low|medium|high|xhigh|max. */
66
+ thinking?: string;
20
67
  timeoutMs?: number;
21
68
  maxBuffer?: number;
22
69
  /** Injected for tests. Defaults to the real (blocking) child_process runner. */
@@ -34,24 +81,61 @@ export interface AgentAttempt {
34
81
  args: string[];
35
82
  }
36
83
 
84
+ export interface AttemptOptions {
85
+ schema?: string;
86
+ model?: string;
87
+ thinking?: string;
88
+ }
89
+
90
+ /**
91
+ * Default thinking level for the judge. The judge is a mechanical transform
92
+ * (facts → recommendations), not deep reasoning, so a heavy default like Opus
93
+ * @ medium thinking just makes it slow. `low` keeps pi fast; override per-run.
94
+ */
95
+ export const DEFAULT_JUDGE_THINKING = 'low';
96
+
37
97
  /**
38
98
  * The ordered invocation attempts for an agent (preferred → fallback), as pure
39
99
  * data so both the sync and async runners share ONE flag ladder (and it's
40
- * unit-testable without spawning anything).
100
+ * unit-testable without spawning anything). `--model` applies to all; pi also
101
+ * takes `--thinking` (the speed lever); both are ignored where unsupported.
41
102
  */
42
- export function agentAttempts(agentType: JudgeAgentType, prompt: string, schema?: string): AgentAttempt[] {
103
+ export function agentAttempts(agentType: JudgeAgentType, prompt: string, opts: AttemptOptions = {}): AgentAttempt[] {
104
+ const { schema, model, thinking } = opts;
43
105
  if (agentType === 'claude') {
44
106
  const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
107
+ if (model) preferred.push('--model', model);
45
108
  if (schema) preferred.push('--json-schema', schema);
46
109
  // Older claude may reject the newer flags — fall back to the plainest form.
47
- return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: ['-p', prompt] }];
110
+ const plain = ['-p', prompt];
111
+ if (model) plain.push('--model', model);
112
+ return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: plain }];
48
113
  }
49
114
  if (agentType === 'opencode') {
50
- return [{ cmd: 'opencode', args: ['run', prompt] }];
115
+ const args = ['run', prompt];
116
+ if (model) args.push('--model', model);
117
+ return [{ cmd: 'opencode', args }];
51
118
  }
52
119
  // pi (and any other): run as lean as possible so a bloated env can't hang it.
53
120
  const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
54
- return [{ cmd: 'pi', args: [...piLean, '-p', prompt] }, { cmd: 'pi', args: ['-p', prompt] }];
121
+ const modelArgs = model ? ['--model', model] : [];
122
+ const tune = thinking ? [...modelArgs, '--thinking', thinking] : modelArgs;
123
+ // Fallback keeps the stable --model but DROPS --thinking: --thinking is the
124
+ // newest flag and the most likely reason an older pi rejects the first
125
+ // attempt, so the safety net must not carry it (else both attempts fail).
126
+ return [
127
+ { cmd: 'pi', args: [...piLean, ...tune, '-p', prompt] },
128
+ { cmd: 'pi', args: [...modelArgs, '-p', prompt] },
129
+ ];
130
+ }
131
+
132
+ /** Resolve the attempt options for a run, applying the pi thinking default. */
133
+ function attemptOptions(agentType: JudgeAgentType, opts: JudgeOptions): AttemptOptions {
134
+ return {
135
+ schema: opts.schema,
136
+ model: opts.model,
137
+ thinking: opts.thinking ?? (agentType === 'pi' ? DEFAULT_JUDGE_THINKING : undefined),
138
+ };
55
139
  }
56
140
 
57
141
  function wrapJudgeError(err: any, agentType: string): Error {
@@ -80,9 +164,11 @@ export function runAgentRaw(prompt: string, opts: JudgeOptions = {}): string {
80
164
  const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
81
165
  const exec =
82
166
  opts.exec ??
83
- ((cmd, args, o) => execFileSync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer }));
167
+ // `input: ''` closes the child's stdin (EOF) so a stdin-reading agent (pi)
168
+ // can't hang the synchronous call — the sync twin of spawnCollect's fix.
169
+ ((cmd, args, o) => execFileSync(cmd, args, { encoding: 'utf-8', input: '', timeout: o.timeout, maxBuffer: o.maxBuffer }));
84
170
 
85
- const attempts = agentAttempts(agentType, prompt, opts.schema);
171
+ const attempts = agentAttempts(agentType, prompt, attemptOptions(agentType, opts));
86
172
  let lastErr: any;
87
173
  for (const a of attempts) {
88
174
  try {
@@ -103,12 +189,9 @@ export async function runAgentRawAsync(prompt: string, opts: JudgeOptions = {}):
103
189
  const agentType = opts.agentType ?? getPrimaryAgent().type;
104
190
  const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
105
191
  const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
106
- const exec =
107
- opts.execAsync ??
108
- (async (cmd, args, o) =>
109
- (await execFileAsync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer })).stdout.toString());
192
+ const exec = opts.execAsync ?? ((cmd, args, o) => spawnCollect(cmd, args, o));
110
193
 
111
- const attempts = agentAttempts(agentType, prompt, opts.schema);
194
+ const attempts = agentAttempts(agentType, prompt, attemptOptions(agentType, opts));
112
195
  let lastErr: any;
113
196
  for (const a of attempts) {
114
197
  try {