@nemus-cli/nemus 0.2.13 → 0.3.1

package/src/utils/agent-judge.test.ts

@@ -0,0 +1,88 @@
+import { describe, it, expect } from 'vitest';
+import { parseAgentJson, runAgentRaw, runAgentRawAsync, agentAttempts } from './agent-judge';
+
+describe('parseAgentJson', () => {
+  it('unwraps the common agent envelopes and shapes', () => {
+    expect(parseAgentJson('{"a":1}')).toEqual({ a: 1 }); // bare object
+    expect(parseAgentJson(J({ structured_output: { a: 2 } }))).toEqual({ a: 2 }); // claude structured
+    expect(parseAgentJson(J({ result: J({ a: 3 }) }))).toEqual({ a: 3 }); // result-as-json-string
+    expect(parseAgentJson(J({ result: { a: 4 } }))).toEqual({ a: 4 }); // result-as-object
+    expect(parseAgentJson('```json\n{"a":5}\n```')).toEqual({ a: 5 }); // fenced
+    expect(parseAgentJson('noise before {"a":6} after')).toEqual({ a: 6 }); // outermost span fallback
+  });
+
+  it('throws when there is no JSON at all', () => {
+    expect(() => parseAgentJson('totally not json')).toThrow(/did not return JSON/);
+  });
+});
+
+describe('agentAttempts', () => {
+  it('claude: preferred (schema + lean flags) then a plain fallback', () => {
+    const a = agentAttempts('claude', 'P', '{"type":"object"}');
+    expect(a[0]).toEqual({ cmd: 'claude', args: expect.arrayContaining(['-p', 'P', '--output-format', 'json', '--json-schema', '{"type":"object"}']) });
+    expect(a[1]).toEqual({ cmd: 'claude', args: ['-p', 'P'] });
+  });
+  it('pi: lean then plain; opencode: single run', () => {
+    const pi = agentAttempts('pi', 'P');
+    expect(pi[0].args).toEqual(expect.arrayContaining(['--no-tools', '--no-skills', '-p', 'P']));
+    expect(pi[1]).toEqual({ cmd: 'pi', args: ['-p', 'P'] });
+    expect(agentAttempts('opencode', 'P')).toEqual([{ cmd: 'opencode', args: ['run', 'P'] }]);
+  });
+});
+
+describe('runAgentRawAsync', () => {
+  it('runs the preferred attempt and returns stdout', async () => {
+    const calls: string[][] = [];
+    const execAsync = async (cmd: string, args: string[]) => {
+      calls.push([cmd, ...args]);
+      return '{"ok":true}';
+    };
+    const out = await runAgentRawAsync('P', { agentType: 'pi', execAsync });
+    expect(out).toBe('{"ok":true}');
+    expect(calls).toHaveLength(1); // first attempt succeeded, no fallback
+  });
+
+  it('turns a timeout into an actionable error', async () => {
+    const execAsync = async () => {
+      throw Object.assign(new Error('spawn pi ETIMEDOUT'), { code: 'ETIMEDOUT', killed: true });
+    };
+    await expect(runAgentRawAsync('P', { agentType: 'pi', execAsync })).rejects.toThrow(/timed out.*NEMUS_JUDGE_TIMEOUT_MS/s);
+  });
+});
+
+describe('runAgentRaw', () => {
+  it('claude: passes the schema + lean flags, falls back on a rejected flag', () => {
+    const calls: string[][] = [];
+    const exec = (cmd: string, args: string[]) => {
+      calls.push([cmd, ...args]);
+      if (calls.length === 1) throw new Error('unknown flag --json-schema'); // old claude
+      return '{"ok":true}';
+    };
+    const out = runAgentRaw('PROMPT', { agentType: 'claude', schema: '{"type":"object"}', exec });
+    expect(out).toBe('{"ok":true}');
+    // first (preferred) attempt carries the schema + structured flags…
+    expect(calls[0]).toEqual(expect.arrayContaining(['claude', '-p', 'PROMPT', '--output-format', 'json', '--json-schema', '{"type":"object"}']));
+    // …the fallback is the plainest form
+    expect(calls[1]).toEqual(['claude', '-p', 'PROMPT']);
+  });
+
+  it('pi: runs with lean flags', () => {
+    let seen: string[] = [];
+    const exec = (cmd: string, args: string[]) => {
+      seen = [cmd, ...args];
+      return 'ok';
+    };
+    runAgentRaw('P', { agentType: 'pi', exec });
+    expect(seen).toEqual(expect.arrayContaining(['pi', '--no-extensions', '--no-skills', '--no-tools', '-p', 'P']));
+  });
+
+  it('wraps a hard failure in a clear error', () => {
+    const exec = () => { throw Object.assign(new Error('boom'), { stderr: 'agent exploded' }); };
+    // pi retries once (lean → plain), then throws
+    expect(() => runAgentRaw('P', { agentType: 'pi', exec })).toThrow(/agent judge failed \(pi\): agent exploded/);
+  });
+});
+
+function J(o: unknown) {
+  return JSON.stringify(o);
+}

package/src/utils/agent-judge.ts

@@ -0,0 +1,168 @@
+import { execFile, execFileSync } from 'child_process';
+import { promisify } from 'util';
+import { getPrimaryAgent } from './agent-config';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Run the user's configured coding agent headlessly as an "LLM-as-a-judge":
+ * feed it a prompt, get back a parsed JSON object. This reuses whatever agent
+ * the user already has authenticated (claude / pi / opencode) — no API keys of
+ * our own. It mirrors the lean, structured invocation used for intent
+ * extraction, but generalized (any prompt + optional JSON schema).
+ *
+ * The call is bounded by a timeout and a large maxBuffer; a transcript-analysis
+ * judge returns more than intent extraction, so the buffer is generous.
+ */
+export interface JudgeOptions {
+  /** JSON schema string passed to `claude --json-schema` (ignored by others). */
+  schema?: string;
+  timeoutMs?: number;
+  maxBuffer?: number;
+  /** Injected for tests. Defaults to the real (blocking) child_process runner. */
+  exec?: (cmd: string, args: string[], opts: { timeout: number; maxBuffer: number }) => string;
+  /** Injected for tests. Async runner used by the non-blocking variants. */
+  execAsync?: (cmd: string, args: string[], opts: { timeout: number; maxBuffer: number }) => Promise<string>;
+  /** Injected for tests. Defaults to the configured primary agent. */
+  agentType?: 'claude' | 'pi' | 'opencode' | 'codex' | 'gemini';
+}
+
+export type JudgeAgentType = NonNullable<JudgeOptions['agentType']>;
+
+export interface AgentAttempt {
+  cmd: string;
+  args: string[];
+}
+
+/**
+ * The ordered invocation attempts for an agent (preferred → fallback), as pure
+ * data so both the sync and async runners share ONE flag ladder (and it's
+ * unit-testable without spawning anything).
+ */
+export function agentAttempts(agentType: JudgeAgentType, prompt: string, schema?: string): AgentAttempt[] {
+  if (agentType === 'claude') {
+    const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
+    if (schema) preferred.push('--json-schema', schema);
+    // Older claude may reject the newer flags — fall back to the plainest form.
+    return [{ cmd: 'claude', args: preferred }, { cmd: 'claude', args: ['-p', prompt] }];
+  }
+  if (agentType === 'opencode') {
+    return [{ cmd: 'opencode', args: ['run', prompt] }];
+  }
+  // pi (and any other): run as lean as possible so a bloated env can't hang it.
+  const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
+  return [{ cmd: 'pi', args: [...piLean, '-p', prompt] }, { cmd: 'pi', args: ['-p', prompt] }];
+}
+
+function wrapJudgeError(err: any, agentType: string): Error {
+  // A timeout is the common failure (big prompt + slow local model), so make it
+  // actionable instead of surfacing a raw `spawn … ETIMEDOUT`.
+  if (err?.killed || err?.code === 'ETIMEDOUT' || err?.signal === 'SIGTERM' || /ETIMEDOUT/.test(String(err?.message ?? ''))) {
+    return new Error(
+      `agent judge (${agentType}) timed out. Try a smaller --limit, a faster agent, or raise the cap with NEMUS_JUDGE_TIMEOUT_MS.`,
+    );
+  }
+  const detail = (err?.stderr || err?.stdout || err?.message || 'unknown error').toString().trim().slice(0, 500);
+  return new Error(`agent judge failed (${agentType}): ${detail}`);
+}
+
+const DEFAULT_TIMEOUT_MS = 300_000; // judging N transcripts is heavier than extraction; a big prompt + slow model can run minutes
+const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024;
+
+/**
+ * Invoke the agent with `prompt` and return the raw stdout. Throws a clear error
+ * on timeout / non-zero exit. Kept separate from parsing so callers can inspect
+ * raw output (e.g. `--dry-run`, debugging).
+ */
+export function runAgentRaw(prompt: string, opts: JudgeOptions = {}): string {
+  const agentType = opts.agentType ?? getPrimaryAgent().type;
+  const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
+  const exec =
+    opts.exec ??
+    ((cmd, args, o) => execFileSync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer }));
+
+  const attempts = agentAttempts(agentType, prompt, opts.schema);
+  let lastErr: any;
+  for (const a of attempts) {
+    try {
+      return exec(a.cmd, a.args, { timeout, maxBuffer });
+    } catch (err) {
+      lastErr = err; // try the next (fallback) form
+    }
+  }
+  throw wrapJudgeError(lastErr, agentType);
+}
+
+/**
+ * Non-blocking twin of {@link runAgentRaw}. Uses `execFile` (async) so the
+ * caller's event loop stays free — letting a spinner/progress UI animate while
+ * the judge (which can take minutes) runs. Prefer this in interactive commands.
+ */
+export async function runAgentRawAsync(prompt: string, opts: JudgeOptions = {}): Promise<string> {
+  const agentType = opts.agentType ?? getPrimaryAgent().type;
+  const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
+  const exec =
+    opts.execAsync ??
+    (async (cmd, args, o) =>
+      (await execFileAsync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer })).stdout.toString());
+
+  const attempts = agentAttempts(agentType, prompt, opts.schema);
+  let lastErr: any;
+  for (const a of attempts) {
+    try {
+      return await exec(a.cmd, a.args, { timeout, maxBuffer });
+    } catch (err) {
+      lastErr = err; // try the next (fallback) form
+    }
+  }
+  throw wrapJudgeError(lastErr, agentType);
+}
+
+/**
+ * Run the agent and parse its reply as JSON, tolerating the shapes different
+ * agents emit: `{ structured_output }`, `{ result: "<json>" }`, a ```json fence,
+ * or a bare object. Returns `unknown`; callers validate/normalize their shape.
+ */
+export function runAgentJson(prompt: string, opts: JudgeOptions = {}): unknown {
+  const raw = runAgentRaw(prompt, opts);
+  return parseAgentJson(raw);
+}
+
+/** Non-blocking twin of {@link runAgentJson}. */
+export async function runAgentJsonAsync(prompt: string, opts: JudgeOptions = {}): Promise<unknown> {
+  const raw = await runAgentRawAsync(prompt, opts);
+  return parseAgentJson(raw);
+}
+
+/** Extract a JSON object from an agent's raw stdout. Exported for tests. */
+export function parseAgentJson(raw: string): unknown {
+  let text = raw.trim();
+  const fence = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+  if (fence) text = fence[1].trim();
+
+  let parsed: any;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    // Last resort: grab the outermost {...} span.
+    const span = text.match(/\{[\s\S]*\}/);
+    if (!span) throw new Error('agent did not return JSON');
+    parsed = JSON.parse(span[0]);
+  }
+
+  // Unwrap the common agent envelopes.
+  if (parsed && typeof parsed === 'object') {
+    if (parsed.structured_output && typeof parsed.structured_output === 'object') return parsed.structured_output;
+    if (typeof parsed.result === 'string') {
+      try {
+        return JSON.parse(parsed.result);
+      } catch {
+        /* fall through — result was plain text, return the envelope */
+      }
+    }
+    if (parsed.result && typeof parsed.result === 'object') return parsed.result;
+  }
+  return parsed;
+}

package/src/utils/reflect.test.ts

@@ -0,0 +1,103 @@
+import { describe, it, expect } from 'vitest';
+import { distillTranscript, buildJudgePrompt, parseReflectionReport, ReflectionCorpus } from './reflect';
+
+const J = (o: unknown) => JSON.stringify(o);
+
+describe('distillTranscript', () => {
+  it('extracts prompts, tools, errors, and turns across pi + claude shapes', () => {
+    const lines = [
+      'not json — skipped',
+      // pi: user prompt
+      J({ type: 'message', message: { role: 'user', content: [{ type: 'text', text: 'do the thing' }] } }),
+      // pi: assistant with a toolCall
+      J({ type: 'message', message: { role: 'assistant', content: [{ type: 'toolCall', name: 'bash' }] } }),
+      // pi: tool result flagged as error
+      J({ type: 'message', message: { role: 'toolResult', toolName: 'bash', isError: true, content: [{ type: 'text', text: 'command failed: boom' }] } }),
+      // claude: user prompt as plain string
+      J({ type: 'user', message: { role: 'user', content: 'claude style prompt' } }),
+      // claude: assistant tool_use
+      J({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'tool_use', name: 'Edit' }] } }),
+      // claude: tool_result-only user message → an error, NOT a prompt
+      J({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', is_error: true, content: 'file not found' }] } }),
+      // a system-reminder-style user text → ignored as a prompt
+      J({ type: 'message', message: { role: 'user', content: '<system-reminder>ignore me</system-reminder>' } }),
+      // UNFLAGGED benign result whose text merely mentions "error" → NOT a failure
+      J({ type: 'message', message: { role: 'toolResult', toolName: 'grep', content: [{ type: 'text', text: '0 results for error' }] } }),
+      // UNFLAGGED result with a strong failure signal → a failure (regex fallback)
+      J({ type: 'message', message: { role: 'toolResult', toolName: 'git', content: [{ type: 'text', text: 'fatal: not a git repository' }] } }),
+      // EXPLICITLY not-an-error, despite failing-looking text → trust the flag
+      J({ type: 'message', message: { role: 'toolResult', isError: false, content: [{ type: 'text', text: 'fatal: boom (but flagged success)' }] } }),
+    ].join('\n');
+
+    const d = distillTranscript(lines, { sessionId: 's1', agentType: 'pi' });
+    expect(d.turns).toBe(2);
+    expect(d.userPrompts).toEqual(['do the thing', 'claude style prompt']);
+    expect(d.tools.sort()).toEqual(['Edit', 'bash', 'git', 'grep']);
+    expect(d.errors).toContain('command failed: boom'); // explicit isError:true
+    expect(d.errors).toContain('file not found'); // explicit is_error:true
+    expect(d.errors).toContain('fatal: not a git repository'); // unflagged + strong signal
+    expect(d.errors).not.toContain('0 results for error'); // unflagged benign 'error' mention
+    expect(d.errors.some((e) => e.includes('flagged success'))).toBe(false); // isError:false trusted
+  });
+
+  it('is bounded and tolerant of empty input', () => {
+    expect(distillTranscript('', { sessionId: 's', agentType: 'pi' })).toMatchObject({
+      turns: 0,
+      userPrompts: [],
+      errors: [],
+      tools: [],
+    });
+  });
+});
+
+const corpus: ReflectionCorpus = {
+  generatedAt: '2026-01-01T00:00:00Z',
+  availableSkills: ['redash', 'datadog'],
+  workspaces: [
+    {
+      name: 'pay-app',
+      repoCount: 1,
+      repos: ['api'],
+      contextFiles: ['AGENTS.md'],
+      session: { sessionId: 's', agentType: 'pi', turns: 12, userPrompts: ['fix the sync bug'], errors: ['gh_pr_create: not a git repository'], tools: ['bash', 'edit'] },
+    },
+    { name: 'empty-ws', repoCount: 0, repos: [], contextFiles: [], session: null },
+  ],
+};
+
+describe('buildJudgePrompt', () => {
+  const p = buildJudgePrompt(corpus);
+  it('frames the judge task and includes the evidence + guardrails', () => {
+    expect(p).toContain('LLM as a judge');
+    expect(p).toContain('redash, datadog'); // available skills (do not re-suggest)
+    expect(p).toContain('## pay-app');
+    expect(p).toContain('fix the sync bug');
+    expect(p).toContain('gh_pr_create: not a git repository');
+    expect(p).toContain('no recent agent session found'); // empty-ws
+    expect(p).toContain('context files: NONE'); // empty-ws has none
+    expect(p).toMatch(/ONLY a JSON object/);
+  });
+});
+
+describe('parseReflectionReport', () => {
+  it('normalizes, clamps enums, and drops empty recs', () => {
+    const report = parseReflectionReport({
+      summary: 'ok',
+      recommendations: [
+        { kind: 'skill', title: 'Add X', detail: 'because Y', priority: 'high', target: ' api ', example: 'stub' },
+        { kind: 'bogus', title: 'clamp me', detail: 'd', priority: 'urgent' }, // kind→other, priority→medium
+        { title: '', detail: '' }, // dropped
+        'garbage', // dropped
+      ],
+    });
+    expect(report.summary).toBe('ok');
+    expect(report.recommendations).toHaveLength(2);
+    expect(report.recommendations[0]).toEqual({ kind: 'skill', title: 'Add X', detail: 'because Y', priority: 'high', target: 'api', example: 'stub' });
+    expect(report.recommendations[1]).toMatchObject({ kind: 'other', priority: 'medium', title: 'clamp me' });
+  });
+
+  it('tolerates a malformed top-level object', () => {
+    expect(parseReflectionReport(null)).toEqual({ summary: '', recommendations: [] });
+    expect(parseReflectionReport({ recommendations: 'nope' })).toEqual({ summary: '', recommendations: [] });
+  });
+});