@nemus-cli/nemus 0.2.13 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nemus-cli/nemus",
-  "version": "0.2.13",
+  "version": "0.3.0",
   "workspaces": [
     "packages/*"
   ],

package/src/cli/ai-prompt.ts

@@ -1,4 +1,5 @@
 import { spawn, execFile, execFileSync } from 'child_process';
+import { parseAgentJson } from '../utils/agent-judge';
 import { promisify } from 'util';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -259,32 +260,21 @@ export async function extractIntent(prompt: string): Promise<ExtractedIntent> {
     result = runExtraction('pi', [...piLean, ...piCore], piCore);
   }
 
-  // Strip markdown code fences if present (Pi may wrap JSON in ```json...```)
-  let jsonStr = result.trim();
-  const fenceMatch = jsonStr.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
-  if (fenceMatch) {
-    jsonStr = fenceMatch[1].trim();
-  }
-
-  const parsed = JSON.parse(jsonStr);
-
-  // Handle different output formats:
-  // - Claude: { structured_output: {...} } or { result: "..." }
-  // - Pi: may return the object directly or wrap it
+  // Unwrap the agent's reply (code fences + the structured_output / result-string
+  // / result-object / bare-object envelopes) with the shared parser, so this and
+  // the reflect judge can't drift when a new agent shape is learned. The
+  // extraction-specific INVOCATION (lean flags + tailored timeout/auth errors)
+  // deliberately stays here — those messages are part of the `nemus --` UX.
   let intent: ExtractedIntent | undefined;
-  if (parsed.structured_output) {
-    intent = parsed.structured_output;
-  } else if (typeof parsed.result === 'string' && parsed.result) {
-    try { intent = JSON.parse(parsed.result); } catch { /* ignore */ }
-  } else if (typeof parsed.result === 'object' && parsed.result !== null) {
-    intent = parsed.result;
-  } else if (parsed.workspaceName || parsed.repos || parsed.remainingIntent !== undefined) {
-    // Pi may return the extracted object directly
-    intent = parsed;
+  try {
+    const parsed = parseAgentJson(result) as any;
+    if (parsed && typeof parsed === 'object') intent = parsed;
+  } catch {
+    throw new Error(`Could not extract intent from agent response: ${result.slice(0, 200)}`);
   }
 
   if (!intent) {
-    throw new Error(`Could not extract intent from agent response. Parsed: ${JSON.stringify(parsed).slice(0, 200)}`);
+    throw new Error(`Could not extract intent from agent response: ${result.slice(0, 200)}`);
   }
 
   // Coerce/validate field types so malformed model output can't crash the

package/src/commands/reflect.ts

@@ -0,0 +1,120 @@
+import { Command } from 'commander';
+import { logError, logInfo, logStep } from '../utils/logger';
+import { outputJson, outputJsonError } from '../utils/output';
+import { colorize } from '../utils/colors';
+import {
+  gatherReflectionCorpus,
+  buildJudgePrompt,
+  parseReflectionReport,
+  REFLECT_SCHEMA,
+  ReflectionReport,
+  Recommendation,
+} from '../utils/reflect';
+import { runAgentJson } from '../utils/agent-judge';
+
+export function registerReflectCommand(parent: Command) {
+  parent
+    .command('reflect')
+    .alias('retro')
+    .description('Analyze your recent workspace sessions and suggest skill/prompt/context improvements (LLM-as-a-judge)')
+    .option('-n, --limit <n>', 'How many recent workspaces to analyze', '10')
+    .option('--json', 'Output the report as JSON')
+    .option('--dry-run', 'Print the assembled corpus + judge prompt without calling the agent')
+    .action(async (opts) => {
+      await handleReflect(opts);
+    });
+}
+
+async function handleReflect(opts: { limit?: string; json?: boolean; dryRun?: boolean }) {
+  const limit = Math.max(1, Number.parseInt(opts.limit ?? '10', 10) || 10);
+  try {
+    if (!opts.json && !opts.dryRun) {
+      logStep(`Analyzing your ${colorize(String(limit), 'cyan')} most recent workspaces…`);
+      logInfo('Reading sessions and distilling prompts + failures…');
+    }
+
+    const corpus = await gatherReflectionCorpus(limit);
+    const withSessions = corpus.workspaces.filter((w) => w.session).length;
+    const prompt = buildJudgePrompt(corpus);
+
+    if (opts.dryRun) {
+      // No LLM call — surface exactly what the judge would see.
+      if (opts.json) outputJson({ corpus, prompt });
+      else {
+        process.stdout.write(prompt + '\n');
+      }
+      return;
+    }
+
+    if (withSessions === 0) {
+      const msg = 'No recent agent sessions found to analyze (need Claude/pi session transcripts).';
+      if (opts.json) outputJsonError(msg);
+      else logError(msg);
+      process.exit(1);
+    }
+
+    if (!opts.json) logInfo(`Judging ${withSessions} session(s) with your configured agent…`);
+    const parsed = runAgentJson(prompt, { schema: REFLECT_SCHEMA });
+    const report = parseReflectionReport(parsed);
+
+    if (opts.json) {
+      outputJson({ analyzed: withSessions, workspaces: corpus.workspaces.length, ...report });
+      return;
+    }
+    printReport(report, corpus.workspaces.length, withSessions);
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : 'reflect failed';
+    if (opts.json) outputJsonError(msg);
+    else {
+      logError('Failed to analyze sessions');
+      logError(msg);
+    }
+    process.exit(1);
+  }
+}
+
+const KIND_LABEL: Record<Recommendation['kind'], string> = {
+  skill: 'Skill',
+  context: 'Context/AGENTS.md',
+  test: 'Test',
+  prompt: 'Prompt',
+  connectivity: 'Connectivity',
+  workflow: 'Workflow',
+  other: 'Other',
+};
+
+function priorityBadge(p: Recommendation['priority']): string {
+  if (p === 'high') return colorize('● high', 'red');
+  if (p === 'medium') return colorize('● med', 'yellow');
+  return colorize('● low', 'gray');
+}
+
+function printReport(report: ReflectionReport, workspaces: number, analyzed: number) {
+  console.log('');
+  console.log(colorize('  Reflection', 'bright') + colorize(`  (${analyzed} sessions across ${workspaces} workspaces)`, 'dim'));
+  console.log(colorize('  ' + '─'.repeat(56), 'dim'));
+  if (report.summary) {
+    console.log('\n  ' + report.summary.replace(/\n/g, '\n  '));
+  }
+
+  if (report.recommendations.length === 0) {
+    console.log('\n  ' + colorize('No specific recommendations — looks solid.', 'green') + '\n');
+    return;
+  }
+
+  // High priority first.
+  const order = { high: 0, medium: 1, low: 2 };
+  const recs = [...report.recommendations].sort((a, b) => order[a.priority] - order[b.priority]);
+
+  console.log('');
+  for (const r of recs) {
+    const target = r.target ? colorize(`  [${r.target}]`, 'cyan') : '';
+    console.log(`  ${priorityBadge(r.priority)}  ${colorize(KIND_LABEL[r.kind], 'bright')}  ${r.title}${target}`);
+    if (r.detail) console.log(`      ${r.detail.replace(/\n/g, '\n      ')}`);
+    if (r.example) {
+      console.log(colorize('      example:', 'dim'));
+      console.log(colorize(r.example.replace(/^/gm, '        '), 'dim'));
+    }
+    console.log('');
+  }
+}

package/src/program.ts

@@ -59,6 +59,7 @@ import { registerSaveContextCommand } from './commands/save-context';
 import { registerMigrateCommand } from './commands/migrate';
 import { registerReportBugCommand } from './commands/report-bug';
 import { registerCompletionCommand } from './commands/completion';
+import { registerReflectCommand } from './commands/reflect';
 
 registerCreateCommand(program);
 registerListCommand(program);
@@ -84,6 +85,7 @@ registerSaveContextCommand(program);
 registerMigrateCommand(program);
 registerReportBugCommand(program);
 registerCompletionCommand(program);
+registerReflectCommand(program);
 
 // Register TUI (delegates to existing Ink/React implementation)
 program

package/src/utils/agent-judge.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect } from 'vitest';
+import { parseAgentJson, runAgentRaw } from './agent-judge';
+
+describe('parseAgentJson', () => {
+  it('unwraps the common agent envelopes and shapes', () => {
+    expect(parseAgentJson('{"a":1}')).toEqual({ a: 1 }); // bare object
+    expect(parseAgentJson(J({ structured_output: { a: 2 } }))).toEqual({ a: 2 }); // claude structured
+    expect(parseAgentJson(J({ result: J({ a: 3 }) }))).toEqual({ a: 3 }); // result-as-json-string
+    expect(parseAgentJson(J({ result: { a: 4 } }))).toEqual({ a: 4 }); // result-as-object
+    expect(parseAgentJson('```json\n{"a":5}\n```')).toEqual({ a: 5 }); // fenced
+    expect(parseAgentJson('noise before {"a":6} after')).toEqual({ a: 6 }); // outermost span fallback
+  });
+
+  it('throws when there is no JSON at all', () => {
+    expect(() => parseAgentJson('totally not json')).toThrow(/did not return JSON/);
+  });
+});
+
+describe('runAgentRaw', () => {
+  it('claude: passes the schema + lean flags, falls back on a rejected flag', () => {
+    const calls: string[][] = [];
+    const exec = (cmd: string, args: string[]) => {
+      calls.push([cmd, ...args]);
+      if (calls.length === 1) throw new Error('unknown flag --json-schema'); // old claude
+      return '{"ok":true}';
+    };
+    const out = runAgentRaw('PROMPT', { agentType: 'claude', schema: '{"type":"object"}', exec });
+    expect(out).toBe('{"ok":true}');
+    // first (preferred) attempt carries the schema + structured flags…
+    expect(calls[0]).toEqual(expect.arrayContaining(['claude', '-p', 'PROMPT', '--output-format', 'json', '--json-schema', '{"type":"object"}']));
+    // …the fallback is the plainest form
+    expect(calls[1]).toEqual(['claude', '-p', 'PROMPT']);
+  });
+
+  it('pi: runs with lean flags', () => {
+    let seen: string[] = [];
+    const exec = (cmd: string, args: string[]) => {
+      seen = [cmd, ...args];
+      return 'ok';
+    };
+    runAgentRaw('P', { agentType: 'pi', exec });
+    expect(seen).toEqual(expect.arrayContaining(['pi', '--no-extensions', '--no-skills', '--no-tools', '-p', 'P']));
+  });
+
+  it('wraps a hard failure in a clear error', () => {
+    const exec = () => { throw Object.assign(new Error('boom'), { stderr: 'agent exploded' }); };
+    // pi retries once (lean → plain), then throws
+    expect(() => runAgentRaw('P', { agentType: 'pi', exec })).toThrow(/agent judge failed \(pi\): agent exploded/);
+  });
+});
+
+function J(o: unknown) {
+  return JSON.stringify(o);
+}

package/src/utils/agent-judge.ts

@@ -0,0 +1,109 @@
+import { execFileSync } from 'child_process';
+import { getPrimaryAgent } from './agent-config';
+
+/**
+ * Run the user's configured coding agent headlessly as an "LLM-as-a-judge":
+ * feed it a prompt, get back a parsed JSON object. This reuses whatever agent
+ * the user already has authenticated (claude / pi / opencode) — no API keys of
+ * our own. It mirrors the lean, structured invocation used for intent
+ * extraction, but generalized (any prompt + optional JSON schema).
+ *
+ * The call is bounded by a timeout and a large maxBuffer; a transcript-analysis
+ * judge returns more than intent extraction, so the buffer is generous.
+ */
+export interface JudgeOptions {
+  /** JSON schema string passed to `claude --json-schema` (ignored by others). */
+  schema?: string;
+  timeoutMs?: number;
+  maxBuffer?: number;
+  /** Injected for tests. Defaults to the real child_process runner. */
+  exec?: (cmd: string, args: string[], opts: { timeout: number; maxBuffer: number }) => string;
+  /** Injected for tests. Defaults to the configured primary agent. */
+  agentType?: 'claude' | 'pi' | 'opencode' | 'codex' | 'gemini';
+}
+
+const DEFAULT_TIMEOUT_MS = 180_000; // judging N transcripts is heavier than extraction
+const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024;
+
+/**
+ * Invoke the agent with `prompt` and return the raw stdout. Throws a clear error
+ * on timeout / non-zero exit. Kept separate from parsing so callers can inspect
+ * raw output (e.g. `--dry-run`, debugging).
+ */
+export function runAgentRaw(prompt: string, opts: JudgeOptions = {}): string {
+  const agentType = opts.agentType ?? getPrimaryAgent().type;
+  const timeout = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxBuffer = opts.maxBuffer ?? DEFAULT_MAX_BUFFER;
+  const exec =
+    opts.exec ??
+    ((cmd, args, o) => execFileSync(cmd, args, { encoding: 'utf-8', timeout: o.timeout, maxBuffer: o.maxBuffer }));
+
+  const attempt = (cmd: string, args: string[]) => exec(cmd, args, { timeout, maxBuffer });
+
+  try {
+    if (agentType === 'claude') {
+      const preferred = ['-p', prompt, '--output-format', 'json', '--bare', '--strict-mcp-config', '--disable-slash-commands'];
+      if (opts.schema) preferred.push('--json-schema', opts.schema);
+      try {
+        return attempt('claude', preferred);
+      } catch {
+        // Older claude may reject the newer flags — fall back to the plainest form.
+        return attempt('claude', ['-p', prompt]);
+      }
+    }
+    if (agentType === 'opencode') {
+      return attempt('opencode', ['run', prompt]);
+    }
+    // pi (and any other): run as lean as possible so a bloated env can't hang it.
+    const piLean = ['--no-extensions', '--no-skills', '--no-prompt-templates', '--no-context-files', '--no-tools', '--no-session'];
+    try {
+      return attempt('pi', [...piLean, '-p', prompt]);
+    } catch {
+      return attempt('pi', ['-p', prompt]);
+    }
+  } catch (err: any) {
+    const detail = (err?.stderr || err?.stdout || err?.message || 'unknown error').toString().trim().slice(0, 500);
+    throw new Error(`agent judge failed (${agentType}): ${detail}`);
+  }
+}
+
+/**
+ * Run the agent and parse its reply as JSON, tolerating the shapes different
+ * agents emit: `{ structured_output }`, `{ result: "<json>" }`, a ```json fence,
+ * or a bare object. Returns `unknown`; callers validate/normalize their shape.
+ */
+export function runAgentJson(prompt: string, opts: JudgeOptions = {}): unknown {
+  const raw = runAgentRaw(prompt, opts);
+  return parseAgentJson(raw);
+}
+
+/** Extract a JSON object from an agent's raw stdout. Exported for tests. */
+export function parseAgentJson(raw: string): unknown {
+  let text = raw.trim();
+  const fence = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+  if (fence) text = fence[1].trim();
+
+  let parsed: any;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    // Last resort: grab the outermost {...} span.
+    const span = text.match(/\{[\s\S]*\}/);
+    if (!span) throw new Error('agent did not return JSON');
+    parsed = JSON.parse(span[0]);
+  }
+
+  // Unwrap the common agent envelopes.
+  if (parsed && typeof parsed === 'object') {
+    if (parsed.structured_output && typeof parsed.structured_output === 'object') return parsed.structured_output;
+    if (typeof parsed.result === 'string') {
+      try {
+        return JSON.parse(parsed.result);
+      } catch {
+        /* fall through — result was plain text, return the envelope */
+      }
+    }
+    if (parsed.result && typeof parsed.result === 'object') return parsed.result;
+  }
+  return parsed;
+}

package/src/utils/reflect.test.ts

@@ -0,0 +1,103 @@
+import { describe, it, expect } from 'vitest';
+import { distillTranscript, buildJudgePrompt, parseReflectionReport, ReflectionCorpus } from './reflect';
+
+const J = (o: unknown) => JSON.stringify(o);
+
+describe('distillTranscript', () => {
+  it('extracts prompts, tools, errors, and turns across pi + claude shapes', () => {
+    const lines = [
+      'not json — skipped',
+      // pi: user prompt
+      J({ type: 'message', message: { role: 'user', content: [{ type: 'text', text: 'do the thing' }] } }),
+      // pi: assistant with a toolCall
+      J({ type: 'message', message: { role: 'assistant', content: [{ type: 'toolCall', name: 'bash' }] } }),
+      // pi: tool result flagged as error
+      J({ type: 'message', message: { role: 'toolResult', toolName: 'bash', isError: true, content: [{ type: 'text', text: 'command failed: boom' }] } }),
+      // claude: user prompt as plain string
+      J({ type: 'user', message: { role: 'user', content: 'claude style prompt' } }),
+      // claude: assistant tool_use
+      J({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'tool_use', name: 'Edit' }] } }),
+      // claude: tool_result-only user message → an error, NOT a prompt
+      J({ type: 'user', message: { role: 'user', content: [{ type: 'tool_result', is_error: true, content: 'file not found' }] } }),
+      // a system-reminder-style user text → ignored as a prompt
+      J({ type: 'message', message: { role: 'user', content: '<system-reminder>ignore me</system-reminder>' } }),
+      // UNFLAGGED benign result whose text merely mentions "error" → NOT a failure
+      J({ type: 'message', message: { role: 'toolResult', toolName: 'grep', content: [{ type: 'text', text: '0 results for error' }] } }),
+      // UNFLAGGED result with a strong failure signal → a failure (regex fallback)
+      J({ type: 'message', message: { role: 'toolResult', toolName: 'git', content: [{ type: 'text', text: 'fatal: not a git repository' }] } }),
+      // EXPLICITLY not-an-error, despite failing-looking text → trust the flag
+      J({ type: 'message', message: { role: 'toolResult', isError: false, content: [{ type: 'text', text: 'fatal: boom (but flagged success)' }] } }),
+    ].join('\n');
+
+    const d = distillTranscript(lines, { sessionId: 's1', agentType: 'pi' });
+    expect(d.turns).toBe(2);
+    expect(d.userPrompts).toEqual(['do the thing', 'claude style prompt']);
+    expect(d.tools.sort()).toEqual(['Edit', 'bash', 'git', 'grep']);
+    expect(d.errors).toContain('command failed: boom'); // explicit isError:true
+    expect(d.errors).toContain('file not found'); // explicit is_error:true
+    expect(d.errors).toContain('fatal: not a git repository'); // unflagged + strong signal
+    expect(d.errors).not.toContain('0 results for error'); // unflagged benign 'error' mention
+    expect(d.errors.some((e) => e.includes('flagged success'))).toBe(false); // isError:false trusted
+  });
+
+  it('is bounded and tolerant of empty input', () => {
+    expect(distillTranscript('', { sessionId: 's', agentType: 'pi' })).toMatchObject({
+      turns: 0,
+      userPrompts: [],
+      errors: [],
+      tools: [],
+    });
+  });
+});
+
+const corpus: ReflectionCorpus = {
+  generatedAt: '2026-01-01T00:00:00Z',
+  availableSkills: ['redash', 'datadog'],
+  workspaces: [
+    {
+      name: 'pay-app',
+      repoCount: 1,
+      repos: ['api'],
+      contextFiles: ['AGENTS.md'],
+      session: { sessionId: 's', agentType: 'pi', turns: 12, userPrompts: ['fix the sync bug'], errors: ['gh_pr_create: not a git repository'], tools: ['bash', 'edit'] },
+    },
+    { name: 'empty-ws', repoCount: 0, repos: [], contextFiles: [], session: null },
+  ],
+};
+
+describe('buildJudgePrompt', () => {
+  const p = buildJudgePrompt(corpus);
+  it('frames the judge task and includes the evidence + guardrails', () => {
+    expect(p).toContain('LLM as a judge');
+    expect(p).toContain('redash, datadog'); // available skills (do not re-suggest)
+    expect(p).toContain('## pay-app');
+    expect(p).toContain('fix the sync bug');
+    expect(p).toContain('gh_pr_create: not a git repository');
+    expect(p).toContain('no recent agent session found'); // empty-ws
+    expect(p).toContain('context files: NONE'); // empty-ws has none
+    expect(p).toMatch(/ONLY a JSON object/);
+  });
+});
+
+describe('parseReflectionReport', () => {
+  it('normalizes, clamps enums, and drops empty recs', () => {
+    const report = parseReflectionReport({
+      summary: 'ok',
+      recommendations: [
+        { kind: 'skill', title: 'Add X', detail: 'because Y', priority: 'high', target: ' api ', example: 'stub' },
+        { kind: 'bogus', title: 'clamp me', detail: 'd', priority: 'urgent' }, // kind→other, priority→medium
+        { title: '', detail: '' }, // dropped
+        'garbage', // dropped
+      ],
+    });
+    expect(report.summary).toBe('ok');
+    expect(report.recommendations).toHaveLength(2);
+    expect(report.recommendations[0]).toEqual({ kind: 'skill', title: 'Add X', detail: 'because Y', priority: 'high', target: 'api', example: 'stub' });
+    expect(report.recommendations[1]).toMatchObject({ kind: 'other', priority: 'medium', title: 'clamp me' });
+  });
+
+  it('tolerates a malformed top-level object', () => {
+    expect(parseReflectionReport(null)).toEqual({ summary: '', recommendations: [] });
+    expect(parseReflectionReport({ recommendations: 'nope' })).toEqual({ summary: '', recommendations: [] });
+  });
+});