@nemus-cli/nemus 0.3.1 → 0.3.3

package/src/utils/reflect-analyze.test.ts

@@ -0,0 +1,126 @@
+import { describe, it, expect } from 'vitest';
+import { normalizeErrorSignature, analyzeCorpus, buildAnalysisPrompt } from './reflect-analyze';
+import { ReflectionCorpus, isCorrectionPrompt } from './reflect';
+
+describe('normalizeErrorSignature', () => {
+  it('collapses paths, numbers, and hashes so variants cluster', () => {
+    const a = normalizeErrorSignature('fatal: not a git repository (or any parent up to /Users/alice/work/repo)');
+    const b = normalizeErrorSignature('fatal: not a git repository (or any parent up to /home/bob/src/thing)');
+    expect(a).toBe(b);
+    expect(a).toContain('<path>');
+
+    expect(normalizeErrorSignature('exit code 137 after 4200ms')).toBe(
+      normalizeErrorSignature('exit code 2 after 9ms'),
+    );
+    expect(normalizeErrorSignature('object abc1234def not found')).toBe(
+      normalizeErrorSignature('object 0f9e8d7c6b not found'),
+    );
+  });
+
+  it('is bounded and safe on empty input', () => {
+    expect(normalizeErrorSignature('')).toBe('');
+    expect(normalizeErrorSignature('x'.repeat(500)).length).toBeLessThanOrEqual(100);
+  });
+});
+
+describe('isCorrectionPrompt', () => {
+  it('flags corrections, not normal instructions', () => {
+    expect(isCorrectionPrompt('no, that is wrong — revert it')).toBe(true);
+    expect(isCorrectionPrompt('actually use the other repo instead')).toBe(true);
+    expect(isCorrectionPrompt('you forgot to run the tests')).toBe(true);
+    expect(isCorrectionPrompt('add a health check to the api')).toBe(false);
+    expect(isCorrectionPrompt('create a workspace for payments')).toBe(false);
+  });
+});
+
+const corpus: ReflectionCorpus = {
+  generatedAt: 'now',
+  availableSkills: ['redash'],
+  workspaces: [
+    {
+      name: 'pay-app',
+      repoCount: 1,
+      repos: ['api'],
+      contextFiles: ['AGENTS.md'],
+      contextQuality: 'substantive',
+      session: {
+        sessionId: 's1',
+        agentType: 'pi',
+        turns: 40,
+        userPrompts: ['fix the sync bug', 'no that is wrong, revert'],
+        reSteerSamples: ['no that is wrong, revert'],
+        errors: [
+          'fatal: not a git repository (at /Users/a/pay-app/api)',
+          'fatal: not a git repository (at /Users/a/pay-app/web)',
+          'gh: Not Found (HTTP 404)',
+        ],
+        tools: ['bash', 'edit'],
+      },
+    },
+    {
+      name: 'ledger',
+      repoCount: 0,
+      repos: [],
+      contextFiles: [], // missing context
+      contextQuality: 'missing',
+      session: {
+        sessionId: 's2',
+        agentType: 'pi',
+        turns: 12,
+        userPrompts: ['add tests'],
+        reSteerSamples: [],
+        errors: ['fatal: not a git repository (at /home/b/ledger)'],
+        tools: ['bash'],
+      },
+    },
+    { name: 'idle-ws', repoCount: 0, repos: [], contextFiles: ['CLAUDE.md'], contextQuality: 'boilerplate', session: null },
+  ],
+};
+
+describe('analyzeCorpus', () => {
+  const a = analyzeCorpus(corpus);
+
+  it('aggregates totals and correction signals', () => {
+    expect(a.totalWorkspaces).toBe(3);
+    expect(a.sessionsAnalyzed).toBe(2); // idle-ws has no session
+    expect(a.totalTurns).toBe(52);
+    expect(a.correctionSignals).toBe(1); // "no that is wrong, revert"
+  });
+
+  it('clusters the recurring failure across workspaces, most frequent first', () => {
+    const top = a.topErrors[0];
+    expect(top.signature).toContain('not a git repository');
+    expect(top.count).toBe(3); // 2 in pay-app + 1 in ledger
+    expect(top.workspaces.sort()).toEqual(['ledger', 'pay-app']);
+    expect(top.example).toContain('fatal: not a git repository');
+  });
+
+  it('reports tools, context quality, re-steers, and per-workspace facts', () => {
+    expect(a.topTools[0]).toEqual({ tool: 'bash', sessions: 2 });
+    expect(a.workspacesMissingContext).toEqual(['ledger']);
+    expect(a.workspacesBoilerplateContext).toEqual(['idle-ws']);
+    expect(a.reSteerSamples).toEqual(['no that is wrong, revert']);
+    const pay = a.workspaces.find((w) => w.name === 'pay-app')!;
+    expect(pay).toMatchObject({ turns: 40, failures: 3, contextQuality: 'substantive' });
+    expect(pay.topFailure).toContain('not a git repository');
+    expect(a.workspaces.find((w) => w.name === 'idle-ws')).toMatchObject({ turns: 0, failures: 0, contextQuality: 'boilerplate' });
+  });
+});
+
+describe('buildAnalysisPrompt', () => {
+  it('is compact and fact-based (no raw transcripts)', () => {
+    const p = buildAnalysisPrompt(analyzeCorpus(corpus));
+    expect(p).toContain('A script has already analyzed');
+    expect(p).toContain('Sessions analyzed: 2 across 3 workspaces (52 total turns).');
+    expect(p).toContain('Top recurring failures');
+    expect(p).toMatch(/\[3\u00d7 in 2 ws\]/); // the clustered failure
+    expect(p).toContain('NO context file (AGENTS.md/CLAUDE.md): ledger');
+    expect(p).toContain('boilerplate/template: idle-ws');
+    expect(p).toContain('Verbatim user corrections');
+    expect(p).toContain('no that is wrong, revert');
+    expect(p).toContain('bash(2)');
+    expect(p).toMatch(/ONLY a JSON object/);
+    // Compact: even this corpus stays well under a few KB.
+    expect(p.length).toBeLessThan(3000);
+  });
+});

package/src/utils/reflect-analyze.ts

@@ -0,0 +1,246 @@
+import { ReflectionCorpus, ContextQuality, isCorrectionPrompt } from './reflect';
+
+/**
+ * Deterministic analysis layer for `reflect`.
+ *
+ * The LLM used to read raw transcripts (every prompt + every error) for every
+ * workspace, which made the judge prompt large and slow (timeouts on local
+ * models). Instead, a *script* does the heavy lifting here — clustering repeated
+ * failures, counting tools, spotting correction/retry loops — and the LLM only
+ * ever sees a small, pre-computed set of FACTS. That keeps the judge call fast
+ * and bounded no matter how many workspaces are analyzed, and grounds every
+ * recommendation in real counts rather than a wall of text.
+ *
+ * Everything here is pure (corpus in → report out) so it's exhaustively
+ * unit-tested without spawning an agent.
+ */
+
+// ------------------------------------------------------------------ types
+
+export interface ErrorCluster {
+  /** Normalized signature (paths/numbers/hashes stripped) used to group. */
+  signature: string;
+  /** Total occurrences across all analyzed sessions. */
+  count: number;
+  /** Distinct workspaces the signature appeared in (recurrence = strong signal). */
+  workspaces: string[];
+  /** One representative raw example (bounded). */
+  example: string;
+}
+
+export interface ToolStat {
+  tool: string;
+  /** Number of sessions that used the tool (not raw invocation count — the
+   *  digest only records distinct tools per session). */
+  sessions: number;
+}
+
+export interface WorkspaceFact {
+  name: string;
+  turns: number;
+  failures: number;
+  /** Whether the primary context file is missing / boilerplate / substantive. */
+  contextQuality: ContextQuality;
+  /** The workspace's single most frequent failure signature, if any. */
+  topFailure?: string;
+}
+
+export interface AnalysisReport {
+  totalWorkspaces: number;
+  sessionsAnalyzed: number;
+  totalTurns: number;
+  /** Recurring failures, most frequent first. */
+  topErrors: ErrorCluster[];
+  /** Most-used tools, most sessions first. */
+  topTools: ToolStat[];
+  /** How many user prompts looked like corrections/retries (a friction signal). */
+  correctionSignals: number;
+  /** Verbatim user re-steer messages across sessions (the sharpest coaching
+   *  signal), most-recent-workspace first, bounded. */
+  reSteerSamples: string[];
+  /** Workspaces with NO context file at all. */
+  workspacesMissingContext: string[];
+  /** Workspaces whose context file exists but is just boilerplate/template. */
+  workspacesBoilerplateContext: string[];
+  /** Skills already installed (so the judge suggests genuine gaps). */
+  availableSkills: string[];
+  /** One compact line of facts per workspace, for grounding. */
+  workspaces: WorkspaceFact[];
+}
+
+// --------------------------------------------------------------- normalizing
+
+/**
+ * Reduce a raw error string to a stable signature so near-identical failures
+ * cluster together: lowercase, drop quotes, replace filesystem paths, hashes,
+ * hex ids and bare numbers with placeholders, collapse whitespace, and cap the
+ * length. `"fatal: not a git repository (or any of the parent up to /Users/x)"`
+ * and the same from another path collapse to one signature.
+ */
+export function normalizeErrorSignature(raw: string): string {
+  let s = (raw ?? '').toLowerCase();
+  s = s.replace(/[`'"]/g, ' ');
+  // Windows paths first (contain ':' and '\'), then unix paths.
+  s = s.replace(/[a-z]:\\[^\s]+/g, '<path>');
+  s = s.replace(/\/[^\s:)'"]+/g, '<path>');
+  // Long hex / uuids / sha-like tokens before bare numbers.
+  s = s.replace(/\b[0-9a-f]{7,}\b/g, '<hash>');
+  // No trailing \b, so a number glued to a unit ('4200ms', '9ms') still clusters.
+  s = s.replace(/\b\d[\d.,:]*/g, '<n>');
+  s = s.replace(/\s+/g, ' ').trim();
+  return s.slice(0, 100);
+}
+
+// ------------------------------------------------------------------ analyze
+
+export interface AnalyzeOptions {
+  /** Max error clusters to surface (default 8). */
+  topErrors?: number;
+  /** Max tools to surface (default 10). */
+  topTools?: number;
+  /** Max chars of a raw error kept as the example (default 160). */
+  exampleChars?: number;
+  /** Max verbatim re-steer samples to surface to the judge (default 8). */
+  maxReSteer?: number;
+}
+
+/**
+ * Turn a distilled corpus into aggregated facts. This is the "script processes
+ * the data" step — the LLM never sees the raw corpus, only the returned report.
+ */
+export function analyzeCorpus(corpus: ReflectionCorpus, opts: AnalyzeOptions = {}): AnalysisReport {
+  const topErrorsK = opts.topErrors ?? 8;
+  const topToolsK = opts.topTools ?? 10;
+  const exampleChars = opts.exampleChars ?? 160;
+  const maxReSteer = opts.maxReSteer ?? 8;
+
+  const errorMap = new Map<string, { count: number; ws: Set<string>; example: string }>();
+  const toolMap = new Map<string, number>();
+  const workspacesMissingContext: string[] = [];
+  const workspacesBoilerplateContext: string[] = [];
+  const reSteerSamples: string[] = [];
+  const workspaces: WorkspaceFact[] = [];
+  let sessionsAnalyzed = 0;
+  let totalTurns = 0;
+  let correctionSignals = 0;
+
+  for (const ws of corpus.workspaces) {
+    if (ws.contextQuality === 'missing') workspacesMissingContext.push(ws.name);
+    else if (ws.contextQuality === 'boilerplate') workspacesBoilerplateContext.push(ws.name);
+
+    const s = ws.session;
+    if (!s) {
+      workspaces.push({ name: ws.name, turns: 0, failures: 0, contextQuality: ws.contextQuality });
+      continue;
+    }
+
+    sessionsAnalyzed++;
+    totalTurns += s.turns;
+    for (const t of s.tools) toolMap.set(t, (toolMap.get(t) ?? 0) + 1);
+    correctionSignals += s.userPrompts.filter(isCorrectionPrompt).length;
+    for (const q of s.reSteerSamples) {
+      if (reSteerSamples.length < maxReSteer) reSteerSamples.push(q);
+    }
+
+    // Per-workspace signature tally (drives the global map + this ws's topFailure).
+    const localSig = new Map<string, number>();
+    for (const e of s.errors) {
+      const sig = normalizeErrorSignature(e);
+      if (!sig) continue;
+      localSig.set(sig, (localSig.get(sig) ?? 0) + 1);
+      let entry = errorMap.get(sig);
+      if (!entry) {
+        entry = { count: 0, ws: new Set(), example: e.slice(0, exampleChars) };
+        errorMap.set(sig, entry);
+      }
+      entry.count++;
+      entry.ws.add(ws.name);
+    }
+    const topFailure = [...localSig.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
+
+    workspaces.push({ name: ws.name, turns: s.turns, failures: s.errors.length, contextQuality: ws.contextQuality, topFailure });
+  }
+
+  const topErrors: ErrorCluster[] = [...errorMap.entries()]
+    .map(([signature, v]) => ({ signature, count: v.count, workspaces: [...v.ws], example: v.example }))
+    // Most frequent first; break ties by cross-workspace spread (recurrence).
+    .sort((a, b) => b.count - a.count || b.workspaces.length - a.workspaces.length)
+    .slice(0, topErrorsK);
+
+  const topTools: ToolStat[] = [...toolMap.entries()]
+    .map(([tool, sessions]) => ({ tool, sessions }))
+    .sort((a, b) => b.sessions - a.sessions || a.tool.localeCompare(b.tool))
+    .slice(0, topToolsK);
+
+  return {
+    totalWorkspaces: corpus.workspaces.length,
+    sessionsAnalyzed,
+    totalTurns,
+    topErrors,
+    topTools,
+    correctionSignals,
+    reSteerSamples,
+    workspacesMissingContext,
+    workspacesBoilerplateContext,
+    availableSkills: corpus.availableSkills,
+    workspaces,
+  };
+}
+
+// --------------------------------------------------------------- judge prompt
+
+/**
+ * Build the judge prompt from the pre-computed facts. Compact by construction
+ * (a handful of clusters + counts), so the LLM call stays small and fast no
+ * matter how many workspaces were analyzed. Output contract is unchanged
+ * (REFLECT_SCHEMA), so parsing/rendering are shared with the old path.
+ */
+export function buildAnalysisPrompt(a: AnalysisReport): string {
+  const L: string[] = [];
+  L.push(
+    'You are an expert reviewer ("LLM as a judge"). A script has already analyzed an engineer\'s',
+    'recent AI coding-agent sessions and distilled them into the FACTS below. Do not ask for the',
+    'raw transcripts — reason only from these facts.',
+    '',
+    'Goal: recommend concrete improvements to their SETUP so the agent works better next time —',
+    'which skills to add and WHERE, which AGENTS.md/context rules are missing, missing connectivity/',
+    'smoke tests, and prompt/workflow habits. Ground every recommendation in a fact below',
+    '(a recurring failure, a correction loop, a missing context file). Prefer a few high-signal',
+    'items over many generic ones. Include a concrete `example` snippet for skills/context rules.',
+    '',
+    `Sessions analyzed: ${a.sessionsAnalyzed} across ${a.totalWorkspaces} workspaces (${a.totalTurns} total turns).`,
+    `Installed skills (don't re-suggest; find genuine gaps): ${a.availableSkills.join(', ') || '(none)'}`,
+    `User correction/retry signals: ${a.correctionSignals} prompt(s) looked like corrections.`,
+    `Workspaces with NO context file (AGENTS.md/CLAUDE.md): ${a.workspacesMissingContext.join(', ') || '(none)'}`,
+    `Workspaces whose context file is just boilerplate/template: ${a.workspacesBoilerplateContext.join(', ') || '(none)'}`,
+  );
+
+  if (a.reSteerSamples.length) {
+    L.push('', 'Verbatim user corrections/re-steers (the sharpest signal — quote/act on these):');
+    for (const q of a.reSteerSamples) L.push(`  - “${q.replace(/\n/g, ' ')}”`);
+  }
+
+  L.push('', 'Top recurring failures (count × workspaces — example):');
+  if (a.topErrors.length === 0) L.push('  (none captured)');
+  for (const e of a.topErrors) {
+    L.push(`  - [${e.count}× in ${e.workspaces.length} ws] ${e.signature}`);
+    L.push(`      e.g. ${e.example.replace(/\n/g, ' ')}`);
+  }
+
+  L.push('', `Most-used tools: ${a.topTools.map((t) => `${t.tool}(${t.sessions})`).join(', ') || '(none)'}`);
+
+  L.push('', 'Per-workspace:');
+  for (const w of a.workspaces) {
+    const top = w.topFailure ? ` · top failure: ${w.topFailure}` : '';
+    L.push(`  - ${w.name}: ${w.turns} turns, ${w.failures} failures, context:${w.contextQuality}${top}`);
+  }
+
+  L.push(
+    '',
+    'Respond with ONLY a JSON object of this shape (no prose, no markdown fence):',
+    '{"summary": string, "recommendations": [{"kind":"skill|context|test|prompt|connectivity|workflow|other",',
+    '"title": string, "detail": string, "target": string(optional workspace/repo/path),',
+    '"priority":"high|medium|low", "example": string(optional snippet)}]}',
+  );
+  return L.join('\n');
+}

package/src/utils/reflect.test.ts

@@ -1,5 +1,7 @@
 import { describe, it, expect } from 'vitest';
-import { distillTranscript, buildJudgePrompt, parseReflectionReport, ReflectionCorpus } from './reflect';
+import { distillTranscript, parseReflectionReport, classifyAgentsMd, isCorrectionPrompt, saveReflectionReport } from './reflect';
+import * as fs from 'fs/promises';
+import * as path from 'path';
 
 const J = (o: unknown) => JSON.stringify(o);
 
@@ -15,6 +17,8 @@ describe('distillTranscript', () => {
       J({ type: 'message', message: { role: 'toolResult', toolName: 'bash', isError: true, content: [{ type: 'text', text: 'command failed: boom' }] } }),
       // claude: user prompt as plain string
       J({ type: 'user', message: { role: 'user', content: 'claude style prompt' } }),
+      // a correction/re-steer → captured verbatim in reSteerSamples
+      J({ type: 'user', message: { role: 'user', content: 'no, that is wrong — revert that change' } }),
       // claude: assistant tool_use
       J({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'tool_use', name: 'Edit' }] } }),
       // claude: tool_result-only user message → an error, NOT a prompt
@@ -31,13 +35,14 @@ describe('distillTranscript', () => {
 
     const d = distillTranscript(lines, { sessionId: 's1', agentType: 'pi' });
     expect(d.turns).toBe(2);
-    expect(d.userPrompts).toEqual(['do the thing', 'claude style prompt']);
+    expect(d.userPrompts).toEqual(['do the thing', 'claude style prompt', 'no, that is wrong — revert that change']);
     expect(d.tools.sort()).toEqual(['Edit', 'bash', 'git', 'grep']);
     expect(d.errors).toContain('command failed: boom'); // explicit isError:true
     expect(d.errors).toContain('file not found'); // explicit is_error:true
     expect(d.errors).toContain('fatal: not a git repository'); // unflagged + strong signal
     expect(d.errors).not.toContain('0 results for error'); // unflagged benign 'error' mention
     expect(d.errors.some((e) => e.includes('flagged success'))).toBe(false); // isError:false trusted
+    expect(d.reSteerSamples).toEqual(['no, that is wrong — revert that change']); // captured verbatim
   });
 
   it('is bounded and tolerant of empty input', () => {
@@ -50,32 +55,50 @@ describe('distillTranscript', () => {
   });
 });
 
-const corpus: ReflectionCorpus = {
-  generatedAt: '2026-01-01T00:00:00Z',
-  availableSkills: ['redash', 'datadog'],
-  workspaces: [
-    {
-      name: 'pay-app',
-      repoCount: 1,
-      repos: ['api'],
-      contextFiles: ['AGENTS.md'],
-      session: { sessionId: 's', agentType: 'pi', turns: 12, userPrompts: ['fix the sync bug'], errors: ['gh_pr_create: not a git repository'], tools: ['bash', 'edit'] },
-    },
-    { name: 'empty-ws', repoCount: 0, repos: [], contextFiles: [], session: null },
-  ],
-};
+describe('saveReflectionReport', () => {
+  it('writes a timestamped JSON report (sanitized scope) and returns its path', async () => {
+    const file = await saveReflectionReport(
+      { summary: 'ok', recommendations: [{ kind: 'skill', title: 't', detail: 'd', priority: 'high' }] },
+      { analyzed: 2, workspaces: 3, workspace: 'pay/app' },
+    );
+    try {
+      expect(file).toMatch(/\.json$/);
+      expect(path.basename(file)).toContain('pay_app'); // scope suffix, path-sanitized
+      const written = JSON.parse(await fs.readFile(file, 'utf-8'));
+      expect(written).toMatchObject({ analyzed: 2, workspaces: 3, workspace: 'pay/app', summary: 'ok' });
+      expect(written.generatedAt).toBeTruthy();
+    } finally {
+      await fs.rm(file, { force: true });
+    }
+  });
+});
+
+describe('classifyAgentsMd', () => {
+  it('distinguishes missing / boilerplate / substantive', () => {
+    expect(classifyAgentsMd('')).toBe('missing');
+    expect(classifyAgentsMd(null)).toBe('missing');
+    // Generated template: headings + markers, little real guidance.
+    expect(classifyAgentsMd('# Workspace\n\nThis workspace was created with Workspace Manager.\n<!-- ws-rules:v2 -->\n## Notes\n- \n')).toBe('boilerplate');
+    // Real, rule-heavy content.
+    const real = Array.from({ length: 12 }, (_, i) => `- Always run the ${i} integration suite before opening a pull request here`).join('\n');
+    expect(classifyAgentsMd(`# Rules\n${real}`)).toBe('substantive');
+  });
+
+  it('is newline-independent: a whitespace-collapsed substantive file still classifies substantive', () => {
+    // Regression for the collapse bug: same content, newlines squashed to spaces.
+    const real = Array.from({ length: 12 }, (_, i) => `- Always run the ${i} integration suite before opening a pull request here`).join('\n');
+    const multiline = `# Rules\n${real}`;
+    const collapsed = multiline.replace(/\s+/g, ' ');
+    expect(classifyAgentsMd(collapsed)).toBe(classifyAgentsMd(multiline));
+    expect(classifyAgentsMd(collapsed)).toBe('substantive');
+  });
+});
 
-describe('buildJudgePrompt', () => {
-  const p = buildJudgePrompt(corpus);
-  it('frames the judge task and includes the evidence + guardrails', () => {
-    expect(p).toContain('LLM as a judge');
-    expect(p).toContain('redash, datadog'); // available skills (do not re-suggest)
-    expect(p).toContain('## pay-app');
-    expect(p).toContain('fix the sync bug');
-    expect(p).toContain('gh_pr_create: not a git repository');
-    expect(p).toContain('no recent agent session found'); // empty-ws
-    expect(p).toContain('context files: NONE'); // empty-ws has none
-    expect(p).toMatch(/ONLY a JSON object/);
+describe('isCorrectionPrompt', () => {
+  it('flags corrections, not normal instructions', () => {
+    expect(isCorrectionPrompt('no, revert that')).toBe(true);
+    expect(isCorrectionPrompt('actually use the other repo instead')).toBe(true);
+    expect(isCorrectionPrompt('add a health check to the api')).toBe(false);
   });
 });