@soleri/core 9.7.2 → 9.8.0

package/src/runtime/quality-signals.test.ts

@@ -0,0 +1,312 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { analyzeQualitySignals, captureQualitySignals } from './quality-signals.js';
+import type { EvidenceReport } from '../planning/evidence-collector.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeReport(overrides: Partial<EvidenceReport> = {}): EvidenceReport {
+  return {
+    planId: 'plan-test',
+    planObjective: 'Test plan',
+    accuracy: 80,
+    evidenceSources: ['git'],
+    taskEvidence: [],
+    unplannedChanges: [],
+    missingWork: [],
+    verificationGaps: [],
+    summary: '',
+    ...overrides,
+  };
+}
+
+function makeVault() {
+  return {
+    search: vi.fn().mockReturnValue([]),
+    add: vi.fn(),
+  } as unknown as ReturnType<
+    (typeof import('../vault/vault.js'))['Vault']['prototype']['search']
+  > & { add: ReturnType<typeof vi.fn>; search: ReturnType<typeof vi.fn> };
+}
+
+function makeBrain() {
+  return {
+    recordFeedback: vi.fn(),
+  } as unknown as { recordFeedback: ReturnType<typeof vi.fn> };
+}
+
+// ---------------------------------------------------------------------------
+// analyzeQualitySignals
+// ---------------------------------------------------------------------------
+
+describe('analyzeQualitySignals', () => {
+  it('detects anti-pattern when fixIterations > 2', () => {
+    const report = makeReport({
+      taskEvidence: [
+        {
+          taskId: 't1',
+          taskTitle: 'Fix login bug',
+          plannedStatus: 'completed',
+          matchedFiles: [],
+          verdict: 'DONE',
+          fixIterations: 3,
+        },
+      ],
+    });
+
+    const result = analyzeQualitySignals(report);
+
+    expect(result.antiPatterns).toHaveLength(1);
+    expect(result.antiPatterns[0].taskId).toBe('t1');
+    expect(result.antiPatterns[0].kind).toBe('anti-pattern');
+    expect(result.antiPatterns[0].fixIterations).toBe(3);
+  });
+
+  it('detects clean task when fixIterations === 0 and verdict DONE', () => {
+    const report = makeReport({
+      taskEvidence: [
+        {
+          taskId: 't2',
+          taskTitle: 'Add feature',
+          plannedStatus: 'completed',
+          matchedFiles: [],
+          verdict: 'DONE',
+          fixIterations: 0,
+        },
+      ],
+    });
+
+    const result = analyzeQualitySignals(report);
+
+    expect(result.cleanTasks).toHaveLength(1);
+    expect(result.cleanTasks[0].taskId).toBe('t2');
+    expect(result.cleanTasks[0].kind).toBe('clean');
+  });
+
+  it('does not flag task with fixIterations === 0 but verdict PARTIAL', () => {
+    const report = makeReport({
+      taskEvidence: [
+        {
+          taskId: 't3',
+          taskTitle: 'Partial work',
+          plannedStatus: 'in_progress',
+          matchedFiles: [],
+          verdict: 'PARTIAL',
+          fixIterations: 0,
+        },
+      ],
+    });
+
+    const result = analyzeQualitySignals(report);
+
+    expect(result.cleanTasks).toHaveLength(0);
+    expect(result.antiPatterns).toHaveLength(0);
+  });
+
+  it('does not flag task with fixIterations === 2 (at threshold, not above)', () => {
+    const report = makeReport({
+      taskEvidence: [
+        {
+          taskId: 't4',
+          taskTitle: 'Borderline task',
+          plannedStatus: 'completed',
+          matchedFiles: [],
+          verdict: 'DONE',
+          fixIterations: 2,
+        },
+      ],
+    });
+
+    const result = analyzeQualitySignals(report);
+
+    expect(result.antiPatterns).toHaveLength(0);
+  });
+
+  it('detects scope creep from unplanned changes', () => {
+    const report = makeReport({
+      unplannedChanges: [
+        {
+          file: { path: 'src/extra.ts', status: 'added' },
+          possibleReason: 'unplanned scope',
+        },
+      ],
+    });
+
+    const result = analyzeQualitySignals(report);
+
+    expect(result.scopeCreep).toHaveLength(1);
+    expect(result.scopeCreep[0].kind).toBe('scope-creep');
+  });
+
+  it('handles undefined fixIterations as 0', () => {
+    const report = makeReport({
+      taskEvidence: [
+        {
+          taskId: 't5',
+          taskTitle: 'No iterations field',
+          plannedStatus: 'completed',
+          matchedFiles: [],
+          verdict: 'DONE',
+          // fixIterations omitted
+        },
+      ],
+    });
+
+    const result = analyzeQualitySignals(report);
+
+    expect(result.cleanTasks).toHaveLength(1);
+    expect(result.antiPatterns).toHaveLength(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// captureQualitySignals
+// ---------------------------------------------------------------------------
+
+describe('captureQualitySignals', () => {
+  let vault: ReturnType<typeof makeVault>;
+  let brain: ReturnType<typeof makeBrain>;
+
+  beforeEach(() => {
+    vault = makeVault();
+    brain = makeBrain();
+  });
+
+  it('captures anti-pattern to vault and records negative brain feedback', () => {
+    const analysis = {
+      antiPatterns: [
+        {
+          taskId: 't1',
+          taskTitle: 'Fix login',
+          kind: 'anti-pattern' as const,
+          fixIterations: 3,
+          verdict: 'DONE',
+        },
+      ],
+      cleanTasks: [],
+      scopeCreep: [],
+    };
+
+    const result = captureQualitySignals(analysis, vault, brain, 'plan-1');
+
+    expect(vault.add).toHaveBeenCalledTimes(1);
+    const entry = vault.add.mock.calls[0][0];
+    expect(entry.type).toBe('anti-pattern');
+    expect(entry.severity).toBe('warning');
+    expect(entry.tags).toContain('rework');
+    expect(entry.tags).toContain('fix-trail');
+    expect(entry.tags).toContain('auto-captured');
+
+    expect(brain.recordFeedback).toHaveBeenCalledWith(
+      'quality-signal:rework:Fix login',
+      't1',
+      'dismissed',
+    );
+
+    expect(result.captured).toBe(1);
+    expect(result.feedback).toBe(1);
+  });
+
+  it('records positive brain feedback for clean tasks', () => {
+    const analysis = {
+      antiPatterns: [],
+      cleanTasks: [
+        {
+          taskId: 't2',
+          taskTitle: 'Add feature',
+          kind: 'clean' as const,
+          fixIterations: 0,
+          verdict: 'DONE',
+        },
+      ],
+      scopeCreep: [],
+    };
+
+    const result = captureQualitySignals(analysis, vault, brain, 'plan-1');
+
+    expect(brain.recordFeedback).toHaveBeenCalledWith(
+      'quality-signal:clean:Add feature',
+      't2',
+      'accepted',
+    );
+    expect(result.feedback).toBe(1);
+    expect(result.captured).toBe(0);
+  });
+
+  it('skips duplicate anti-patterns when vault search returns high-score match', () => {
+    vault.search.mockReturnValue([{ entry: { id: 'existing' }, score: 0.8 }]);
+
+    const analysis = {
+      antiPatterns: [
+        {
+          taskId: 't1',
+          taskTitle: 'Fix login',
+          kind: 'anti-pattern' as const,
+          fixIterations: 3,
+          verdict: 'DONE',
+        },
+      ],
+      cleanTasks: [],
+      scopeCreep: [],
+    };
+
+    const result = captureQualitySignals(analysis, vault, brain, 'plan-1');
+
+    expect(vault.add).not.toHaveBeenCalled();
+    expect(result.skipped).toBe(1);
+    expect(result.captured).toBe(0);
+    // Brain feedback still recorded even for deduplicated captures
+    expect(brain.recordFeedback).toHaveBeenCalledTimes(1);
+  });
+
+  it('assigns critical severity for fixIterations > 4', () => {
+    const analysis = {
+      antiPatterns: [
+        {
+          taskId: 't1',
+          taskTitle: 'Hard bug',
+          kind: 'anti-pattern' as const,
+          fixIterations: 5,
+          verdict: 'DONE',
+        },
+      ],
+      cleanTasks: [],
+      scopeCreep: [],
+    };
+
+    captureQualitySignals(analysis, vault, brain, 'plan-1');
+
+    const entry = vault.add.mock.calls[0][0];
+    expect(entry.severity).toBe('critical');
+  });
+
+  it('handles mixed signals correctly', () => {
+    const analysis = {
+      antiPatterns: [
+        {
+          taskId: 't1',
+          taskTitle: 'Rework task',
+          kind: 'anti-pattern' as const,
+          fixIterations: 4,
+          verdict: 'DONE',
+        },
+      ],
+      cleanTasks: [
+        {
+          taskId: 't2',
+          taskTitle: 'Clean task',
+          kind: 'clean' as const,
+          fixIterations: 0,
+          verdict: 'DONE',
+        },
+      ],
+      scopeCreep: [],
+    };
+
+    const result = captureQualitySignals(analysis, vault, brain, 'plan-1');
+
+    expect(result.captured).toBe(1);
+    expect(result.feedback).toBe(2); // 1 dismissed + 1 accepted
+  });
+});

package/src/runtime/quality-signals.ts

@@ -0,0 +1,169 @@
+/**
+ * Quality Signals — analyze evidence reports for rework patterns and clean execution.
+ *
+ * Extracts anti-patterns (high rework), clean tasks (first-pass success),
+ * and scope creep signals from evidence reports. Captures findings to vault
+ * and feeds brain feedback. Best-effort — never throws.
+ */
+
+import type { EvidenceReport } from '../planning/evidence-collector.js';
+import type { Plan } from '../planning/planner.js';
+import type { Vault } from '../vault/vault.js';
+import type { Brain } from '../brain/brain.js';
+import type { IntelligenceEntry } from '../intelligence/types.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface QualitySignal {
+  taskId: string;
+  taskTitle: string;
+  kind: 'anti-pattern' | 'clean' | 'scope-creep';
+  fixIterations: number;
+  verdict: string;
+}
+
+export interface QualityAnalysis {
+  antiPatterns: QualitySignal[];
+  cleanTasks: QualitySignal[];
+  scopeCreep: QualitySignal[];
+}
+
+// ---------------------------------------------------------------------------
+// Thresholds
+// ---------------------------------------------------------------------------
+
+/** Tasks with more than this many fix iterations are flagged as anti-patterns. */
+const REWORK_THRESHOLD = 2;
+
+// ---------------------------------------------------------------------------
+// Analysis
+// ---------------------------------------------------------------------------
+
+/**
+ * Analyze an evidence report for quality signals.
+ *
+ * - fixIterations > 2 → anti-pattern (rework)
+ * - fixIterations === 0 + verdict DONE → clean (first-pass success)
+ * - unplannedChanges → scope-creep signals
+ */
+export function analyzeQualitySignals(
+  report: EvidenceReport,
+  _plan?: Plan | null,
+): QualityAnalysis {
+  const antiPatterns: QualitySignal[] = [];
+  const cleanTasks: QualitySignal[] = [];
+  const scopeCreep: QualitySignal[] = [];
+
+  for (const te of report.taskEvidence) {
+    const iterations = te.fixIterations ?? 0;
+
+    if (iterations > REWORK_THRESHOLD) {
+      antiPatterns.push({
+        taskId: te.taskId,
+        taskTitle: te.taskTitle,
+        kind: 'anti-pattern',
+        fixIterations: iterations,
+        verdict: te.verdict,
+      });
+    } else if (iterations === 0 && te.verdict === 'DONE') {
+      cleanTasks.push({
+        taskId: te.taskId,
+        taskTitle: te.taskTitle,
+        kind: 'clean',
+        fixIterations: 0,
+        verdict: te.verdict,
+      });
+    }
+  }
+
+  // Unplanned changes signal scope creep
+  for (const uc of report.unplannedChanges) {
+    scopeCreep.push({
+      taskId: 'unplanned',
+      taskTitle: uc.file.path,
+      kind: 'scope-creep',
+      fixIterations: 0,
+      verdict: uc.possibleReason,
+    });
+  }
+
+  return { antiPatterns, cleanTasks, scopeCreep };
+}
+
+// ---------------------------------------------------------------------------
+// Capture to vault + brain
+// ---------------------------------------------------------------------------
+
+/**
+ * Persist quality signals to vault (anti-patterns) and brain (feedback).
+ * Deduplicates anti-patterns via vault search before adding.
+ * Best-effort — swallows all errors.
+ */
+export function captureQualitySignals(
+  analysis: QualityAnalysis,
+  vault: Vault,
+  brain: Brain,
+  planId: string,
+): { captured: number; skipped: number; feedback: number } {
+  let captured = 0;
+  let skipped = 0;
+  let feedback = 0;
+
+  // Capture anti-patterns to vault (dedup first)
+  for (const ap of analysis.antiPatterns) {
+    try {
+      const query = `rework fix-trail ${ap.taskTitle}`;
+      const existing = vault.search(query, { type: 'anti-pattern', limit: 3 });
+      const isDuplicate = existing.some((r) => r.score > 0.7);
+
+      if (isDuplicate) {
+        skipped++;
+        continue;
+      }
+
+      const severity = ap.fixIterations > 4 ? 'critical' : 'warning';
+      const entry: IntelligenceEntry = {
+        id: `qs-ap-${planId}-${ap.taskId}-${Date.now()}`,
+        type: 'anti-pattern',
+        domain: 'engineering',
+        title: `Rework detected: ${ap.taskTitle}`,
+        severity,
+        description:
+          `Task "${ap.taskTitle}" required ${ap.fixIterations} fix iterations ` +
+          `(threshold: ${REWORK_THRESHOLD}). Investigate root cause — ` +
+          `unclear requirements, missing tests, or incomplete understanding.`,
+        tags: ['rework', 'fix-trail', 'auto-captured'],
+        origin: 'agent',
+      };
+
+      vault.add(entry);
+      captured++;
+    } catch {
+      // Best-effort — skip this signal
+    }
+  }
+
+  // Record negative brain feedback for rework tasks
+  for (const ap of analysis.antiPatterns) {
+    try {
+      brain.recordFeedback(`quality-signal:rework:${ap.taskTitle}`, ap.taskId, 'dismissed');
+      feedback++;
+    } catch {
+      // Best-effort
+    }
+  }
+
+  // Record positive brain feedback for clean tasks
+  for (const ct of analysis.cleanTasks) {
+    try {
+      brain.recordFeedback(`quality-signal:clean:${ct.taskTitle}`, ct.taskId, 'accepted');
+      feedback++;
+    } catch {
+      // Best-effort
+    }
+  }
+
+  return { captured, skipped, feedback };
+}

package/src/skills/trust-classifier.ts

@@ -92,7 +92,7 @@ function walkDir(rootDir: string, currentDir: string, inventory: SkillInventoryI
 
     if (!stat.isFile()) continue;
 
-    const relPath = relative(rootDir, fullPath);
+    const relPath = relative(rootDir, fullPath).replaceAll('\\', '/');
     const ext = extname(name).toLowerCase();
     const kind = classifyFile(name, ext);