@aprimediet/codewalker 1.3.0 → 1.4.0

package/src/analyze/debt.ts

@@ -0,0 +1,180 @@
+/**
+ * Technical debt scanner for codewalker v1.4.
+ *
+ * PURE module — no I/O. Scans source file content for:
+ * - TODO/FIXME/HACK/XXX markers
+ * - @ts-ignore / @ts-nocheck usage
+ * - Oversized files (>400 lines)
+ * - Long functions (>120 lines, derived from existing symbol rows)
+ */
+
+/** Default threshold for oversized file warning (lines). */
+export const LARGE_FILE_LINES = 400;
+
+/** Default threshold for long function warning (lines). */
+export const LONG_FN_LINES = 120;
+
+/** A single debt finding. */
+export interface DebtFinding {
+  title: string;
+  file_path: string;
+  line_start: number;
+  line_end: number;
+  marker: string;
+  severity: "info" | "warn" | "high";
+  metric: string;
+  body: string;
+}
+
+/** A symbol row passed for function-length analysis. */
+interface SymbolSpan {
+  name: string;
+  kind: string;
+  file_path: string;
+  line_start: number;
+  line_end: number;
+}
+
+// Word-boundary regex for each marker type.
+const MARKER_RE = /\b(TODO|FIXME|HACK|XXX)\b(?:\s*[:-]?\s*(.*))?/g;
+const TS_IGNORE_RE = /\/\/\s*@ts-ignore\b/g;
+const TS_NOCHECK_RE = /\/\/\s*@ts-nocheck\b/g;
+
+/**
+ * Scan a file's content for debt markers and heuristics.
+ *
+ * @param filePath - Absolute or relative file path (for the finding).
+ * @param content - The full file content string.
+ * @param symbols - Existing symbol rows for this file (for function-length analysis).
+ * @returns An array of debt findings.
+ */
+export function scanDebt(
+  filePath: string,
+  content: string,
+  symbols: SymbolSpan[],
+): DebtFinding[] {
+  const findings: DebtFinding[] = [];
+  const lines = content ? content.split("\n") : [];
+
+  // --- Scan markers in content ---
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i] ?? "";
+
+    // TODO/FIXME/HACK/XXX
+    MARKER_RE.lastIndex = 0;
+    let m: RegExpExecArray | null;
+    while ((m = MARKER_RE.exec(line)) !== null) {
+      const marker = m[1]!;
+      const detail = (m[2] ?? "").trim();
+      const severity = marker === "FIXME" || marker === "HACK" ? "warn" : "info";
+      findings.push({
+        title: `${marker}: ${detail || line.trim().slice(0, 50)}`,
+        file_path: filePath,
+        line_start: i + 1,
+        line_end: i + 1,
+        marker,
+        severity,
+        metric: marker,
+        body: line.trim(),
+      });
+    }
+
+    // @ts-ignore
+    TS_IGNORE_RE.lastIndex = 0;
+    if (TS_IGNORE_RE.test(line)) {
+      findings.push({
+        title: `@ts-ignore at line ${i + 1}`,
+        file_path: filePath,
+        line_start: i + 1,
+        line_end: i + 1,
+        marker: "@ts-ignore",
+        severity: "warn",
+        metric: "@ts-ignore",
+        body: line.trim(),
+      });
+    }
+
+    // @ts-nocheck
+    TS_NOCHECK_RE.lastIndex = 0;
+    if (TS_NOCHECK_RE.test(line)) {
+      findings.push({
+        title: `@ts-nocheck in ${filePath}`,
+        file_path: filePath,
+        line_start: i + 1,
+        line_end: i + 1,
+        marker: "@ts-nocheck",
+        severity: "high",
+        metric: "@ts-nocheck",
+        body: line.trim(),
+      });
+    }
+  }
+
+  // --- Oversized file heuristic ---
+  if (lines.length > LARGE_FILE_LINES) {
+    findings.push({
+      title: `Oversized file: ${lines.length} lines`,
+      file_path: filePath,
+      line_start: 0,
+      line_end: lines.length,
+      marker: "oversized-file",
+      severity: "warn",
+      metric: `${lines.length} lines`,
+      body: `File has ${lines.length} lines, exceeding the ${LARGE_FILE_LINES}-line threshold. Consider splitting into smaller modules.`,
+    });
+  }
+
+  // --- Long function heuristic (from existing symbols) ---
+  for (const sym of symbols) {
+    if (sym.file_path !== filePath) continue;
+    const fnLength = sym.line_end - sym.line_start;
+    if (fnLength > LONG_FN_LINES) {
+      findings.push({
+        title: `Long function: ${sym.name} (${fnLength} lines)`,
+        file_path: filePath,
+        line_start: sym.line_start,
+        line_end: sym.line_end,
+        marker: "long-function",
+        severity: "warn",
+        metric: `fn length ${fnLength}`,
+        body: `Function "${sym.name}" spans ${fnLength} lines (limit: ${LONG_FN_LINES}). Consider refactoring.`,
+      });
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * Group and summarize debt findings for a file.
+ * Returns at most one finding per marker type with aggregated counts.
+ */
+export function summarizeDebt(findings: DebtFinding[]): DebtFinding[] {
+  if (findings.length === 0) return [];
+
+  const groups = new Map<string, DebtFinding[]>();
+  for (const f of findings) {
+    const key = f.marker;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key)!.push(f);
+  }
+
+  const result: DebtFinding[] = [];
+  for (const [, group] of groups) {
+    const first = group[0]!;
+    if (group.length === 1) {
+      result.push(first);
+    } else {
+      result.push({
+        ...first,
+        title: `${first.marker}: ${group.length} occurrences`,
+        metric: `${first.marker} x${group.length}`,
+        body: group.map(f => `  line ${f.line_start}: ${f.body}`).join("\n"),
+        line_start: group[0]!.line_start,
+        line_end: group[group.length - 1]!.line_end,
+      });
+    }
+  }
+
+  return result;
+}

package/src/analyze/review.test.ts

@@ -0,0 +1,127 @@
+import { describe, it, expect } from 'vitest';
+import {
+  validateReviewPath,
+  checkReviewCap,
+  selectFilesForReview,
+  formatReviewWorklist,
+  DEFAULT_REVIEW_CAP,
+} from './review.ts';
+
+describe('validateReviewPath', () => {
+  it('accepts a non-empty path', () => {
+    const result = validateReviewPath('src/auth');
+    expect(result.valid).toBe(true);
+    expect(result.error).toBeUndefined();
+  });
+
+  it('rejects empty string', () => {
+    const result = validateReviewPath('');
+    expect(result.valid).toBe(false);
+    expect(result.error).toContain('Specify a path');
+  });
+
+  it('rejects whitespace-only', () => {
+    const result = validateReviewPath('   ');
+    expect(result.valid).toBe(false);
+    expect(result.error).toContain('Specify a path');
+  });
+
+  it('rejects undefined/null', () => {
+    const result = validateReviewPath(undefined as any);
+    expect(result.valid).toBe(false);
+    expect(result.error).toContain('Specify a path');
+  });
+});
+
+describe('checkReviewCap', () => {
+  it('returns ok when count within cap', () => {
+    const result = checkReviewCap(5, 25);
+    expect(result.ok).toBe(true);
+    expect(result.count).toBe(5);
+    expect(result.skipped).toBe(0);
+    expect(result.error).toBeUndefined();
+  });
+
+  it('rejects when count exceeds cap', () => {
+    const result = checkReviewCap(100, 25);
+    expect(result.ok).toBe(false);
+    expect(result.count).toBe(100);
+    expect(result.skipped).toBe(75);
+    expect(result.error).toContain('Narrow your path');
+  });
+
+  it('uses default cap when not specified', () => {
+    const result = checkReviewCap(50);
+    expect(result.ok).toBe(false);
+    expect(result.error).toContain('--max');
+  });
+
+  it('accepts edge case at exactly cap', () => {
+    const result = checkReviewCap(25, 25);
+    expect(result.ok).toBe(true);
+    expect(result.count).toBe(25);
+  });
+});
+
+describe('selectFilesForReview', () => {
+  const files = [
+    'src/auth/token.ts',
+    'src/auth/session.ts',
+    'src/api/route.ts',
+    'src/api/handler.ts',
+    'src/db/query.ts',
+  ];
+
+  it('selects files under a path prefix', () => {
+    const result = selectFilesForReview(files, 'src/auth', 25);
+    expect(result).toHaveLength(2);
+    expect(result).toContain('src/auth/token.ts');
+    expect(result).toContain('src/auth/session.ts');
+  });
+
+  it('respects the cap', () => {
+    const result = selectFilesForReview(files, 'src/', 3);
+    expect(result).toHaveLength(3);
+  });
+
+  it('returns empty array for non-matching prefix', () => {
+    const result = selectFilesForReview(files, 'src/nonexistent', 25);
+    expect(result).toEqual([]);
+  });
+
+  it('returns all matching files when under cap', () => {
+    const result = selectFilesForReview(files, 'src/api', 25);
+    expect(result).toHaveLength(2);
+  });
+
+  it('returns exact cap when match count equals cap', () => {
+    const result = selectFilesForReview(files, '', 5);
+    expect(result).toHaveLength(5);
+  });
+});
+
+describe('formatReviewWorklist', () => {
+  it('formats a worklist with file list and instructions', () => {
+    const files = ['src/auth/token.ts', 'src/auth/session.ts'];
+    const result = formatReviewWorklist(files, 'src/auth');
+    expect(result).toContain('2 file(s) under');
+    expect(result).toContain('src/auth');
+    expect(result).toContain('src/auth/token.ts');
+    expect(result).toContain('src/auth/session.ts');
+    expect(result).toContain('codewalker_finding');
+    expect(result).toContain('conventions');
+    expect(result).toContain('decisions');
+  });
+
+  it('handles empty file list', () => {
+    const result = formatReviewWorklist([], 'src/auth');
+    expect(result).toContain('No files found');
+  });
+
+  it('includes the max files info when files hit the cap', () => {
+    const files = Array.from({ length: 25 }, (_, i) => `src/a${i}.ts`);
+    const result = formatReviewWorklist(files, 'src/');
+    expect(result).toContain('25');
+    expect(result).toContain('selected for review');
+  });
+});

package/src/analyze/review.ts

@@ -0,0 +1,127 @@
+/**
+ * Review helpers for codewalker v1.4.
+ *
+ * Pure/lightweight helpers for the agent-driven best-practice review workflow:
+ * - `validateReviewPath()` — ensures path is provided
+ * - `checkReviewCap()` — enforces the review cap guardrail
+ * - `selectFilesForReview()` — selects files under a path prefix with cap
+ * - `formatReviewWorklist()` — formats worklist items for agent display
+ *
+ * Mirrors the shape of enrich.ts.
+ */
+
+/** Default maximum files per review command. */
+export const DEFAULT_REVIEW_CAP = 25;
+
+/** Result of validateReviewPath. */
+export interface PathValidation {
+  valid: boolean;
+  error?: string;
+}
+
+/** Result of checkReviewCap. */
+export interface CapCheck {
+  ok: boolean;
+  count: number;
+  /** Number of files over the cap (only when !ok). */
+  skipped: number;
+  error?: string;
+}
+
+/**
+ * Validate that a review path was provided.
+ */
+export function validateReviewPath(path: string | undefined | null): PathValidation {
+  const trimmed = (path ?? "").trim();
+  if (!trimmed) {
+    return {
+      valid: false,
+      error: 'Specify a path, e.g. src/auth. Bare /codewalker review with no path is not allowed.',
+    };
+  }
+  return { valid: true };
+}
+
+/**
+ * Check whether a count of files exceeds the cap.
+ * If ok is false, the caller should refuse and tell the user to narrow the path.
+ *
+ * @param count - Number of files found.
+ * @param cap - Maximum allowed (default: DEFAULT_REVIEW_CAP = 25).
+ */
+export function checkReviewCap(count: number, cap: number = DEFAULT_REVIEW_CAP): CapCheck {
+  if (count > cap) {
+    const skipped = count - cap;
+    return {
+      ok: false,
+      count,
+      skipped,
+      error: `Refusing to review ${count} files (cap ${cap}). ` +
+        `Narrow your path or raise the cap with --max=${count}.`,
+    };
+  }
+  return { ok: true, count, skipped: 0 };
+}
+
+/**
+ * Select files under a path prefix, respecting the cap.
+ * Files not matching the prefix are excluded.
+ *
+ * @param files - Full list of source file paths.
+ * @param pathPrefix - The path prefix to filter by (empty string = all files).
+ * @param cap - Maximum number to select.
+ * @returns Array of file paths matching the prefix, up to cap.
+ */
+export function selectFilesForReview(
+  files: string[],
+  pathPrefix: string,
+  cap: number,
+): string[] {
+  const matching = pathPrefix
+    ? files.filter(f => f.startsWith(pathPrefix))
+    : files;
+  return matching.slice(0, cap);
+}
+
+/**
+ * Format a review worklist for the agent to process.
+ * Each line shows one file to review. Includes instructions about grounding
+ * findings in the project's own conventions and decisions.
+ */
+export function formatReviewWorklist(
+  files: string[],
+  pathPrefix: string,
+): string {
+  if (files.length === 0) {
+    return `No files found under "${pathPrefix}" to review.`;
+  }
+
+  const lines: string[] = [
+    `Found ${files.length} file(s) under "${pathPrefix}" selected for review:`,
+    "",
+  ];
+
+  for (const file of files) {
+    lines.push(`  ${file}`);
+  }
+
+  lines.push(
+    "",
+    "Instructions:",
+    `  For each file above, read its content and judge it against the project's`,
+    `  conventions and decisions. Before starting, query for relevant context:`,
+    `    codewalker_query source:'all' query:'conventions'`,
+    `    codewalker_query source:'all' query:'decisions'`,
+    "",
+    `  For each issue found, call \`codewalker_finding\` with kind='practice',`,
+    `  an appropriate severity, and a body grounded in a specific convention or`,
+    `  decision. Avoid generic style nits — only flag what the project's own`,
+    `  conventions would flag.`,
+    "",
+    `  Example: \`codewalker_finding { kind: "practice", title: "...",`,
+    `    file: "src/auth/token.ts", severity: "warn",`,
+    `    body: "Convention X says Y, but this code does Z." }\``,
+  );
+
+  return lines.join("\n");
+}

package/src/db.test.ts

@@ -3,7 +3,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import * as os from 'node:os';
 import Database from 'better-sqlite3';
-import { openDb, bootstrapDb, upsertSymbol, searchSymbols, getMeta, setMeta, deleteFileSymbols, upsertLibrary, upsertLibSymbol, deleteLibrary, searchLibSymbols, upsertNote, searchNotes, deleteNote, updateSymbolSummary, selectUnenrichedSymbols } from './db.ts';
+import { openDb, bootstrapDb, upsertSymbol, searchSymbols, getMeta, setMeta, deleteFileSymbols, upsertLibrary, upsertLibSymbol, deleteLibrary, searchLibSymbols, upsertNote, searchNotes, deleteNote, updateSymbolSummary, selectUnenrichedSymbols, upsertFinding, deleteFindingsForFile, searchFindings } from './db.ts';
 
 describe('db.ts', () => {
   let tmpDir: string;
@@ -47,11 +47,11 @@ describe('db.ts', () => {
       // No error = idempotent
     });
 
-    it('sets user_version to 3 (v1.3 schema)', () => {
+    it('sets user_version to 4 (v1.4 schema)', () => {
       const db = new Database(dbPath);
       bootstrapDb(db);
       const version = db.pragma('user_version', { simple: true }) as number;
-      expect(version).toBe(3);
+      expect(version).toBe(4);
       db.close();
     });
 
@@ -89,6 +89,19 @@ describe('db.ts', () => {
 
       const ftsTables = db.prepare("SELECT name FROM sqlite_master WHERE name='notes_fts'").all() as { name: string }[];
       expect(ftsTables.length).toBe(1);
+
+      // Analysis tables exist (v1.4 additive upgrade)
+      expect(tables.map(t => t.name)).toContain('analysis');
+      const analysisFts = db.prepare("SELECT name FROM sqlite_master WHERE name='analysis_fts'").all() as { name: string }[];
+      expect(analysisFts.length).toBe(1);
+
+      // Analysis triggers exist
+      const triggers = db.prepare("SELECT name FROM sqlite_master WHERE type='trigger'").all() as { name: string }[];
+      const triggerNames = triggers.map(t => t.name);
+      expect(triggerNames).toContain('analysis_ai');
+      expect(triggerNames).toContain('analysis_ad');
+      expect(triggerNames).toContain('analysis_au');
+
       db.close();
     });
   });
@@ -575,6 +588,213 @@ describe('db.ts', () => {
     });
   });
 
+  describe('analysis CRUD + FTS via triggers', () => {
+    it('creates analysis + analysis_fts tables on bootstrap', () => {
+      const db = openDb(dbPath);
+      const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[];
+      expect(tables.map(t => t.name)).toContain('analysis');
+
+      const ftsTables = db.prepare("SELECT name FROM sqlite_master WHERE name='analysis_fts'").all() as { name: string }[];
+      expect(ftsTables.length).toBe(1);
+
+      const triggers = db.prepare("SELECT name FROM sqlite_master WHERE type='trigger'").all() as { name: string }[];
+      const triggerNames = triggers.map(t => t.name);
+      expect(triggerNames).toContain('analysis_ai');
+      expect(triggerNames).toContain('analysis_ad');
+      expect(triggerNames).toContain('analysis_au');
+      db.close();
+    });
+
+    it('upsertFinding inserts a finding and FTS MATCH finds it', () => {
+      const db = openDb(dbPath);
+
+      const id = upsertFinding(db, {
+        finding_kind: 'coverage',
+        title: 'Low coverage: src/auth/token.ts',
+        severity: 'warn',
+        file_path: 'src/auth/token.ts',
+        line_start: 0,
+        line_end: 0,
+        metric: '38% (24/63 lines)',
+        body: 'Auth token refresh path is under-tested — 38% line coverage.',
+        related: 'refreshToken, token.ts:42-71',
+        card_path: '/entries/analysis/coverage/low-coverage-src-auth-token.md',
+      });
+
+      expect(typeof id).toBe('number');
+      expect(id).toBeGreaterThan(0);
+
+      // FTS search
+      const results = searchFindings(db, 'coverage');
+      expect(results).toHaveLength(1);
+      expect(results[0]!.name).toBe('Low coverage: src/auth/token.ts');
+      expect(results[0]!.finding_kind).toBe('coverage');
+      expect(results[0]!.severity).toBe('warn');
+      expect(results[0]!.source).toBe('analysis');
+
+      db.close();
+    });
+
+    it('upsertFinding upserts on (finding_kind, file_path, title) — update refreshes FTS via analysis_au', () => {
+      const db = openDb(dbPath);
+
+      upsertFinding(db, {
+        finding_kind: 'coverage',
+        title: 'Low coverage: token.ts',
+        severity: 'warn',
+        file_path: 'src/auth/token.ts',
+        line_start: 0, line_end: 0,
+        metric: '38%',
+        body: 'Old description',
+        related: '',
+        card_path: '/cards/coverage-token.md',
+      });
+
+      // Update
+      upsertFinding(db, {
+        finding_kind: 'coverage',
+        title: 'Low coverage: token.ts',
+        severity: 'high',
+        file_path: 'src/auth/token.ts',
+        line_start: 0, line_end: 0,
+        metric: '25%',
+        body: 'Worse now with uniqueTermFindABC',
+        related: '',
+        card_path: '/cards/coverage-token.md',
+      });
+
+      // Search for new text (proves analysis_au fired)
+      const results = searchFindings(db, 'uniqueTermFindABC');
+      expect(results).toHaveLength(1);
+      expect(results[0]!.metric).toBe('25%');
+
+      // No duplicate
+      const all = searchFindings(db, '');
+      expect(all).toHaveLength(1);
+
+      db.close();
+    });
+
+    it('deleteFindingsForFile removes findings and FTS rows for a file', () => {
+      const db = openDb(dbPath);
+
+      upsertFinding(db, {
+        finding_kind: 'debt', title: 'TODO: fix me', severity: 'info',
+        file_path: 'src/a.ts', line_start: 5, line_end: 5,
+        metric: 'TODO', body: 'FixThisUniqueXYZ123', related: '',
+        card_path: '/cards/debt-a.md',
+      });
+      upsertFinding(db, {
+        finding_kind: 'debt', title: 'HACK: workaround', severity: 'high',
+        file_path: 'src/b.ts', line_start: 10, line_end: 10,
+        metric: 'HACK', body: 'Ugly workaround', related: '',
+        card_path: '/cards/debt-b.md',
+      });
+
+      deleteFindingsForFile(db, 'debt', 'src/a.ts');
+
+      const all = searchFindings(db, '');
+      expect(all).toHaveLength(1);
+      expect(all[0]!.name).toBe('HACK: workaround');
+
+      // FTS should be clean too — no result for the deleted finding's unique text
+      const ftsResults = searchFindings(db, 'FixThisUniqueXYZ123');
+      expect(ftsResults).toHaveLength(0);
+
+      db.close();
+    });
+
+    it('searchFindings empty query returns all findings ordered by severity, title', () => {
+      const db = openDb(dbPath);
+
+      upsertFinding(db, {
+        finding_kind: 'debt', title: 'Alpha debt', severity: 'info',
+        file_path: 'src/a.ts', line_start: 0, line_end: 0,
+        metric: 'TODO', body: '', related: '', card_path: '',
+      });
+      upsertFinding(db, {
+        finding_kind: 'coverage', title: 'Zebra coverage', severity: 'high',
+        file_path: 'src/b.ts', line_start: 0, line_end: 0,
+        metric: '10%', body: '', related: '', card_path: '',
+      });
+
+      const results = searchFindings(db, '');
+      expect(results.length).toBeGreaterThanOrEqual(2);
+
+      db.close();
+    });
+
+    it('searchFindings with kindFilter narrows by finding_kind', () => {
+      const db = openDb(dbPath);
+
+      upsertFinding(db, {
+        finding_kind: 'coverage', title: 'Coverage gap', severity: 'warn',
+        file_path: 'src/a.ts', line_start: 0, line_end: 0,
+        metric: '50%', body: '', related: '', card_path: '',
+      });
+      upsertFinding(db, {
+        finding_kind: 'debt', title: 'Debt item', severity: 'high',
+        file_path: 'src/b.ts', line_start: 0, line_end: 0,
+        metric: 'TODO', body: '', related: '', card_path: '',
+      });
+
+      const coverageResults = searchFindings(db, '', 'coverage');
+      expect(coverageResults).toHaveLength(1);
+      expect(coverageResults[0]!.finding_kind).toBe('coverage');
+
+      db.close();
+    });
+
+    it('searchFindings with query returns bm25-ranked results', () => {
+      const db = openDb(dbPath);
+
+      upsertFinding(db, {
+        finding_kind: 'debt', title: 'General', severity: 'info',
+        file_path: 'src/a.ts', line_start: 0, line_end: 0,
+        metric: 'TODO', body: 'Some text about authentication token refresh', related: '', card_path: '',
+      });
+      upsertFinding(db, {
+        finding_kind: 'debt', title: 'Token issue', severity: 'high',
+        file_path: 'src/b.ts', line_start: 0, line_end: 0,
+        metric: 'FIXME', body: 'Token validation is weak', related: '', card_path: '',
+      });
+
+      const results = searchFindings(db, 'token');
+      expect(results).toHaveLength(2);
+
+      db.close();
+    });
+  });
+
+  describe('convention notes', () => {
+    it('upsertNote accepts convention note_kind and it is searchable', () => {
+      const db = openDb(dbPath);
+
+      const id = upsertNote(db, {
+        note_kind: 'convention',
+        title: 'Use functional components',
+        body: 'All React components must be pure functions, not classes.',
+        tags: 'react,style',
+        related: 'Component, renderFunction',
+        card_path: '/entries/conventions/use-functional-components.md',
+      });
+
+      expect(id).toBeGreaterThan(0);
+
+      // Search by note_kind
+      const results = searchNotes(db, '', 'convention');
+      expect(results).toHaveLength(1);
+      expect(results[0]!.name).toBe('Use functional components');
+      expect(results[0]!.note_kind).toBe('convention');
+
+      // FTS search
+      const ftsResults = searchNotes(db, 'functional');
+      expect(ftsResults).toHaveLength(1);
+
+      db.close();
+    });
+  });
+
   describe('meta', () => {
     it('setMeta and getMeta round-trip values', () => {
       const db = openDb(dbPath);