ai-spec-dev 0.31.0 → 0.33.0

package/core/run-trend.ts

@@ -0,0 +1,241 @@
+import * as fs from "fs-extra";
+import * as path from "path";
+import chalk from "chalk";
+import { RunLog } from "./run-logger";
+
+const LOG_DIR = ".ai-spec-logs";
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export interface TrendEntry {
+  runId: string;
+  startedAt: string;
+  promptHash: string | null;
+  harnessScore: number | null;
+  specPath: string | null;
+  provider: string | null;
+  model: string | null;
+  filesWritten: number;
+  totalDurationMs: number | null;
+  errors: number;
+}
+
+export interface PromptGroupSummary {
+  promptHash: string;
+  runs: number;
+  avg: number;
+  best: number;
+  worst: number;
+  firstSeen: string;
+  lastSeen: string;
+  /** true if this is the most recently used prompt hash */
+  isCurrent: boolean;
+}
+
+export interface TrendReport {
+  entries: TrendEntry[];
+  promptGroups: PromptGroupSummary[];
+  totalRuns: number;
+}
+
+// ─── Loader ──────────────────────────────────────────────────────────────────
+
+/**
+ * Read all RunLog JSON files from `.ai-spec-logs/`, sorted newest-first.
+ * Silently skips unreadable / corrupt files.
+ */
+export async function loadRunLogs(workingDir: string): Promise<RunLog[]> {
+  const logDir = path.join(workingDir, LOG_DIR);
+  if (!(await fs.pathExists(logDir))) return [];
+
+  const files = await fs.readdir(logDir);
+  const jsonFiles = files.filter((f) => f.endsWith(".json")).sort().reverse();
+
+  const logs: RunLog[] = [];
+  for (const file of jsonFiles) {
+    try {
+      const log: RunLog = await fs.readJson(path.join(logDir, file));
+      // only include runs that have a startedAt (minimal validity check)
+      if (log.runId && log.startedAt) {
+        logs.push(log);
+      }
+    } catch {
+      // corrupt file — skip silently
+    }
+  }
+  return logs;
+}
+
+// ─── Aggregation ─────────────────────────────────────────────────────────────
+
+export function buildTrendReport(
+  logs: RunLog[],
+  opts: { last?: number; promptFilter?: string } = {}
+): TrendReport {
+  let entries: TrendEntry[] = logs.map((log) => ({
+    runId: log.runId,
+    startedAt: log.startedAt,
+    promptHash: log.promptHash ?? null,
+    harnessScore: log.harnessScore ?? null,
+    specPath: log.specPath ?? null,
+    provider: log.provider ?? null,
+    model: log.model ?? null,
+    filesWritten: log.filesWritten?.length ?? 0,
+    totalDurationMs: log.totalDurationMs ?? null,
+    errors: log.errors?.length ?? 0,
+  }));
+
+  // filter: only runs with a harnessScore (create runs)
+  entries = entries.filter((e) => e.harnessScore !== null);
+
+  // filter by prompt hash if requested
+  if (opts.promptFilter) {
+    entries = entries.filter((e) =>
+      e.promptHash?.startsWith(opts.promptFilter!)
+    );
+  }
+
+  // limit to last N
+  if (opts.last && opts.last > 0) {
+    entries = entries.slice(0, opts.last);
+  }
+
+  // build prompt group summaries (only from filtered entries)
+  const groupMap = new Map<string, TrendEntry[]>();
+  for (const e of entries) {
+    const key = e.promptHash ?? "(none)";
+    if (!groupMap.has(key)) groupMap.set(key, []);
+    groupMap.get(key)!.push(e);
+  }
+
+  // determine "current" = the prompt hash of the most recent run
+  const currentHash = entries[0]?.promptHash ?? null;
+
+  const promptGroups: PromptGroupSummary[] = [];
+  for (const [hash, group] of groupMap.entries()) {
+    const scores = group.map((e) => e.harnessScore as number);
+    promptGroups.push({
+      promptHash: hash,
+      runs: group.length,
+      avg: Math.round((scores.reduce((a, b) => a + b, 0) / scores.length) * 10) / 10,
+      best: Math.max(...scores),
+      worst: Math.min(...scores),
+      firstSeen: group[group.length - 1].startedAt,
+      lastSeen: group[0].startedAt,
+      isCurrent: hash === currentHash,
+    });
+  }
+
+  // sort groups: most recently used first
+  promptGroups.sort((a, b) => b.lastSeen.localeCompare(a.lastSeen));
+
+  return { entries, promptGroups, totalRuns: entries.length };
+}
+
+// ─── Display ─────────────────────────────────────────────────────────────────
+
+function scoreBar(score: number): string {
+  const filled = Math.round(score);
+  return "█".repeat(filled) + "░".repeat(10 - filled);
+}
+
+function scoreColor(score: number, text: string): string {
+  if (score >= 8) return chalk.green(text);
+  if (score >= 6) return chalk.yellow(text);
+  return chalk.red(text);
+}
+
+function formatDate(iso: string): string {
+  return iso.slice(0, 10); // YYYY-MM-DD
+}
+
+function formatDuration(ms: number | null): string {
+  if (ms === null) return "  —  ";
+  const s = Math.round(ms / 1000);
+  if (s < 60) return `${s}s`;
+  return `${Math.floor(s / 60)}m${s % 60}s`;
+}
+
+function shortSpec(specPath: string | null): string {
+  if (!specPath) return chalk.gray("—");
+  return path.basename(specPath);
+}
+
+export function printTrendReport(report: TrendReport, workingDir: string): void {
+  const { entries, promptGroups } = report;
+
+  console.log(chalk.cyan("\n─── Harness Trend ───────────────────────────────────────────"));
+
+  if (entries.length === 0) {
+    console.log(chalk.gray("  No scored runs found. Run `ai-spec create` to start tracking."));
+    console.log(chalk.cyan("─".repeat(63)));
+    return;
+  }
+
+  // ── Prompt Version Summary ────────────────────────────────────────
+  if (promptGroups.length > 0) {
+    console.log(chalk.bold("\n  Prompt Versions:\n"));
+
+    const colWidths = {
+      hash:  10,
+      runs:   5,
+      avg:    5,
+      best:   5,
+      worst:  5,
+    };
+
+    // header
+    console.log(
+      chalk.gray(
+        "  " +
+        "Hash      ".padEnd(colWidths.hash) + " " +
+        "Runs ".padStart(colWidths.runs) + " " +
+        "  Avg" + " " +
+        " Best" + " " +
+        "Worst" + "  " +
+        "Last seen"
+      )
+    );
+    console.log(chalk.gray("  " + "─".repeat(55)));
+
+    for (const g of promptGroups) {
+      const currentMark = g.isCurrent ? chalk.cyan(" ◀ current") : "";
+      const avgStr = scoreColor(g.avg, g.avg.toFixed(1).padStart(5));
+      const bestStr = chalk.green(g.best.toFixed(1).padStart(5));
+      const worstStr = g.worst < 6 ? chalk.red(g.worst.toFixed(1).padStart(5)) : chalk.yellow(g.worst.toFixed(1).padStart(5));
+
+      console.log(
+        "  " +
+        chalk.white(g.promptHash.padEnd(colWidths.hash)) + " " +
+        chalk.gray(String(g.runs).padStart(colWidths.runs)) + " " +
+        avgStr + " " +
+        bestStr + " " +
+        worstStr + "  " +
+        chalk.gray(formatDate(g.lastSeen)) +
+        currentMark
+      );
+    }
+  }
+
+  // ── Run History ───────────────────────────────────────────────────
+  console.log(chalk.bold("\n  Run History:\n"));
+
+  for (const e of entries) {
+    const score = e.harnessScore as number;
+    const bar   = scoreColor(score, `[${scoreBar(score)}]`);
+    const scoreStr = scoreColor(score, score.toFixed(1).padStart(4));
+    const hash  = e.promptHash ? chalk.gray(e.promptHash) : chalk.gray("(no hash)");
+    const dur   = chalk.gray(formatDuration(e.totalDurationMs));
+    const errMark = e.errors > 0 ? chalk.yellow(` ⚠${e.errors}err`) : "";
+    const spec  = chalk.gray(shortSpec(e.specPath));
+
+    console.log(
+      `  ${chalk.gray(formatDate(e.startedAt))}  ${bar}${scoreStr}  ${hash}  ${dur}${errMark}  ${spec}`
+    );
+  }
+
+  // ── Footer ────────────────────────────────────────────────────────
+  const logRelDir = path.relative(workingDir, path.join(workingDir, LOG_DIR));
+  console.log(chalk.gray(`\n  ${entries.length} run(s) shown  ·  logs: ${logRelDir}/`));
+  console.log(chalk.cyan("─".repeat(63)));
+}

package/core/self-evaluator.ts

@@ -18,8 +18,14 @@ export interface SelfEvalResult {
   detail: {
     endpointsTotal: number;
     endpointLayerCovered: boolean;
+    /** Number of endpoint-layer files generated */
+    endpointLayerFiles: number;
     modelsTotal: number;
     modelLayerCovered: boolean;
+    /** 0-1: fraction of DSL model names found in generated file paths */
+    modelNameCoverage: number;
+    /** Number of DSL model names actually matched in file paths */
+    modelNameMatched: number;
     filesWritten: number;
   };
 }
@@ -57,6 +63,32 @@ function extractReviewScore(reviewText: string): number | null {
 
 // ─── Main ─────────────────────────────────────────────────────────────────────
 
+/**
+ * Normalize a PascalCase or camelCase model name to a set of search tokens
+ * that would appear in file paths.
+ *
+ * "OrderItem" → ["orderitem", "order-item", "order_item"]
+ * "User"      → ["user"]
+ */
+export function modelNameTokens(name: string): string[] {
+  const lower = name.toLowerCase();
+  // split on uppercase boundaries: "OrderItem" → ["order", "item"]
+  const parts = name
+    .replace(/([A-Z])/g, "-$1")
+    .toLowerCase()
+    .replace(/^-/, "")
+    .split("-")
+    .filter(Boolean);
+
+  const tokens = new Set<string>();
+  tokens.add(lower);
+  if (parts.length > 1) {
+    tokens.add(parts.join("-"));
+    tokens.add(parts.join("_"));
+  }
+  return [...tokens];
+}
+
 /**
  * Run a lightweight self-evaluation at the end of `ai-spec create`.
  *
@@ -71,6 +103,18 @@ function extractReviewScore(reviewText: string): number | null {
  *  | DSL Coverage    | 40 %                 | 55 %                    |
  *  | Compile/Error   | 30 %                 | 45 %                    |
  *  | Review Score    | 30 %                 | —                       |
+ *
+ * DSL Coverage Score breakdown (0-10):
+ *  Tier 1 — Layer existence (same as before):
+ *    - No files generated                        → 0 (early exit)
+ *    - Endpoints declared but no endpoint layer  → -4
+ *    - Models declared but no model layer        → -3
+ *  Tier 2 — Model name coverage (new):
+ *    - coverage < 50 %                           → -2
+ *    - coverage 50–79 %                          → -1
+ *    - coverage ≥ 80 %                           → 0
+ *  Tier 3 — Endpoint file adequacy (new):
+ *    - ≥5 endpoints declared but only 1 endpoint-layer file → -1
  */
 export function runSelfEval(opts: {
   dsl: SpecDSL | null;
@@ -91,18 +135,55 @@ export function runSelfEval(opts: {
   const endpointLayerCovered = generatedFiles.some((f) =>
     ENDPOINT_LAYER_PATTERNS.some((p) => p.test(f))
   );
+  const endpointLayerFiles = generatedFiles.filter((f) =>
+    ENDPOINT_LAYER_PATTERNS.some((p) => p.test(f))
+  ).length;
   const modelLayerCovered = generatedFiles.some((f) =>
     MODEL_LAYER_PATTERNS.some((p) => p.test(f))
   );
 
+  // ── Tier 2: Model name coverage ───────────────────────────────────────────
+  // For each DSL model, check if its name (lowercased/tokenized) appears
+  // in any generated file path. This catches "User model was declared but
+  // no user.ts / user.model.ts was generated".
+  let modelNameMatched = 0;
+  if (modelsTotal > 0 && dsl?.models) {
+    for (const model of dsl.models) {
+      const tokens = modelNameTokens(model.name);
+      const found = generatedFiles.some((f) => {
+        const lf = f.toLowerCase();
+        return tokens.some((t) => lf.includes(t));
+      });
+      if (found) modelNameMatched++;
+    }
+  }
+  const modelNameCoverage = modelsTotal > 0 ? modelNameMatched / modelsTotal : 1;
+
+  // ── Compute DSL Coverage Score ────────────────────────────────────────────
   let dslCoverageScore = 10;
+
   if (generatedFiles.length === 0) {
     dslCoverageScore = 0;
   } else {
+    // Tier 1: layer existence
     if (endpointsTotal > 0 && !endpointLayerCovered) dslCoverageScore -= 4;
     if (modelsTotal    > 0 && !modelLayerCovered)    dslCoverageScore -= 3;
+
+    // Tier 2: model name coverage (only meaningful when model layer exists)
+    if (modelsTotal > 0 && modelLayerCovered) {
+      if (modelNameCoverage < 0.5)       dslCoverageScore -= 2;
+      else if (modelNameCoverage < 0.8)  dslCoverageScore -= 1;
+    }
+
+    // Tier 3: endpoint file adequacy (many endpoints, very few files)
+    if (endpointsTotal >= 5 && endpointLayerCovered && endpointLayerFiles < 2) {
+      dslCoverageScore -= 1;
+    }
   }
 
+  // clamp to [0, 10]
+  dslCoverageScore = Math.max(0, Math.min(10, dslCoverageScore));
+
   // ── Compile Score ─────────────────────────────────────────────────────────
   // 10 = clean pass, 5 = error feedback ran but didn't fully clear / was skipped
   const compileScore = compilePassed ? 10 : 5;
@@ -124,8 +205,11 @@ export function runSelfEval(opts: {
     detail: {
       endpointsTotal,
       endpointLayerCovered,
+      endpointLayerFiles,
       modelsTotal,
       modelLayerCovered,
+      modelNameCoverage: Math.round(modelNameCoverage * 100) / 100,
+      modelNameMatched,
       filesWritten: generatedFiles.length,
     },
   };
@@ -138,6 +222,9 @@ export function runSelfEval(opts: {
     compileScore,
     reviewScore: reviewScore ?? undefined,
     promptHash,
+    modelNameCoverage: result.detail.modelNameCoverage,
+    modelNameMatched:  result.detail.modelNameMatched,
+    endpointLayerFiles: result.detail.endpointLayerFiles,
   });
 
   return result;
@@ -161,12 +248,29 @@ export function printSelfEval(result: SelfEvalResult): void {
     ? `Review: ${result.reviewScore}/10`
     : chalk.gray("Review: skipped");
 
+  // Model coverage tag (only shown when there are declared models)
+  let modelCoverageTag = "";
+  if (result.detail.modelsTotal > 0) {
+    const pct = Math.round(result.detail.modelNameCoverage * 100);
+    const tag = `Models: ${result.detail.modelNameMatched}/${result.detail.modelsTotal} (${pct}%)`;
+    modelCoverageTag = pct >= 80
+      ? chalk.green(tag)
+      : pct >= 50
+        ? chalk.yellow(tag)
+        : chalk.red(tag);
+  }
+
   console.log(chalk.cyan("\n─── Harness Self-Eval ───────────────────────────"));
   console.log(`  Score  : ${scoreColor(`[${bar}] ${result.harnessScore}/10`)}`);
   console.log(
-    `  DSL    : ${scoreColor(result.dslCoverageScore + "/10")}  ` +
+    `  DSL    : ${scoreColor(String(result.dslCoverageScore) + "/10")}  ` +
     `Compile: ${compileTag}  ${reviewTag}`
   );
+  if (modelCoverageTag) {
+    console.log(`  Detail : ${modelCoverageTag}  ` +
+      chalk.gray(`Endpoints: ${result.detail.endpointsTotal}  Files: ${result.detail.filesWritten}`)
+    );
+  }
   console.log(chalk.gray(`  Prompt : ${result.promptHash}`));
-  console.log(chalk.gray("─".repeat(49)));
+  console.log(chalk.cyan("─".repeat(49)));
 }