ai-spec-dev 0.30.1 → 0.33.0

package/core/dsl-feedback.ts

@@ -0,0 +1,255 @@
+/**
+ * dsl-feedback.ts — Two pipeline feedback loops for ai-spec create
+ *
+ * Loop 1 (DSL → Spec):  after DSL extraction, detect sparse/incomplete DSL
+ *   and offer a targeted spec refinement pass before codegen starts.
+ *
+ * Loop 2 (Review → DSL): after 3-pass review, detect design-level findings
+ *   (as opposed to implementation issues) and offer to amend the spec + DSL
+ *   so the next update/regen starts from a corrected contract.
+ *
+ * Design constraints:
+ *  - Both loops are SKIPPED in --auto / --fast / --skip-dsl modes.
+ *  - Zero extra AI calls until the user explicitly opts in.
+ *  - Non-blocking: user can always skip.
+ */
+
+import chalk from "chalk";
+import { SpecDSL } from "./dsl-types";
+
+// ─── Loop 1 Types ─────────────────────────────────────────────────────────────
+
+export interface DslGap {
+  /** Short machine key for RunLog serialisation */
+  code: "sparse_model" | "missing_errors" | "generic_endpoint_desc" | "no_models_no_endpoints";
+  /** Human-readable message shown to the user */
+  message: string;
+  /** Concrete suggestion injected into the refinement prompt */
+  hint: string;
+}
+
+// ─── Loop 1: DSL Richness Assessment ─────────────────────────────────────────
+
+/**
+ * Inspect a freshly-extracted DSL for common completeness gaps.
+ * Returns a list of DslGap objects (empty = DSL looks adequate).
+ *
+ * All checks are pure heuristics — zero AI calls.
+ */
+export function assessDslRichness(dsl: SpecDSL): DslGap[] {
+  const gaps: DslGap[] = [];
+
+  // ── No endpoints AND no models ────────────────────────────────────────────
+  if (dsl.endpoints.length === 0 && dsl.models.length === 0) {
+    gaps.push({
+      code: "no_models_no_endpoints",
+      message: "DSL has no endpoints and no models — spec may be too abstract for structured extraction",
+      hint: "Please add explicit API endpoint definitions (method, path, request/response) and any data models that this feature requires.",
+    });
+    return gaps; // no point checking the rest
+  }
+
+  // ── Endpoints with very generic / short descriptions ─────────────────────
+  const GENERIC_DESC_KEYWORDS = ["handles", "processes", "manages", "操作", "处理", "管理"];
+  const GENERIC_DESC_MIN_LEN  = 15;
+
+  for (const ep of dsl.endpoints) {
+    const desc = (ep.description ?? "").trim();
+    const isGeneric =
+      desc.length < GENERIC_DESC_MIN_LEN ||
+      GENERIC_DESC_KEYWORDS.some((kw) => desc.toLowerCase().startsWith(kw));
+
+    if (isGeneric) {
+      gaps.push({
+        code: "generic_endpoint_desc",
+        message: `Endpoint ${ep.method} ${ep.path} has a vague description: "${desc}"`,
+        hint: `Clarify what ${ep.method} ${ep.path} does: what inputs are required, what the success response contains, and what business rule it enforces.`,
+      });
+    }
+  }
+
+  // ── Endpoints with no error definitions (but spec text likely mentions them) ──
+  const endpointsWithoutErrors = dsl.endpoints.filter(
+    (ep) => !ep.errors || ep.errors.length === 0
+  );
+  if (endpointsWithoutErrors.length > 0 && dsl.endpoints.length >= 2) {
+    gaps.push({
+      code: "missing_errors",
+      message: `${endpointsWithoutErrors.length}/${dsl.endpoints.length} endpoints have no error definitions`,
+      hint: `For each endpoint, specify at least the main error cases: e.g. 400 validation errors, 401 auth failures, 404 not found, 409 conflict. Include an error code (e.g. INVALID_INPUT) and description for each.`,
+    });
+  }
+
+  // ── Models with fewer than 2 fields ──────────────────────────────────────
+  for (const model of dsl.models) {
+    if (!model.fields || model.fields.length < 2) {
+      gaps.push({
+        code: "sparse_model",
+        message: `Model "${model.name}" has only ${model.fields?.length ?? 0} field(s) — likely incomplete`,
+        hint: `List all fields for "${model.name}" with their types and whether they are required. Include at minimum an id, created_at, and the core domain fields this model needs.`,
+      });
+    }
+  }
+
+  return gaps;
+}
+
+// ─── Loop 1: Targeted Spec Refinement Prompt ─────────────────────────────────
+
+/**
+ * Build a targeted AI refinement prompt that focuses the LLM on filling
+ * only the specific gaps detected by `assessDslRichness`.
+ */
+export function buildDslGapRefinementPrompt(spec: string, gaps: DslGap[]): string {
+  const gapList = gaps
+    .map((g, i) => `${i + 1}. [${g.code}] ${g.message}\n   → ${g.hint}`)
+    .join("\n\n");
+
+  return `The following feature spec has been structurally analysed. The DSL extracted from it was found to be incomplete in these specific areas:
+
+${gapList}
+
+Your task: revise the spec below to address ONLY the gaps listed above.
+- Do NOT change the overall feature scope or business logic.
+- Do NOT rewrite sections that are already complete.
+- Add missing error cases, clarify vague endpoint descriptions, complete sparse model field lists.
+- Output ONLY the complete revised Markdown spec. No preamble, no explanation.
+
+=== Current Spec ===
+${spec}`;
+}
+
+// ─── Loop 2 Types ─────────────────────────────────────────────────────────────
+
+export interface StructuralFinding {
+  /** Short label for display + RunLog */
+  category: "auth_design" | "model_design" | "api_contract" | "layer_violation" | "other_design";
+  description: string;
+}
+
+// ─── Loop 2: Review Structural Issue Classifier ───────────────────────────────
+
+/**
+ * Parse a 3-pass review text to extract Pass 1 (architecture) findings
+ * that indicate design-level issues in the Spec/DSL — as opposed to
+ * implementation-level issues that belong in §9 knowledge.
+ *
+ * Returns an empty array if no structural issues are found or if the
+ * review score for Pass 1 is high (≥ 8), indicating overall approval.
+ */
+export function extractStructuralFindings(reviewText: string): StructuralFinding[] {
+  // Split by the separator used between passes ("─────...")
+  const parts = reviewText.split(/─{20,}/);
+  // Pass 1 is always the first section
+  const pass1Text = parts[0] ?? "";
+
+  // If Pass 1 scored well, treat as no structural issues
+  const pass1Score = extractPassScore(pass1Text);
+  if (pass1Score !== null && pass1Score >= 8) return [];
+
+  const findings: StructuralFinding[] = [];
+
+  // ── Auth / 认证 design issues ──────────────────────────────────────────
+  if (
+    /缺少认证|missing auth|auth.*false|未加认证|鉴权.*缺|endpoint.*public.*should/i.test(pass1Text)
+  ) {
+    const match = pass1Text.match(/[^。\n]*(?:缺少认证|missing auth|auth.*false|未加认证|鉴权.*缺|endpoint.*public.*should)[^。\n]*/i);
+    findings.push({
+      category: "auth_design",
+      description: match ? match[0].trim() : "One or more endpoints may have incorrect authentication requirements",
+    });
+  }
+
+  // ── API contract / 接口设计 issues ────────────────────────────────────
+  if (
+    /接口设计.*问题|接口.*不合理|API design|response.*missing|request.*missing|接口.*缺少/i.test(pass1Text)
+  ) {
+    const match = pass1Text.match(/[^。\n]*(?:接口设计.*问题|接口.*不合理|API design|response.*missing|接口.*缺少)[^。\n]*/i);
+    findings.push({
+      category: "api_contract",
+      description: match ? match[0].trim() : "API contract design may have issues",
+    });
+  }
+
+  // ── Model / 数据模型 design issues ────────────────────────────────────
+  if (
+    /模型.*缺少字段|model.*missing field|数据结构.*问题|schema.*incomplete|字段.*missing/i.test(pass1Text)
+  ) {
+    const match = pass1Text.match(/[^。\n]*(?:模型.*缺少字段|model.*missing field|数据结构.*问题|schema.*incomplete)[^。\n]*/i);
+    findings.push({
+      category: "model_design",
+      description: match ? match[0].trim() : "Data model design may be incomplete",
+    });
+  }
+
+  // ── Layer separation / 层级分离 violations ────────────────────────────
+  if (
+    /层级.*违反|layer.*violation|business logic.*controller|controller.*service.*混|分层.*问题/i.test(pass1Text)
+  ) {
+    const match = pass1Text.match(/[^。\n]*(?:层级.*违反|layer.*violation|business logic.*controller|分层.*问题)[^。\n]*/i);
+    findings.push({
+      category: "layer_violation",
+      description: match ? match[0].trim() : "Layer separation may be violated in the generated code",
+    });
+  }
+
+  return findings;
+}
+
+/** Extract the numeric score from a single pass section. */
+function extractPassScore(text: string): number | null {
+  const m = text.match(/Score:\s*(\d+(?:\.\d+)?)\s*\/\s*10/i);
+  return m ? parseFloat(m[1]) : null;
+}
+
+// ─── Loop 2: Spec Amendment Prompt ────────────────────────────────────────────
+
+/**
+ * Build a prompt asking the AI to produce a minimal spec amendment
+ * that addresses the structural findings from the review.
+ *
+ * The amendment is a targeted addition/correction — NOT a full rewrite.
+ */
+export function buildStructuralAmendmentPrompt(
+  spec: string,
+  findings: StructuralFinding[]
+): string {
+  const findingList = findings
+    .map((f, i) => `${i + 1}. [${f.category}] ${f.description}`)
+    .join("\n");
+
+  return `A code review of the feature built from this spec found the following DESIGN-LEVEL issues.
+These are problems in the spec/contract itself, not in the implementation.
+
+=== Structural Findings ===
+${findingList}
+
+Your task:
+- Revise the spec below to correct the design issues listed above.
+- Do NOT change the feature scope, business logic, or sections unrelated to these findings.
+- Be minimal: only change what is necessary to fix the design issues.
+- Output ONLY the complete revised Markdown spec. No preamble, no explanation.
+
+=== Current Spec ===
+${spec}`;
+}
+
+// ─── Display Helpers ──────────────────────────────────────────────────────────
+
+export function printDslGaps(gaps: DslGap[]): void {
+  console.log(chalk.yellow("\n  ⚠ DSL Completeness Check — gaps detected:"));
+  for (const gap of gaps) {
+    console.log(chalk.yellow(`    · ${gap.message}`));
+  }
+  console.log(chalk.gray("    → A targeted spec refinement can fill these gaps before codegen."));
+}
+
+export function printStructuralFindings(findings: StructuralFinding[]): void {
+  console.log(chalk.yellow("\n  ⚠ Review — structural (design-level) issues found:"));
+  for (const f of findings) {
+    const label = chalk.gray(`[${f.category}]`);
+    console.log(`    ${label} ${f.description}`);
+  }
+  console.log(chalk.gray("    → These are contract issues in the Spec/DSL, not just implementation problems."));
+  console.log(chalk.gray("    → Fixing the spec now means the next run generates correct code from the start."));
+}

package/core/prompt-hasher.ts

@@ -0,0 +1,42 @@
+import { createHash } from "crypto";
+
+import { codeGenSystemPrompt } from "../prompts/codegen.prompt";
+import {
+  reviewArchitectureSystemPrompt,
+  reviewImplementationSystemPrompt,
+  reviewImpactComplexitySystemPrompt,
+} from "../prompts/codegen.prompt";
+import { dslSystemPrompt } from "../prompts/dsl.prompt";
+import { specPrompt } from "../prompts/spec.prompt";
+
+/**
+ * Compute a short deterministic hash of the key prompt strings used in a run.
+ *
+ * Why this matters (Harness Engineering):
+ *   When you change a prompt and re-run `ai-spec create`, the resulting RunLog
+ *   will have a different promptHash. Cross-referencing RunLogs by promptHash
+ *   lets you quantify whether a prompt change improved or degraded harnessScore
+ *   without keeping a separate changelog.
+ *
+ * Coverage: codegen system prompt (TS), DSL extractor, spec generator, and all
+ *   three review-pass prompts — these drive the vast majority of token spend and
+ *   output variance.
+ *
+ * Returns: 8-char lowercase hex (e.g. "a3f2c1d8"). Collision probability for
+ *   practical prompt-tweak scenarios is negligible.
+ */
+export function computePromptHash(): string {
+  const segments = [
+    codeGenSystemPrompt,
+    dslSystemPrompt,
+    specPrompt,
+    reviewArchitectureSystemPrompt,
+    reviewImplementationSystemPrompt,
+    reviewImpactComplexitySystemPrompt,
+  ];
+
+  return createHash("sha256")
+    .update(segments.join("\x00"))   // \x00 separator prevents segment-boundary collisions
+    .digest("hex")
+    .slice(0, 8);
+}

package/core/run-logger.ts

@@ -20,6 +20,15 @@ export interface RunLog {
   provider?: string;
   model?: string;
   specPath?: string;
+  /**
+   * 8-char hex hash of the key prompt strings used in this run.
+   * Changes whenever any of: codegen, DSL, spec, or review prompts are edited.
+   * Use this to correlate RunLogs across runs and measure whether a prompt
+   * change improved or degraded harnessScore (Harness Engineering observability).
+   */
+  promptHash?: string;
+  /** Harness self-evaluation score recorded at end of `create` (0-10). */
+  harnessScore?: number;
   entries: LogEntry[];
   filesWritten: string[];
   errors: string[];
@@ -73,6 +82,18 @@ export class RunLogger {
     this.flush();
   }
 
+  /** Record the prompt hash for this run (call once at run start). */
+  setPromptHash(hash: string): void {
+    this.log.promptHash = hash;
+    this.flush();
+  }
+
+  /** Record the harness self-eval score (call once at run end). */
+  setHarnessScore(score: number): void {
+    this.log.harnessScore = score;
+    this.flush();
+  }
+
   fileWritten(filePath: string): void {
     if (!this.log.filesWritten.includes(filePath)) {
       this.log.filesWritten.push(filePath);

package/core/run-trend.ts

@@ -0,0 +1,241 @@
+import * as fs from "fs-extra";
+import * as path from "path";
+import chalk from "chalk";
+import { RunLog } from "./run-logger";
+
+const LOG_DIR = ".ai-spec-logs";
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export interface TrendEntry {
+  runId: string;
+  startedAt: string;
+  promptHash: string | null;
+  harnessScore: number | null;
+  specPath: string | null;
+  provider: string | null;
+  model: string | null;
+  filesWritten: number;
+  totalDurationMs: number | null;
+  errors: number;
+}
+
+export interface PromptGroupSummary {
+  promptHash: string;
+  runs: number;
+  avg: number;
+  best: number;
+  worst: number;
+  firstSeen: string;
+  lastSeen: string;
+  /** true if this is the most recently used prompt hash */
+  isCurrent: boolean;
+}
+
+export interface TrendReport {
+  entries: TrendEntry[];
+  promptGroups: PromptGroupSummary[];
+  totalRuns: number;
+}
+
+// ─── Loader ──────────────────────────────────────────────────────────────────
+
+/**
+ * Read all RunLog JSON files from `.ai-spec-logs/`, sorted newest-first.
+ * Silently skips unreadable / corrupt files.
+ */
+export async function loadRunLogs(workingDir: string): Promise<RunLog[]> {
+  const logDir = path.join(workingDir, LOG_DIR);
+  if (!(await fs.pathExists(logDir))) return [];
+
+  const files = await fs.readdir(logDir);
+  const jsonFiles = files.filter((f) => f.endsWith(".json")).sort().reverse();
+
+  const logs: RunLog[] = [];
+  for (const file of jsonFiles) {
+    try {
+      const log: RunLog = await fs.readJson(path.join(logDir, file));
+      // only include runs that have a startedAt (minimal validity check)
+      if (log.runId && log.startedAt) {
+        logs.push(log);
+      }
+    } catch {
+      // corrupt file — skip silently
+    }
+  }
+  return logs;
+}
+
+// ─── Aggregation ─────────────────────────────────────────────────────────────
+
+export function buildTrendReport(
+  logs: RunLog[],
+  opts: { last?: number; promptFilter?: string } = {}
+): TrendReport {
+  let entries: TrendEntry[] = logs.map((log) => ({
+    runId: log.runId,
+    startedAt: log.startedAt,
+    promptHash: log.promptHash ?? null,
+    harnessScore: log.harnessScore ?? null,
+    specPath: log.specPath ?? null,
+    provider: log.provider ?? null,
+    model: log.model ?? null,
+    filesWritten: log.filesWritten?.length ?? 0,
+    totalDurationMs: log.totalDurationMs ?? null,
+    errors: log.errors?.length ?? 0,
+  }));
+
+  // filter: only runs with a harnessScore (create runs)
+  entries = entries.filter((e) => e.harnessScore !== null);
+
+  // filter by prompt hash if requested
+  if (opts.promptFilter) {
+    entries = entries.filter((e) =>
+      e.promptHash?.startsWith(opts.promptFilter!)
+    );
+  }
+
+  // limit to last N
+  if (opts.last && opts.last > 0) {
+    entries = entries.slice(0, opts.last);
+  }
+
+  // build prompt group summaries (only from filtered entries)
+  const groupMap = new Map<string, TrendEntry[]>();
+  for (const e of entries) {
+    const key = e.promptHash ?? "(none)";
+    if (!groupMap.has(key)) groupMap.set(key, []);
+    groupMap.get(key)!.push(e);
+  }
+
+  // determine "current" = the prompt hash of the most recent run
+  const currentHash = entries[0]?.promptHash ?? null;
+
+  const promptGroups: PromptGroupSummary[] = [];
+  for (const [hash, group] of groupMap.entries()) {
+    const scores = group.map((e) => e.harnessScore as number);
+    promptGroups.push({
+      promptHash: hash,
+      runs: group.length,
+      avg: Math.round((scores.reduce((a, b) => a + b, 0) / scores.length) * 10) / 10,
+      best: Math.max(...scores),
+      worst: Math.min(...scores),
+      firstSeen: group[group.length - 1].startedAt,
+      lastSeen: group[0].startedAt,
+      isCurrent: hash === currentHash,
+    });
+  }
+
+  // sort groups: most recently used first
+  promptGroups.sort((a, b) => b.lastSeen.localeCompare(a.lastSeen));
+
+  return { entries, promptGroups, totalRuns: entries.length };
+}
+
+// ─── Display ─────────────────────────────────────────────────────────────────
+
+function scoreBar(score: number): string {
+  const filled = Math.round(score);
+  return "█".repeat(filled) + "░".repeat(10 - filled);
+}
+
+function scoreColor(score: number, text: string): string {
+  if (score >= 8) return chalk.green(text);
+  if (score >= 6) return chalk.yellow(text);
+  return chalk.red(text);
+}
+
+function formatDate(iso: string): string {
+  return iso.slice(0, 10); // YYYY-MM-DD
+}
+
+function formatDuration(ms: number | null): string {
+  if (ms === null) return "  —  ";
+  const s = Math.round(ms / 1000);
+  if (s < 60) return `${s}s`;
+  return `${Math.floor(s / 60)}m${s % 60}s`;
+}
+
+function shortSpec(specPath: string | null): string {
+  if (!specPath) return chalk.gray("—");
+  return path.basename(specPath);
+}
+
+export function printTrendReport(report: TrendReport, workingDir: string): void {
+  const { entries, promptGroups } = report;
+
+  console.log(chalk.cyan("\n─── Harness Trend ───────────────────────────────────────────"));
+
+  if (entries.length === 0) {
+    console.log(chalk.gray("  No scored runs found. Run `ai-spec create` to start tracking."));
+    console.log(chalk.cyan("─".repeat(63)));
+    return;
+  }
+
+  // ── Prompt Version Summary ────────────────────────────────────────
+  if (promptGroups.length > 0) {
+    console.log(chalk.bold("\n  Prompt Versions:\n"));
+
+    const colWidths = {
+      hash:  10,
+      runs:   5,
+      avg:    5,
+      best:   5,
+      worst:  5,
+    };
+
+    // header
+    console.log(
+      chalk.gray(
+        "  " +
+        "Hash      ".padEnd(colWidths.hash) + " " +
+        "Runs ".padStart(colWidths.runs) + " " +
+        "  Avg" + " " +
+        " Best" + " " +
+        "Worst" + "  " +
+        "Last seen"
+      )
+    );
+    console.log(chalk.gray("  " + "─".repeat(55)));
+
+    for (const g of promptGroups) {
+      const currentMark = g.isCurrent ? chalk.cyan(" ◀ current") : "";
+      const avgStr = scoreColor(g.avg, g.avg.toFixed(1).padStart(5));
+      const bestStr = chalk.green(g.best.toFixed(1).padStart(5));
+      const worstStr = g.worst < 6 ? chalk.red(g.worst.toFixed(1).padStart(5)) : chalk.yellow(g.worst.toFixed(1).padStart(5));
+
+      console.log(
+        "  " +
+        chalk.white(g.promptHash.padEnd(colWidths.hash)) + " " +
+        chalk.gray(String(g.runs).padStart(colWidths.runs)) + " " +
+        avgStr + " " +
+        bestStr + " " +
+        worstStr + "  " +
+        chalk.gray(formatDate(g.lastSeen)) +
+        currentMark
+      );
+    }
+  }
+
+  // ── Run History ───────────────────────────────────────────────────
+  console.log(chalk.bold("\n  Run History:\n"));
+
+  for (const e of entries) {
+    const score = e.harnessScore as number;
+    const bar   = scoreColor(score, `[${scoreBar(score)}]`);
+    const scoreStr = scoreColor(score, score.toFixed(1).padStart(4));
+    const hash  = e.promptHash ? chalk.gray(e.promptHash) : chalk.gray("(no hash)");
+    const dur   = chalk.gray(formatDuration(e.totalDurationMs));
+    const errMark = e.errors > 0 ? chalk.yellow(` ⚠${e.errors}err`) : "";
+    const spec  = chalk.gray(shortSpec(e.specPath));
+
+    console.log(
+      `  ${chalk.gray(formatDate(e.startedAt))}  ${bar}${scoreStr}  ${hash}  ${dur}${errMark}  ${spec}`
+    );
+  }
+
+  // ── Footer ────────────────────────────────────────────────────────
+  const logRelDir = path.relative(workingDir, path.join(workingDir, LOG_DIR));
+  console.log(chalk.gray(`\n  ${entries.length} run(s) shown  ·  logs: ${logRelDir}/`));
+  console.log(chalk.cyan("─".repeat(63)));
+}