ai-spec-dev 0.37.0 → 0.41.0

package/core/codegen/topo-sort.ts

@@ -0,0 +1,98 @@
+import chalk from "chalk";
+import { SpecTask } from "../task-generator";
+
+// ─── Topological Batch Sort ────────────────────────────────────────────────────
+
+/**
+ * Partition tasks within a layer into ordered batches that respect the
+ * `dependencies` field.  Tasks in the same batch have no intra-layer
+ * dependencies on each other and can run in parallel.  Tasks in later batches
+ * wait for earlier batches to complete.
+ *
+ * Only intra-layer dependencies (i.e. deps whose IDs also appear in `tasks`)
+ * are considered — cross-layer ordering is already handled by LAYER_ORDER.
+ *
+ * Returns at least one batch.  On circular-dependency detection the remaining
+ * tasks are dumped into a final batch so execution always completes.
+ */
+export function topoSortLayerTasks(tasks: SpecTask[]): SpecTask[][] {
+  if (tasks.length <= 1) return [tasks];
+
+  const idSet = new Set(tasks.map((t) => t.id));
+  const taskById = new Map(tasks.map((t) => [t.id, t]));
+  const inDegree = new Map<string, number>();
+  const dependents = new Map<string, string[]>(); // dep → tasks that depend on it
+
+  for (const task of tasks) {
+    inDegree.set(task.id, 0);
+    dependents.set(task.id, []);
+  }
+
+  for (const task of tasks) {
+    const intraDeps = task.dependencies.filter((dep) => idSet.has(dep));
+    inDegree.set(task.id, intraDeps.length);
+    for (const dep of intraDeps) {
+      dependents.get(dep)!.push(task.id);
+    }
+  }
+
+  const batches: SpecTask[][] = [];
+  const remaining = new Set(tasks.map((t) => t.id));
+
+  while (remaining.size > 0) {
+    const batch = [...remaining]
+      .filter((id) => inDegree.get(id) === 0)
+      .map((id) => taskById.get(id)!);
+
+    if (batch.length === 0) {
+      // Circular dependency — run all remaining tasks in parallel to avoid deadlock
+      batches.push([...remaining].map((id) => taskById.get(id)!));
+      break;
+    }
+
+    batches.push(batch);
+    for (const task of batch) {
+      remaining.delete(task.id);
+      for (const dependent of dependents.get(task.id)!) {
+        inDegree.set(dependent, inDegree.get(dependent)! - 1);
+      }
+    }
+  }
+
+  return batches;
+}
+
+// ─── Progress Bar Helper ───────────────────────────────────────────────────────
+
+export const LAYER_ICONS: Record<string, string> = {
+  data: "💾",
+  infra: "⚙️ ",
+  service: "🔧",
+  api: "🌐",
+  view: "🖥️ ",
+  route: "🗺️ ",
+  test: "🧪",
+};
+
+export function printTaskProgress(
+  completed: number,
+  total: number,
+  task: SpecTask,
+  mode: "run" | "skip"
+): void {
+  const pct = total > 0 ? Math.round((completed / total) * 100) : 0;
+  const barWidth = 20;
+  const filled = Math.round((pct / 100) * barWidth);
+  const bar = chalk.green("█".repeat(filled)) + chalk.gray("░".repeat(barWidth - filled));
+  const icon = LAYER_ICONS[task.layer] ?? "  ";
+
+  if (mode === "skip") {
+    console.log(
+      chalk.gray(`\n  [${bar}] ${pct}% ✓ ${task.id} ${icon} ${task.title} — already done`)
+    );
+  } else {
+    console.log(
+      chalk.bold(`\n  [${bar}] ${pct}% → ${task.id} ${icon} ${task.title}`)
+    );
+  }
+}

package/core/constitution-consolidator.ts

@@ -108,8 +108,8 @@ export class ConstitutionConsolidator {
     // ── Show diff ───────────────────────────────────────────────────────────
     const diff = computeDiff(original, consolidated);
     console.log(chalk.blue("\n  Changes preview:"));
-    printDiff(diff, 4);
-    printDiffSummary(diff);
+    printDiff(diff);
+    printDiffSummary(diff, "consolidation");
 
     console.log(chalk.cyan("\n  After consolidation:"));
     console.log(chalk.gray(`  Size    : ${after.totalLines} lines (was ${before.totalLines})`));

package/core/dsl-coverage-checker.ts

@@ -0,0 +1,298 @@
+/**
+ * dsl-coverage-checker.ts — Verify that DSL covers all Spec requirements.
+ *
+ * Extracts User Stories and Functional Requirements from Spec markdown,
+ * then checks each against DSL endpoints/models/behaviors using keyword
+ * matching. Uncovered requirements are reported as DslGap entries.
+ */
+
+import { SpecDSL } from "./dsl-types";
+
+// ─── Types ──────────────────────────────────────────────────────────────────────
+
+export interface SpecRequirement {
+  id: string;
+  text: string;
+  section: "user_story" | "functional_req" | "boundary_condition";
+}
+
+export interface CoverageResult {
+  covered: SpecRequirement[];
+  uncovered: SpecRequirement[];
+  coverageRatio: number;
+}
+
+// ─── Keyword Extraction ─────────────────────────────────────────────────────────
+
+/** CJK character range regex. */
+const CJK_RANGE = /[\u4e00-\u9fff\u3400-\u4dbf]/g;
+
+/** Common stopwords to ignore (Chinese + English). */
+const STOPWORDS = new Set([
+  // Chinese
+  "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一",
+  "一个", "上", "也", "到", "说", "要", "去", "你", "会", "着", "没有",
+  "看", "好", "自己", "这", "他", "她", "它", "我们", "可以", "能", "能够",
+  "需要", "应该", "作为", "希望", "以便", "通过", "使用", "进行", "支持",
+  "包括", "提供", "实现", "系统", "功能", "用户",
+  // English
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "can", "shall", "to", "of", "in", "for",
+  "on", "with", "at", "by", "from", "as", "into", "through", "during",
+  "before", "after", "and", "or", "but", "not", "no", "if", "then",
+  "than", "so", "that", "this", "these", "those", "it", "its",
+  "i", "we", "you", "they", "he", "she", "my", "our", "your",
+  "able", "want", "need", "use", "make", "get", "set",
+]);
+
+/**
+ * Extract meaningful keywords from text (handles mixed CJK + English).
+ * CJK: split into individual characters and bigrams.
+ * English: split by non-alpha, filter stopwords, lowercase.
+ */
+export function extractKeywords(text: string): Set<string> {
+  const keywords = new Set<string>();
+
+  // Extract CJK characters and form bigrams
+  const cjkChars = text.match(CJK_RANGE) ?? [];
+  for (const ch of cjkChars) {
+    if (!STOPWORDS.has(ch)) keywords.add(ch);
+  }
+  for (let i = 0; i < cjkChars.length - 1; i++) {
+    const bigram = cjkChars[i] + cjkChars[i + 1];
+    if (!STOPWORDS.has(bigram)) keywords.add(bigram);
+  }
+
+  // Extract English words
+  const englishWords = text
+    .replace(CJK_RANGE, " ")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((w) => w.length >= 2 && !STOPWORDS.has(w));
+  for (const w of englishWords) keywords.add(w);
+
+  return keywords;
+}
+
+// ─── Spec Requirement Extraction ────────────────────────────────────────────────
+
+/**
+ * Parse User Stories from Spec markdown.
+ * Matches patterns like: "作为 **角色**，我希望 **动作**，以便 **目的**"
+ * and English "As a **role**, I want **action**, so that **purpose**"
+ */
+function extractUserStories(spec: string): SpecRequirement[] {
+  const reqs: SpecRequirement[] = [];
+  const lines = spec.split("\n");
+  let storyIdx = 0;
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    // Chinese format: "- 作为 ..." or "1. 作为 ..." or "作为 ..."
+    if (/^[-*]\s+作为\s/.test(trimmed) || /^\d+[.)]\s*作为\s/.test(trimmed) || /^作为\s/.test(trimmed)) {
+      storyIdx++;
+      reqs.push({
+        id: `US-${storyIdx}`,
+        text: trimmed.replace(/^[-*]\s+/, "").replace(/^\d+[.)]\s*/, ""),
+        section: "user_story",
+      });
+      continue;
+    }
+    // English format: "- As a ..." or "1. As a ..." or "As a ..."
+    if (/^[-*]\s+As an?\s/i.test(trimmed) || /^\d+[.)]\s*As an?\s/i.test(trimmed) || /^As an?\s/i.test(trimmed)) {
+      storyIdx++;
+      reqs.push({
+        id: `US-${storyIdx}`,
+        text: trimmed.replace(/^[-*]\s+/, "").replace(/^\d+[.)]\s*/, ""),
+        section: "user_story",
+      });
+    }
+  }
+
+  return reqs;
+}
+
+/**
+ * Parse Functional Requirements from Spec markdown.
+ * Matches checklist items: "- [ ] requirement text" and numbered items under §4.
+ */
+function extractFunctionalReqs(spec: string): SpecRequirement[] {
+  const reqs: SpecRequirement[] = [];
+  let inSection4 = false;
+  let reqIdx = 0;
+  const lines = spec.split("\n");
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    // Detect section 4 heading (functional requirements)
+    if (/^#{1,3}\s*(4\.|四|功能需求|Functional\s+Req)/i.test(trimmed)) {
+      inSection4 = true;
+      continue;
+    }
+    // Next section heading exits section 4
+    if (inSection4 && /^#{1,3}\s*(\d+\.|五|六|七|八|九|API|Data|Non-Func)/i.test(trimmed)) {
+      inSection4 = false;
+      continue;
+    }
+
+    if (!inSection4) continue;
+
+    // Checklist items: - [ ] or - [x]
+    const checklistMatch = trimmed.match(/^-\s*\[[ x]\]\s*(.+)/i);
+    if (checklistMatch) {
+      reqIdx++;
+      reqs.push({
+        id: `FR-${reqIdx}`,
+        text: checklistMatch[1],
+        section: "functional_req",
+      });
+      continue;
+    }
+
+    // Numbered sub-items: 4.1.1, 4.2.3, etc.
+    const numberedMatch = trimmed.match(/^(\d+\.)+\d*\s+(.+)/);
+    if (numberedMatch) {
+      reqIdx++;
+      reqs.push({
+        id: `FR-${reqIdx}`,
+        text: numberedMatch[2],
+        section: "functional_req",
+      });
+    }
+  }
+
+  return reqs;
+}
+
+/**
+ * Parse Boundary Conditions from Spec markdown (section 4.2 or edge cases).
+ */
+function extractBoundaryConditions(spec: string): SpecRequirement[] {
+  const reqs: SpecRequirement[] = [];
+  let inBoundary = false;
+  let bcIdx = 0;
+  const lines = spec.split("\n");
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+
+    if (/边界|boundary|edge\s+case|异常|错误处理/i.test(trimmed) && /^#{1,4}/.test(trimmed)) {
+      inBoundary = true;
+      continue;
+    }
+    if (inBoundary && /^#{1,3}\s/.test(trimmed)) {
+      inBoundary = false;
+      continue;
+    }
+
+    if (!inBoundary) continue;
+
+    const itemMatch = trimmed.match(/^[-*]\s+(.+)/) || trimmed.match(/^\d+[.)]\s*(.+)/);
+    if (itemMatch && itemMatch[1].length > 5) {
+      bcIdx++;
+      reqs.push({
+        id: `BC-${bcIdx}`,
+        text: itemMatch[1],
+        section: "boundary_condition",
+      });
+    }
+  }
+
+  return reqs;
+}
+
+// ─── Public API ──────────────────────────────────────────────────────────────────
+
+/**
+ * Extract all requirements from a Spec markdown document.
+ */
+export function extractSpecRequirements(spec: string): SpecRequirement[] {
+  return [
+    ...extractUserStories(spec),
+    ...extractFunctionalReqs(spec),
+    ...extractBoundaryConditions(spec),
+  ];
+}
+
+/**
+ * Build a keyword index from all DSL elements for fast matching.
+ */
+function buildDslKeywordIndex(dsl: SpecDSL): Set<string> {
+  const allText: string[] = [];
+
+  // Feature
+  allText.push(dsl.feature.title, dsl.feature.description);
+
+  // Models
+  for (const m of dsl.models) {
+    allText.push(m.name, m.description ?? "");
+    for (const f of m.fields) allText.push(f.name, f.description ?? "");
+    for (const r of m.relations ?? []) allText.push(r);
+  }
+
+  // Endpoints
+  for (const ep of dsl.endpoints) {
+    allText.push(ep.description, ep.path);
+    if (ep.request?.body) allText.push(...Object.keys(ep.request.body));
+    if (ep.request?.query) allText.push(...Object.keys(ep.request.query));
+    for (const err of ep.errors ?? []) allText.push(err.code, err.description);
+  }
+
+  // Behaviors
+  for (const b of dsl.behaviors) {
+    allText.push(b.description, b.trigger ?? "");
+    for (const c of b.constraints ?? []) allText.push(c);
+  }
+
+  // Components
+  for (const c of dsl.components ?? []) {
+    allText.push(c.name, c.description);
+    for (const p of c.props) allText.push(p.name, p.description ?? "");
+    for (const e of c.events) allText.push(e.name, e.payload ?? "");
+  }
+
+  return extractKeywords(allText.join(" "));
+}
+
+/** Minimum keyword overlap to consider a requirement "covered". */
+const MIN_KEYWORD_OVERLAP = 2;
+
+/**
+ * Check how well the DSL covers the Spec requirements.
+ * Uses keyword overlap: a requirement is "covered" if it shares
+ * ≥ MIN_KEYWORD_OVERLAP significant keywords with any DSL element.
+ */
+export function checkDslCoverage(
+  requirements: SpecRequirement[],
+  dsl: SpecDSL
+): CoverageResult {
+  if (requirements.length === 0) {
+    return { covered: [], uncovered: [], coverageRatio: 1.0 };
+  }
+
+  const dslKeywords = buildDslKeywordIndex(dsl);
+  const covered: SpecRequirement[] = [];
+  const uncovered: SpecRequirement[] = [];
+
+  for (const req of requirements) {
+    const reqKeywords = extractKeywords(req.text);
+    let overlap = 0;
+    for (const kw of reqKeywords) {
+      if (dslKeywords.has(kw)) overlap++;
+    }
+
+    if (overlap >= MIN_KEYWORD_OVERLAP) {
+      covered.push(req);
+    } else {
+      uncovered.push(req);
+    }
+  }
+
+  return {
+    covered,
+    uncovered,
+    coverageRatio: covered.length / requirements.length,
+  };
+}

package/core/dsl-extractor.ts

@@ -11,6 +11,8 @@ import {
   buildDslExtractionPrompt,
   buildDslRetryPrompt,
 } from "../prompts/dsl.prompt";
+import { estimateTokens, getDefaultBudget } from "./token-budget";
+import { parseJsonFromAiOutput } from "./safe-json";
 
 // ─── DSL Sanitizer ───────────────────────────────────────────────────────────
 
@@ -50,8 +52,8 @@ function sanitizeDsl(raw: unknown): unknown {
 /** Maximum AI attempts (1 initial + up to this many retries). */
 const MAX_RETRIES = 2;
 
-/** Maximum spec length passed to AI to avoid token/context blow-up. */
-const MAX_SPEC_CHARS = 12_000;
+/** Default maximum spec length passed to AI. Overridden by token budget when provider is known. */
+const DEFAULT_MAX_SPEC_CHARS = 12_000;
 
 // ─── DSL file naming ──────────────────────────────────────────────────────────
 
@@ -63,45 +65,8 @@ export function dslFilePath(specFilePath: string): string {
 
 // ─── Parser ───────────────────────────────────────────────────────────────────
 
-/**
- * Parse JSON from raw AI output.
- * Handles two cases:
- *   1. Bare JSON object starting with `{`
- *   2. JSON wrapped in a ```json ... ``` fence (model sometimes ignores instructions)
- *
- * Does NOT use eval or Function() — only JSON.parse().
- */
-function parseJsonFromOutput(raw: string): unknown {
-  const trimmed = raw.trim();
-
-  // Case 1: bare JSON
-  if (trimmed.startsWith("{")) {
-    return JSON.parse(trimmed);
-  }
-
-  // Case 2: fenced JSON  — extract content between first ``` and last ```
-  const fenceStart = trimmed.indexOf("```");
-  if (fenceStart !== -1) {
-    const afterFence = trimmed.slice(fenceStart + 3);
-    // Skip optional language tag (e.g. "json\n")
-    const newlinePos = afterFence.indexOf("\n");
-    const jsonStart = newlinePos !== -1 ? newlinePos + 1 : 0;
-    const fenceEnd = afterFence.lastIndexOf("```");
-    if (fenceEnd > jsonStart) {
-      const jsonStr = afterFence.slice(jsonStart, fenceEnd).trim();
-      return JSON.parse(jsonStr);
-    }
-  }
-
-  // Case 3: try to find the first `{` and last `}` pair
-  const objStart = trimmed.indexOf("{");
-  const objEnd = trimmed.lastIndexOf("}");
-  if (objStart !== -1 && objEnd > objStart) {
-    return JSON.parse(trimmed.slice(objStart, objEnd + 1));
-  }
-
-  throw new SyntaxError("No JSON object found in AI output");
-}
+// Uses shared parseJsonFromAiOutput from safe-json.ts
+const parseJsonFromOutput = parseJsonFromAiOutput;
 
 // ─── DslExtractor ────────────────────────────────────────────────────────────
 
@@ -125,12 +90,20 @@ export class DslExtractor {
     specContent: string,
     opts: { auto?: boolean; isFrontend?: boolean } = {}
   ): Promise<SpecDSL | null> {
+    // Compute dynamic spec char limit based on provider's token budget.
+    // Reserve ~30% of budget for DSL extraction prompt + response; use 70% for spec content.
+    const providerBudget = getDefaultBudget(this.provider.providerName);
+    const maxSpecChars = Math.max(
+      DEFAULT_MAX_SPEC_CHARS,
+      Math.floor(providerBudget * 0.7 * 3) // ~3 chars per token, 70% of budget
+    );
+
     // Truncate very long specs to avoid token issues
     const specForAI =
-      specContent.length > MAX_SPEC_CHARS
+      specContent.length > maxSpecChars
         ? (() => {
-            console.log(chalk.yellow(`  ⚠ Spec is ${specContent.length} chars — truncating to ${MAX_SPEC_CHARS} for DSL extraction. Details at the end may be lost.`));
-            return specContent.slice(0, MAX_SPEC_CHARS) + "\n... (truncated for DSL extraction)";
+            console.log(chalk.yellow(`  ⚠ Spec is ${specContent.length} chars — truncating to ${maxSpecChars} for DSL extraction (${this.provider.providerName} budget: ${Math.round(providerBudget / 1000)}K tokens).`));
+            return specContent.slice(0, maxSpecChars) + "\n... (truncated for DSL extraction)";
           })()
         : specContent;
 
@@ -170,8 +143,8 @@ export class DslExtractor {
         console.log(chalk.red(`  ✘ Failed to parse JSON from AI output: ${(parseErr as Error).message}`));
         const preview = rawOutput.slice(0, 500).replace(/\n/g, "\\n");
         console.log(chalk.gray(`  AI output preview (first 500 chars): ${preview}`));
-        if (rawOutput.length > MAX_SPEC_CHARS) {
-          console.log(chalk.gray(`  Note: spec was truncated to ${MAX_SPEC_CHARS} chars — long specs may lose context`));
+        if (rawOutput.length > maxSpecChars) {
+          console.log(chalk.gray(`  Note: spec was truncated to ${maxSpecChars} chars — long specs may lose context`));
         }
         lastErrors = [{ path: "root", message: "Output is not valid JSON — see raw output above" }];
 

package/core/dsl-feedback.ts

@@ -21,7 +21,7 @@ import { SpecDSL } from "./dsl-types";
 
 export interface DslGap {
   /** Short machine key for RunLog serialisation */
-  code: "sparse_model" | "missing_errors" | "generic_endpoint_desc" | "no_models_no_endpoints";
+  code: "sparse_model" | "missing_errors" | "generic_endpoint_desc" | "no_models_no_endpoints" | "uncovered_requirement";
   /** Human-readable message shown to the user */
   message: string;
   /** Concrete suggestion injected into the refinement prompt */

package/core/dsl-validator.ts

@@ -119,6 +119,9 @@ export function validateDsl(raw: unknown): DslValidationResult {
     }
   }
 
+  // ── Cross-reference checks ──────────────────────────────────────────────
+  crossReferenceChecks(obj, errors);
+
   if (errors.length > 0) {
     return { valid: false, errors };
   }
@@ -428,6 +431,77 @@ function validateComponent(
   }
 }
 
+// ─── Cross-reference checks ──────────────────────────────────────────────────
+
+function crossReferenceChecks(
+  obj: Record<string, unknown>,
+  errors: DslValidationError[]
+): void {
+  const models = Array.isArray(obj["models"]) ? (obj["models"] as Record<string, unknown>[]) : [];
+  const endpoints = Array.isArray(obj["endpoints"]) ? (obj["endpoints"] as Record<string, unknown>[]) : [];
+  const components = Array.isArray(obj["components"]) ? (obj["components"] as Record<string, unknown>[]) : [];
+
+  // 1. Duplicate path+method detection
+  const seenRoutes = new Map<string, number>();
+  for (let i = 0; i < endpoints.length; i++) {
+    const ep = endpoints[i];
+    if (typeof ep?.["method"] === "string" && typeof ep?.["path"] === "string") {
+      const route = `${(ep["method"] as string).toUpperCase()} ${ep["path"]}`;
+      if (seenRoutes.has(route)) {
+        errors.push({
+          path: `endpoints[${i}]`,
+          message: `Duplicate route "${route}" — also defined at endpoints[${seenRoutes.get(route)}]`,
+        });
+      } else {
+        seenRoutes.set(route, i);
+      }
+    }
+  }
+
+  // 2. Model relations reference existing model names
+  const modelNames = new Set(
+    models.filter((m) => typeof m?.["name"] === "string").map((m) => m["name"] as string)
+  );
+  for (let i = 0; i < models.length; i++) {
+    const m = models[i];
+    if (!Array.isArray(m?.["relations"])) continue;
+    for (const rel of m["relations"] as string[]) {
+      if (typeof rel !== "string") continue;
+      // Extract referenced model name: "User hasMany Post" → "Post", "belongsTo Category" → "Category"
+      const refMatch = rel.match(/(?:hasMany|hasOne|belongsTo|manyToMany)\s+(\w+)/i);
+      if (refMatch) {
+        const refName = refMatch[1];
+        if (!modelNames.has(refName)) {
+          errors.push({
+            path: `models[${i}].relations`,
+            message: `Relation references model "${refName}" which is not defined in models[]`,
+          });
+        }
+      }
+    }
+  }
+
+  // 3. Component apiCalls reference existing endpoint IDs
+  const endpointIds = new Set(
+    endpoints.filter((e) => typeof e?.["id"] === "string").map((e) => e["id"] as string)
+  );
+  if (endpointIds.size > 0) {
+    for (let i = 0; i < components.length; i++) {
+      const c = components[i];
+      if (!Array.isArray(c?.["apiCalls"])) continue;
+      for (const call of c["apiCalls"] as string[]) {
+        if (typeof call !== "string") continue;
+        if (!endpointIds.has(call)) {
+          errors.push({
+            path: `components[${i}].apiCalls`,
+            message: `References endpoint "${call}" which is not defined in endpoints[]`,
+          });
+        }
+      }
+    }
+  }
+}
+
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 
 function requireNonEmptyString(