gsd-pi 2.28.0 → 2.29.0-dev.2ccf3fb

package/src/resources/extensions/gsd/auto-prompts.ts

@@ -21,6 +21,10 @@ import type { GSDPreferences } from "./preferences.js";
 import { join } from "node:path";
 import { existsSync } from "node:fs";
 import { computeBudgets, resolveExecutorContextWindow } from "./context-budget.js";
+import { compressToTarget } from "./prompt-compressor.js";
+import { distillSummaries } from "./summary-distiller.js";
+import { formatDecisionsCompact, formatRequirementsCompact } from "./structured-data-formatter.js";
+import { chunkByRelevance, formatChunks } from "./semantic-chunker.js";
 
 // ─── Executor Constraints ─────────────────────────────────────────────────────
 
@@ -81,6 +85,43 @@ export async function inlineFileOptional(
   return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`;
 }
 
+/**
+ * Smart file inlining — for large files, use semantic chunking to include
+ * only the most relevant portions based on the task context.
+ * Falls back to full content for small files or when no query is provided.
+ *
+ * @param absPath Absolute file path
+ * @param relPath Relative display path
+ * @param label Section label
+ * @param query Task description for relevance scoring (optional)
+ * @param threshold Character threshold for chunking (default: 3000)
+ */
+export async function inlineFileSmart(
+  absPath: string | null, relPath: string, label: string,
+  query?: string, threshold = 3000,
+): Promise<string> {
+  const content = absPath ? await loadFile(absPath) : null;
+  if (!content) {
+    return `### ${label}\nSource: \`${relPath}\`\n\n_(not found — file does not exist yet)_`;
+  }
+
+  // For small files or no query, include full content
+  if (content.length <= threshold || !query) {
+    return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`;
+  }
+
+  // Use semantic chunking for large files
+  const result = chunkByRelevance(content, query, { maxChunks: 5, minScore: 0.05 });
+
+  // If chunking didn't save much (< 20%), just include full content
+  if (result.savingsPercent < 20) {
+    return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`;
+  }
+
+  const formatted = formatChunks(result, relPath);
+  return `### ${label} (${result.omittedChunks} sections omitted for relevance)\nSource: \`${relPath}\`\n\n${formatted}`;
+}
+
 /**
  * Load and inline dependency slice summaries (full content, not just paths).
  */
@@ -111,8 +152,21 @@ export async function inlineDependencySummaries(
   }
 
   const result = sections.join("\n\n");
-  // When a budget is provided, truncate at section boundaries to fit
   if (budgetChars !== undefined && result.length > budgetChars) {
+    // For 3+ summaries, try distillation first (preserves more information)
+    if (sections.length >= 3) {
+      const rawSummaries = sections.map(s => {
+        // Extract content after the header line
+        const lines = s.split("\n");
+        const contentStart = lines.findIndex(l => l.startsWith("Source:"));
+        return contentStart >= 0 ? lines.slice(contentStart + 1).join("\n").trim() : s;
+      });
+      const distilled = distillSummaries(rawSummaries, budgetChars);
+      if (distilled.content.length <= budgetChars) {
+        return distilled.content;
+      }
+    }
+    // Fall back to section-boundary truncation
     const { truncateAtSectionBoundary } = await import("./context-budget.js");
     return truncateAtSectionBoundary(result, budgetChars).content;
   }
@@ -139,15 +193,19 @@ export async function inlineGsdRootFile(
  * Falls back to filesystem via inlineGsdRootFile when DB unavailable or empty.
  */
 export async function inlineDecisionsFromDb(
-  base: string, milestoneId?: string, scope?: string,
+  base: string, milestoneId?: string, scope?: string, level?: InlineLevel,
 ): Promise<string | null> {
+  const inlineLevel = level ?? resolveInlineLevel();
   try {
     const { isDbAvailable } = await import("./gsd-db.js");
     if (isDbAvailable()) {
       const { queryDecisions, formatDecisionsForPrompt } = await import("./context-store.js");
       const decisions = queryDecisions({ milestoneId, scope });
       if (decisions.length > 0) {
-        const formatted = formatDecisionsForPrompt(decisions);
+        // Use compact format for non-full levels to save ~35% tokens
+        const formatted = inlineLevel !== "full"
+          ? formatDecisionsCompact(decisions)
+          : formatDecisionsForPrompt(decisions);
         return `### Decisions\nSource: \`.gsd/DECISIONS.md\`\n\n${formatted}`;
       }
     }
@@ -162,15 +220,19 @@ export async function inlineDecisionsFromDb(
  * Falls back to filesystem via inlineGsdRootFile when DB unavailable or empty.
  */
 export async function inlineRequirementsFromDb(
-  base: string, sliceId?: string,
+  base: string, sliceId?: string, level?: InlineLevel,
 ): Promise<string | null> {
+  const inlineLevel = level ?? resolveInlineLevel();
   try {
     const { isDbAvailable } = await import("./gsd-db.js");
     if (isDbAvailable()) {
       const { queryRequirements, formatRequirementsForPrompt } = await import("./context-store.js");
       const requirements = queryRequirements({ sliceId });
       if (requirements.length > 0) {
-        const formatted = formatRequirementsForPrompt(requirements);
+        // Use compact format for non-full levels to save ~40% tokens
+        const formatted = inlineLevel !== "full"
+          ? formatRequirementsCompact(requirements)
+          : formatRequirementsForPrompt(requirements);
         return `### Requirements\nSource: \`.gsd/REQUIREMENTS.md\`\n\n${formatted}`;
       }
     }
@@ -407,6 +469,17 @@ export async function checkNeedsReassessment(
 
   if (hasAssessment) return null;
 
+  // Fallback: check the expected path directly via existsSync.
+  // resolveSliceFile relies on directory listing (readdirSync) which may not
+  // reflect a freshly written file in git worktree directories on some
+  // filesystems (observed on macOS APFS). A direct existsSync on the
+  // constructed path bypasses directory listing entirely. (#1112)
+  const sliceDir = resolveSlicePath(base, mid, lastCompleted.id);
+  if (sliceDir) {
+    const directPath = join(sliceDir, `${lastCompleted.id}-ASSESSMENT.md`);
+    if (existsSync(directPath)) return null;
+  }
+
   // Also need a summary to reassess against
   const summaryFile = resolveSliceFile(base, mid, lastCompleted.id, "SUMMARY");
   const hasSummary = !!(summaryFile && await loadFile(summaryFile));
@@ -519,9 +592,9 @@ export async function buildPlanMilestonePrompt(mid: string, midTitle: string, ba
   if (inlineLevel !== "minimal") {
     const projectInline = await inlineProjectFromDb(base);
     if (projectInline) inlined.push(projectInline);
-    const requirementsInline = await inlineRequirementsFromDb(base);
+    const requirementsInline = await inlineRequirementsFromDb(base, undefined, inlineLevel);
     if (requirementsInline) inlined.push(requirementsInline);
-    const decisionsInline = await inlineDecisionsFromDb(base, mid);
+    const decisionsInline = await inlineDecisionsFromDb(base, mid, undefined, inlineLevel);
     if (decisionsInline) inlined.push(decisionsInline);
   }
   const knowledgeInlinePM = await inlineGsdRootFile(base, "knowledge.md", "Project Knowledge");
@@ -614,9 +687,9 @@ export async function buildPlanSlicePrompt(
   const researchInline = await inlineFileOptional(researchPath, researchRel, "Slice Research");
   if (researchInline) inlined.push(researchInline);
   if (inlineLevel !== "minimal") {
-    const decisionsInline = await inlineDecisionsFromDb(base, mid);
+    const decisionsInline = await inlineDecisionsFromDb(base, mid, undefined, inlineLevel);
     if (decisionsInline) inlined.push(decisionsInline);
-    const requirementsInline = await inlineRequirementsFromDb(base, sid);
+    const requirementsInline = await inlineRequirementsFromDb(base, sid, inlineLevel);
     if (requirementsInline) inlined.push(requirementsInline);
   }
   const knowledgeInlinePS = await inlineGsdRootFile(base, "knowledge.md", "Project Knowledge");
@@ -640,7 +713,7 @@ export async function buildPlanSlicePrompt(
   const prefs = loadEffectiveGSDPreferences();
   const commitDocsEnabled = prefs?.preferences?.git?.commit_docs !== false;
   const commitInstruction = commitDocsEnabled
-    ? `Commit: \`docs(${sid}): add slice plan\``
+    ? `Commit the plan files only: \`git add ${relSlicePath(base, mid, sid)}/ .gsd/DECISIONS.md .gitignore && git commit -m "docs(${sid}): add slice plan"\`. Do not stage .gsd/STATE.md or other runtime files — the system manages those.`
     : "Do not commit — planning docs are not tracked in git for this project.";
   return loadPrompt("plan-slice", {
     workingDirectory: base,
@@ -706,15 +779,25 @@ export async function buildExecuteTaskPrompt(
     : priorSummaries;
   const carryForwardSection = await buildCarryForwardSection(effectivePriorSummaries, base);
 
-  // Inline project knowledge if available
-  const knowledgeInlineET = await inlineGsdRootFile(base, "knowledge.md", "Project Knowledge");
+  // Inline project knowledge if available (smart-chunked for relevance)
+  const knowledgeAbsPath = resolveGsdRootFile(base, "KNOWLEDGE");
+  const knowledgeInlineET = existsSync(knowledgeAbsPath)
+    ? await inlineFileSmart(
+        knowledgeAbsPath,
+        relGsdRootFile("KNOWLEDGE"),
+        "Project Knowledge",
+        `${tTitle} ${sTitle}`,  // use task + slice title as relevance query
+      )
+    : null;
+  // Only include if it has content (not a "not found" result)
+  const knowledgeContent = knowledgeInlineET && !knowledgeInlineET.includes("not found") ? knowledgeInlineET : null;
 
   const inlinedTemplates = inlineLevel === "minimal"
     ? inlineTemplate("task-summary", "Task Summary")
     : [
         inlineTemplate("task-summary", "Task Summary"),
         inlineTemplate("decisions", "Decisions"),
-        ...(knowledgeInlineET ? [knowledgeInlineET] : []),
+        ...(knowledgeContent ? [knowledgeContent] : []),
       ].join("\n\n---\n\n");
 
   const taskSummaryPath = join(base, `${relSlicePath(base, mid, sid)}/tasks/${tid}-SUMMARY.md`);
@@ -728,6 +811,17 @@ export async function buildExecuteTaskPrompt(
   const budgets = computeBudgets(contextWindow);
   const verificationBudget = `~${Math.round(budgets.verificationBudgetChars / 1000)}K chars`;
 
+  // Compress carry-forward section when it exceeds 40% of inline context budget.
+  // Only compress when compression_strategy is "compress" (budget/balanced profiles).
+  const carryForwardBudget = Math.floor(budgets.inlineContextBudgetChars * 0.4);
+  let finalCarryForward = carryForwardSection;
+  if (carryForwardSection.length > carryForwardBudget) {
+    const { resolveCompressionStrategy } = await import("./preferences.js");
+    if (resolveCompressionStrategy() === "compress") {
+      finalCarryForward = compressToTarget(carryForwardSection, carryForwardBudget).content;
+    }
+  }
+
   return loadPrompt("execute-task", {
     overridesSection,
     workingDirectory: base,
@@ -737,7 +831,7 @@ export async function buildExecuteTaskPrompt(
     taskPlanPath: taskPlanRelPath,
     taskPlanInline,
     slicePlanExcerpt,
-    carryForwardSection,
+    carryForwardSection: finalCarryForward,
     resumeSection,
     priorTaskLines: priorLines,
     taskSummaryPath,
@@ -760,7 +854,7 @@ export async function buildCompleteSlicePrompt(
   inlined.push(await inlineFile(roadmapPath, roadmapRel, "Milestone Roadmap"));
   inlined.push(await inlineFile(slicePlanPath, slicePlanRel, "Slice Plan"));
   if (inlineLevel !== "minimal") {
-    const requirementsInline = await inlineRequirementsFromDb(base, sid);
+    const requirementsInline = await inlineRequirementsFromDb(base, sid, inlineLevel);
     if (requirementsInline) inlined.push(requirementsInline);
   }
   const knowledgeInlineCS = await inlineGsdRootFile(base, "knowledge.md", "Project Knowledge");
@@ -831,9 +925,9 @@ export async function buildCompleteMilestonePrompt(
 
   // Inline root GSD files (skip for minimal — completion can read these if needed)
   if (inlineLevel !== "minimal") {
-    const requirementsInline = await inlineRequirementsFromDb(base);
+    const requirementsInline = await inlineRequirementsFromDb(base, undefined, inlineLevel);
     if (requirementsInline) inlined.push(requirementsInline);
-    const decisionsInline = await inlineDecisionsFromDb(base, mid);
+    const decisionsInline = await inlineDecisionsFromDb(base, mid, undefined, inlineLevel);
     if (decisionsInline) inlined.push(decisionsInline);
     const projectInline = await inlineProjectFromDb(base);
     if (projectInline) inlined.push(projectInline);
@@ -903,9 +997,9 @@ export async function buildValidateMilestonePrompt(
 
   // Inline root GSD files
   if (inlineLevel !== "minimal") {
-    const requirementsInline = await inlineRequirementsFromDb(base);
+    const requirementsInline = await inlineRequirementsFromDb(base, undefined, inlineLevel);
     if (requirementsInline) inlined.push(requirementsInline);
-    const decisionsInline = await inlineDecisionsFromDb(base, mid);
+    const decisionsInline = await inlineDecisionsFromDb(base, mid, undefined, inlineLevel);
     if (decisionsInline) inlined.push(decisionsInline);
     const projectInline = await inlineProjectFromDb(base);
     if (projectInline) inlined.push(projectInline);
@@ -1051,9 +1145,9 @@ export async function buildReassessRoadmapPrompt(
   if (inlineLevel !== "minimal") {
     const projectInline = await inlineProjectFromDb(base);
     if (projectInline) inlined.push(projectInline);
-    const requirementsInline = await inlineRequirementsFromDb(base);
+    const requirementsInline = await inlineRequirementsFromDb(base, undefined, inlineLevel);
     if (requirementsInline) inlined.push(requirementsInline);
-    const decisionsInline = await inlineDecisionsFromDb(base, mid);
+    const decisionsInline = await inlineDecisionsFromDb(base, mid, undefined, inlineLevel);
     if (decisionsInline) inlined.push(decisionsInline);
   }
   const knowledgeInlineRA = await inlineGsdRootFile(base, "knowledge.md", "Project Knowledge");
@@ -1080,7 +1174,7 @@ export async function buildReassessRoadmapPrompt(
   const reassessPrefs = loadEffectiveGSDPreferences();
   const reassessCommitDocsEnabled = reassessPrefs?.preferences?.git?.commit_docs !== false;
   const reassessCommitInstruction = reassessCommitDocsEnabled
-    ? `Commit: \`docs(${mid}): reassess roadmap after ${completedSliceId}\``
+    ? `Commit: \`docs(${mid}): reassess roadmap after ${completedSliceId}\`. Stage only the .gsd/milestones/ files you changed — do not stage .gsd/STATE.md or other runtime files.`
     : "Do not commit — planning docs are not tracked in git for this project.";
 
   return loadPrompt("reassess-roadmap", {

package/src/resources/extensions/gsd/auto-recovery.ts

@@ -36,7 +36,10 @@ import {
   clearPathCache,
   resolveGsdRootFile,
 } from "./paths.js";
-import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, renameSync } from "node:fs";
+import { isValidationTerminal } from "./state.js";
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync } from "node:fs";
+import { atomicWriteSync } from "./atomic-write.js";
+import { loadJsonFileOrNull } from "./json-persistence.js";
 import { dirname, join } from "node:path";
 
 // ─── Artifact Resolution & Verification ───────────────────────────────────────
@@ -136,6 +139,21 @@ export function verifyExpectedArtifact(unitType: string, unitId: string, base: s
   if (!absPath) return false;
   if (!existsSync(absPath)) return false;
 
+  // validate-milestone must have a VALIDATION file with a terminal verdict
+  // (pass, needs-attention, or needs-remediation). Without this check, a
+  // VALIDATION file with missing/malformed frontmatter or an unrecognized
+  // verdict is treated as "complete" by the artifact check but deriveState
+  // still returns phase:"validating-milestone" (because isValidationTerminal
+  // returns false), creating an infinite skip loop that hits the lifetime cap.
+  if (unitType === "validate-milestone") {
+    try {
+      const validationContent = readFileSync(absPath, "utf-8");
+      if (!isValidationTerminal(validationContent)) return false;
+    } catch {
+      return false;
+    }
+  }
+
   // plan-slice must produce a plan with actual task entries, not just a scaffold.
   // The plan file may exist from a prior discussion/context step with only headings
   // but no tasks. Without this check the artifact is considered "complete" and the
@@ -210,7 +228,7 @@ export function verifyExpectedArtifact(unitType: string, unitId: string, base: s
         try {
           const roadmapContent = readFileSync(roadmapFile, "utf-8");
           const roadmap = parseRoadmap(roadmapContent);
-          const slice = roadmap.slices.find(s => s.id === sid);
+          const slice = (roadmap.slices ?? []).find(s => s.id === sid);
           if (slice && !slice.done) return false;
         } catch {
           // Corrupt/unparseable roadmap — fail verification so the unit
@@ -337,6 +355,10 @@ export function skipExecuteTask(
 
 // ─── Disk-backed completed-unit helpers ───────────────────────────────────────
 
+function isStringArray(data: unknown): data is string[] {
+  return Array.isArray(data) && data.every(item => typeof item === "string");
+}
+
 /** Path to the persisted completed-unit keys file. */
 export function completedKeysPath(base: string): string {
   return join(base, ".gsd", "completed-units.json");
@@ -345,49 +367,32 @@ export function completedKeysPath(base: string): string {
 /** Write a completed unit key to disk (read-modify-write append to set). */
 export function persistCompletedKey(base: string, key: string): void {
   const file = completedKeysPath(base);
-  let keys: string[] = [];
-  try {
-    if (existsSync(file)) {
-      keys = JSON.parse(readFileSync(file, "utf-8"));
-    }
-  } catch (e) { /* corrupt file — start fresh */ void e; }
+  const keys = loadJsonFileOrNull(file, isStringArray) ?? [];
   const keySet = new Set(keys);
   if (!keySet.has(key)) {
     keys.push(key);
-    // Atomic write: tmp file + rename prevents partial writes on crash
-    const tmpFile = file + ".tmp";
-    writeFileSync(tmpFile, JSON.stringify(keys), "utf-8");
-    renameSync(tmpFile, file);
+    atomicWriteSync(file, JSON.stringify(keys));
   }
 }
 
 /** Remove a stale completed unit key from disk. */
 export function removePersistedKey(base: string, key: string): void {
   const file = completedKeysPath(base);
-  try {
-    if (existsSync(file)) {
-      const keys: string[] = JSON.parse(readFileSync(file, "utf-8"));
-      const filtered = keys.filter(k => k !== key);
-      // Only write if the key was actually present
-      if (filtered.length !== keys.length) {
-        // Atomic write: tmp file + rename prevents partial writes on crash
-        const tmpFile = file + ".tmp";
-        writeFileSync(tmpFile, JSON.stringify(filtered), "utf-8");
-        renameSync(tmpFile, file);
-      }
-    }
-  } catch (e) { /* non-fatal: removePersistedKey failure */ void e; }
+  const keys = loadJsonFileOrNull(file, isStringArray);
+  if (!keys) return;
+  const filtered = keys.filter(k => k !== key);
+  if (filtered.length !== keys.length) {
+    atomicWriteSync(file, JSON.stringify(filtered));
+  }
 }
 
 /** Load all completed unit keys from disk into the in-memory set. */
 export function loadPersistedKeys(base: string, target: Set<string>): void {
   const file = completedKeysPath(base);
-  try {
-    if (existsSync(file)) {
-      const keys: string[] = JSON.parse(readFileSync(file, "utf-8"));
-      for (const k of keys) target.add(k);
-    }
-  } catch (e) { /* non-fatal: loadPersistedKeys failure */ void e; }
+  const keys = loadJsonFileOrNull(file, isStringArray);
+  if (keys) {
+    for (const k of keys) target.add(k);
+  }
 }
 
 // ─── Merge State Reconciliation ───────────────────────────────────────────────