@opengsd/gsd-pi 1.1.1-dev.9f86580 → 1.1.1-dev.b2556262

package/src/resources/extensions/gsd/source-observations.ts

@@ -0,0 +1,402 @@
+// Project/App: gsd-pi
+// File Purpose: Active-unit source observations for provider payload context.
+
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+
+import type { TaskRow } from "./db-task-slice-rows.js";
+import { extractPlanningPathReference, normalizeFilePath } from "./pre-execution-checks.js";
+
+export const WHOLE_FILE_OBSERVATION_MAX_BYTES = 50 * 1024;
+export const WHOLE_FILE_OBSERVATION_MAX_LINES = 2000;
+
+export type SourceObservationStatus =
+  | "whole"
+  | "missing"
+  | "binary/image"
+  | "over-threshold"
+  | "glob"
+  | "directory"
+  | "unresolved selector";
+
+export interface SourceObservationUnit {
+  unitType: string;
+  unitId: string;
+  startedAt: number;
+  basePath: string;
+}
+
+export type SourceObservationSource = "plan" | "read" | "mutation";
+
+export interface SourceObservation {
+  path: string;
+  absolutePath: string | null;
+  status: SourceObservationStatus;
+  source: SourceObservationSource;
+  text?: string;
+  bytes?: number;
+  lines?: number;
+  reason?: string;
+}
+
+interface ActiveSourceObservationSet {
+  unit: SourceObservationUnit;
+  observations: Map<string, SourceObservation>;
+}
+
+const SOURCE_CONTEXT_TITLE = "## Source Context Block";
+const SOURCE_OBSERVATION_UNIT_TYPE = "execute-task";
+
+export function supportsSourceObservationsForUnit(unitType: string): boolean {
+  return unitType === SOURCE_OBSERVATION_UNIT_TYPE;
+}
+
+export class SourceObservationStore {
+  private active: ActiveSourceObservationSet | null = null;
+
+  beginUnit(unit: SourceObservationUnit): void {
+    if (!supportsSourceObservationsForUnit(unit.unitType)) {
+      this.clear();
+      return;
+    }
+    if (this.matches(unit)) return;
+    this.active = { unit: { ...unit }, observations: new Map() };
+  }
+
+  clear(): void {
+    this.active = null;
+  }
+
+  degradeUnit(unit: Pick<SourceObservationUnit, "unitType" | "unitId" | "startedAt">): void {
+    if (!this.active) return;
+    const current = this.active.unit;
+    if (
+      current.unitType === unit.unitType &&
+      current.unitId === unit.unitId &&
+      current.startedAt === unit.startedAt
+    ) {
+      this.active = null;
+    }
+  }
+
+  observePlanTask(task: TaskRow): void {
+    if (!this.active) return;
+    for (const entry of planDeclaredSourceEntries(task)) {
+      this.observePath(entry.path, "plan");
+    }
+  }
+
+  observeRead(input: { path?: unknown; file_path?: unknown; [key: string]: unknown }): void {
+    if (!this.active) return;
+    const rawPath = readPathFromInput(input);
+    if (!rawPath.trim()) return;
+    this.observePath(rawPath, "read");
+  }
+
+  observeMutation(input: { path?: unknown; file_path?: unknown; [key: string]: unknown }): void {
+    if (!this.active) return;
+    const rawPath = readPathFromInput(input);
+    if (!rawPath.trim()) return;
+    this.observePath(rawPath, "mutation", { replaceExisting: true });
+  }
+
+  renderActiveBlock(): string | null {
+    if (!this.active || this.active.observations.size === 0) return null;
+    if (!supportsSourceObservationsForUnit(this.active.unit.unitType)) return null;
+    const observations = [...this.active.observations.values()]
+      .sort((a, b) => a.path.localeCompare(b.path));
+    const lines = [
+      SOURCE_CONTEXT_TITLE,
+      `Active Unit: ${this.active.unit.unitType} ${this.active.unit.unitId}`,
+      "",
+      "The files below are protected active-Unit source context. " +
+        "Use this block instead of rereading small files just to recover line windows.",
+    ];
+
+    const wholeFiles = observations.filter((observation) => observation.status === "whole");
+    const unavailable = observations.filter((observation) => observation.status !== "whole");
+
+    if (wholeFiles.length > 0) {
+      lines.push("", "### Whole-File Observations");
+      for (const observation of wholeFiles) {
+        lines.push(
+          "",
+          `#### ${observation.path}`,
+          `Status: whole-file (${observation.lines ?? 0} lines, ${formatSize(observation.bytes ?? 0)})`,
+          fencedSource(observation.text ?? ""),
+        );
+      }
+    }
+
+    if (unavailable.length > 0) {
+      lines.push("", "### Unavailable Source Observations");
+      for (const observation of unavailable) {
+        const detail = observation.reason ? ` - ${observation.reason}` : "";
+        lines.push(`- ${observation.path}: ${observation.status}${detail}`);
+      }
+    }
+
+    return lines.join("\n");
+  }
+
+  private matches(unit: SourceObservationUnit): boolean {
+    if (!this.active) return false;
+    const current = this.active.unit;
+    return current.unitType === unit.unitType &&
+      current.unitId === unit.unitId &&
+      current.startedAt === unit.startedAt &&
+      current.basePath === unit.basePath;
+  }
+
+  private observePath(
+    rawPath: string,
+    source: SourceObservationSource,
+    options: { replaceExisting?: boolean } = {},
+  ): void {
+    if (!this.active) return;
+    const observation = observeSourcePath(this.active.unit.basePath, rawPath, source);
+    const key = observation.absolutePath ?? `${observation.status}:${observation.path}`;
+    const existing = this.active.observations.get(key);
+    if (options.replaceExisting || !existing || observation.status === "whole" || existing.status !== "whole") {
+      this.active.observations.set(key, observation);
+    }
+  }
+}
+
+export function planDeclaredSourceEntries(
+  task: TaskRow,
+): Array<{ path: string; field: "files" | "inputs" }> {
+  const entries: Array<{ path: string; field: "files" | "inputs" }> = [];
+  for (const file of task.files) {
+    const path = extractPlanningPathReference(file) ?? file.trim();
+    if (path) entries.push({ path, field: "files" });
+  }
+  for (const input of task.inputs) {
+    const path = extractPlanningPathReference(input);
+    if (path) entries.push({ path, field: "inputs" });
+  }
+  return entries;
+}
+
+export function observeSourcePath(
+  basePath: string,
+  rawPath: string,
+  source: SourceObservationSource,
+): SourceObservation {
+  const normalizedRaw = normalizeFilePath(rawPath.trim());
+  const displayPath = normalizedRaw || rawPath.trim();
+  if (!normalizedRaw) {
+    return unavailable(displayPath || "<empty>", null, "unresolved selector", source);
+  }
+  if (containsGlobPattern(normalizedRaw)) {
+    return unavailable(displayPath, null, "glob", source, "glob selectors are not whole files");
+  }
+  if (rawPath.trim().endsWith("/")) {
+    return unavailable(displayPath, null, "directory", source, "directory selectors are not whole files");
+  }
+
+  const rootPath = resolve(basePath);
+  const absolutePath = isAbsolute(normalizedRaw) ? resolve(normalizedRaw) : resolve(rootPath, normalizedRaw);
+  const path = formatObservationPath(rootPath, absolutePath, displayPath);
+
+  if (!isPathInsideRoot(rootPath, absolutePath)) {
+    return unavailable(displayPath, absolutePath, "unresolved selector", source, "path is outside active Unit root");
+  }
+
+  if (!existsSync(absolutePath)) {
+    return unavailable(path, absolutePath, "missing", source, "file does not exist in the active Unit root");
+  }
+
+  let stat;
+  try {
+    stat = statSync(absolutePath);
+  } catch {
+    return unavailable(path, absolutePath, "missing", source, "file could not be inspected");
+  }
+
+  if (stat.isDirectory()) {
+    return unavailable(path, absolutePath, "directory", source, "directory selectors are not whole files");
+  }
+  if (!stat.isFile()) {
+    return unavailable(path, absolutePath, "unresolved selector", source, "path is not a regular file");
+  }
+  if (stat.size > WHOLE_FILE_OBSERVATION_MAX_BYTES) {
+    return unavailable(
+      path,
+      absolutePath,
+      "over-threshold",
+      source,
+      `${formatSize(stat.size)} exceeds ${formatSize(WHOLE_FILE_OBSERVATION_MAX_BYTES)}`,
+    );
+  }
+
+  let buffer: Buffer;
+  try {
+    buffer = readFileSync(absolutePath);
+  } catch {
+    return unavailable(path, absolutePath, "missing", source, "file could not be read");
+  }
+
+  if (isBinaryOrImage(buffer)) {
+    return unavailable(path, absolutePath, "binary/image", source, "binary or image files are not inlined as source text");
+  }
+
+  const text = buffer.toString("utf8");
+  const lines = countLines(text);
+  if (lines > WHOLE_FILE_OBSERVATION_MAX_LINES) {
+    return unavailable(
+      path,
+      absolutePath,
+      "over-threshold",
+      source,
+      `${lines} lines exceeds ${WHOLE_FILE_OBSERVATION_MAX_LINES}`,
+    );
+  }
+
+  return {
+    path,
+    absolutePath,
+    status: "whole",
+    source,
+    text,
+    bytes: buffer.byteLength,
+    lines,
+  };
+}
+
+export function injectSourceContextBlockIntoPayload(
+  payload: Record<string, unknown>,
+  block: string,
+): Record<string, unknown> {
+  const messages = payload.messages;
+  if (Array.isArray(messages)) {
+    return {
+      ...payload,
+      messages: [
+        ...withoutExistingSourceContextMessages(messages),
+        { role: "user", content: [{ type: "text", text: block }] },
+      ],
+    };
+  }
+
+  const input = payload.input;
+  if (Array.isArray(input)) {
+    return {
+      ...payload,
+      input: [
+        ...withoutExistingSourceContextItems(input),
+        { role: "user", content: [{ type: "input_text", text: block }] },
+      ],
+    };
+  }
+
+  return payload;
+}
+
+function unavailable(
+  path: string,
+  absolutePath: string | null,
+  status: Exclude<SourceObservationStatus, "whole">,
+  source: SourceObservationSource,
+  reason?: string,
+): SourceObservation {
+  return { path, absolutePath, status, source, reason };
+}
+
+function formatObservationPath(basePath: string, absolutePath: string, fallback: string): string {
+  const rel = relative(basePath, absolutePath);
+  if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
+    return rel.split(sep).join("/");
+  }
+  return fallback;
+}
+
+function isPathInsideRoot(rootPath: string, absolutePath: string): boolean {
+  const rel = relative(rootPath, absolutePath);
+  return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
+}
+
+function containsGlobPattern(candidate: string): boolean {
+  return ["*", "?", "[", "]", "{", "}"].some((char) => candidate.includes(char));
+}
+
+function readPathFromInput(input: { path?: unknown; file_path?: unknown }): string {
+  if (typeof input.path === "string") return input.path;
+  if (typeof input.file_path === "string") return input.file_path;
+  return "";
+}
+
+function formatSize(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+
+function isBinaryOrImage(buffer: Buffer): boolean {
+  if (buffer.includes(0)) return true;
+  if (startsWith(buffer, [0xff, 0xd8, 0xff])) return true;
+  if (startsWith(buffer, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return true;
+  if (startsWithAscii(buffer, "GIF")) return true;
+  if (
+    startsWithAscii(buffer, "RIFF") &&
+    buffer.length >= 12 &&
+    buffer.subarray(8, 12).toString("ascii") === "WEBP"
+  ) return true;
+  return false;
+}
+
+function startsWith(buffer: Buffer, signature: readonly number[]): boolean {
+  if (buffer.length < signature.length) return false;
+  return signature.every((byte, index) => buffer[index] === byte);
+}
+
+function startsWithAscii(buffer: Buffer, text: string): boolean {
+  return buffer.length >= text.length && buffer.subarray(0, text.length).toString("ascii") === text;
+}
+
+function countLines(text: string): number {
+  if (text.length === 0) return 0;
+  return text.endsWith("\n") ? text.split("\n").length - 1 : text.split("\n").length;
+}
+
+function fencedSource(text: string): string {
+  const longest = longestBacktickRun(text);
+  const fence = longest >= 3 ? "`".repeat(longest + 1) : "```";
+  return `${fence}\n${text}\n${fence}`;
+}
+
+function longestBacktickRun(text: string): number {
+  let longest = 0;
+  let current = 0;
+  for (const char of text) {
+    if (char === "`") {
+      current += 1;
+      longest = Math.max(longest, current);
+    } else {
+      current = 0;
+    }
+  }
+  return longest;
+}
+
+function firstText(content: unknown): string | null {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return null;
+  const block = content.find((entry) =>
+    entry && typeof entry === "object" && "text" in entry,
+  ) as { text?: unknown } | undefined;
+  return typeof block?.text === "string" ? block.text : null;
+}
+
+function isSourceContextMessage(message: unknown): boolean {
+  if (!message || typeof message !== "object") return false;
+  return firstText((message as { content?: unknown }).content)?.startsWith(SOURCE_CONTEXT_TITLE) === true;
+}
+
+function withoutExistingSourceContextMessages(messages: unknown[]): unknown[] {
+  return messages.filter((message) => !isSourceContextMessage(message));
+}
+
+function withoutExistingSourceContextItems(items: unknown[]): unknown[] {
+  return items.filter((item) => !isSourceContextMessage(item));
+}

package/src/resources/extensions/gsd/state-reconciliation/drift/completion.ts

@@ -9,6 +9,7 @@
 import { existsSync, statSync } from "node:fs";
 
 import {
+  getAllMilestones,
   getMilestone,
   getMilestoneSlices,
   getSliceTasks,
@@ -41,17 +42,30 @@ function summaryMtimeIso(path: string): string | null {
 }
 
 export function detectMissingCompletionTimestampDrift(
-  state: GSDState,
+  _state: GSDState,
   ctx: DriftContext,
 ): CompletionTimestampDrift[] {
   if (!isDbAvailable()) return [];
-  const mid = state.activeMilestone?.id;
-  if (!mid) return [];
-
-  const milestone = getMilestone(mid);
-  if (!milestone) return [];
 
+  // Scan every milestone, not just the active one. Markdown artifacts exist on
+  // disk for all milestones, so a user can manually complete a queued/parked
+  // milestone (edit the roadmap + drop a SUMMARY) and leave completed_at=null
+  // in the DB. Gating on the active milestone left that drift unrepaired until
+  // the milestone happened to become active.
   const drifts: CompletionTimestampDrift[] = [];
+  for (const { id: mid } of getAllMilestones()) {
+    collectMilestoneCompletionDrift(mid, ctx, drifts);
+  }
+  return drifts;
+}
+
+function collectMilestoneCompletionDrift(
+  mid: string,
+  ctx: DriftContext,
+  drifts: CompletionTimestampDrift[],
+): void {
+  const milestone = getMilestone(mid);
+  if (!milestone) return;
 
   // Milestone-level
   if (
@@ -105,8 +119,6 @@ export function detectMissingCompletionTimestampDrift(
       }
     }
   }
-
-  return drifts;
 }
 
 export function repairMissingCompletionTimestamp(

package/src/resources/extensions/gsd/state-reconciliation/drift/stale-render.ts

@@ -61,6 +61,18 @@ function isRepairableStaleRenderReason(reason: string): boolean {
   );
 }
 
+function canonicalizeMilestoneId(dirSegment: string): string {
+  if (getMilestone(dirSegment)) return dirSegment;
+  const suffixId = dirSegment.match(/^(M\d+-[a-z0-9]{6})(?:$|-)/)?.[1];
+  if (suffixId && getMilestone(suffixId)) return suffixId;
+
+  // Descriptor layout: e.g. M001-DESCRIPTOR → M001
+  const baseId = dirSegment.match(/^(M\d+)(?:$|-)/i)?.[1];
+  if (baseId && getMilestone(baseId)) return baseId;
+
+  return suffixId ?? baseId ?? dirSegment;
+}
+
 function resolveRoadmapMilestoneIdFromPath(normPath: string): string {
   const milestoneMatch = normPath.match(/milestones\/([^/]+)\//);
   if (!milestoneMatch) {
@@ -106,7 +118,19 @@ async function repairStaleRenderFromBasePath(
         `stale-render drift: plan path missing milestone/slice segments: ${record.renderPath}`,
       );
     }
-    await renderPlanCheckboxes(basePath, pathMatch[1], pathMatch[2]);
+    const milestoneId = canonicalizeMilestoneId(pathMatch[1]);
+    const wrote = await renderPlanCheckboxes(
+      basePath,
+      milestoneId,
+      pathMatch[2],
+      record.renderPath,
+    );
+    if (!wrote) {
+      throw new Error(
+        `stale-render drift: plan re-render wrote nothing for ${milestoneId}/${pathMatch[2]} ` +
+          `(${record.renderPath}); slice has no tasks or its path is unresolvable`,
+      );
+    }
     return;
   }
 
@@ -120,7 +144,15 @@ async function repairStaleRenderFromBasePath(
         `stale-render drift: task summary path/reason malformed: ${record.renderPath} reason=${reason}`,
       );
     }
-    await renderTaskSummary(basePath, pathMatch[1], pathMatch[2], taskMatch[1]);
+    const milestoneId = canonicalizeMilestoneId(pathMatch[1]);
+    const wrote = await renderTaskSummary(basePath, milestoneId, pathMatch[2], taskMatch[1]);
+    if (!wrote) {
+      throw new Error(
+        `stale-render drift: task summary re-render wrote nothing for ` +
+          `${milestoneId}/${pathMatch[2]}/${taskMatch[1]} (${record.renderPath}); ` +
+          `task has no summary in DB or its slice path is unresolvable`,
+      );
+    }
     return;
   }
 
@@ -131,7 +163,7 @@ async function repairStaleRenderFromBasePath(
         `stale-render drift: slice summary path missing milestone/slice segments: ${record.renderPath}`,
       );
     }
-    const milestoneId = pathMatch[1];
+    const milestoneId = canonicalizeMilestoneId(pathMatch[1]);
     const sliceId = pathMatch[2];
     const slice = getSlice(milestoneId, sliceId);
     const uatPath = join(dirname(record.renderPath), buildSliceFileName(sliceId, "UAT"));
@@ -139,7 +171,14 @@ async function repairStaleRenderFromBasePath(
     if (slice?.full_uat_md && !existsSync(uatPath)) {
       setSliceSummaryMd(milestoneId, sliceId, slice.full_summary_md ?? "", "");
     }
-    await renderSliceSummary(basePath, milestoneId, sliceId);
+    const wrote = await renderSliceSummary(basePath, milestoneId, sliceId);
+    if (!wrote) {
+      throw new Error(
+        `stale-render drift: slice summary re-render wrote nothing for ` +
+          `${milestoneId}/${sliceId} (${record.renderPath}); slice has no summary/UAT ` +
+          `in DB or its path is unresolvable`,
+      );
+    }
     return;
   }
 
@@ -152,7 +191,7 @@ async function repairStaleRenderFromBasePath(
     }
     // When UAT.md is removed from disk, mirror that intent by clearing stale
     // persisted UAT content instead of rehydrating it back onto disk.
-    const milestoneId = pathMatch[1];
+    const milestoneId = canonicalizeMilestoneId(pathMatch[1]);
     const sliceId = pathMatch[2];
     const slice = getSlice(milestoneId, sliceId);
     if (!slice) {

package/src/resources/extensions/gsd/state-reconciliation/drift/stale-worker.ts

@@ -10,6 +10,11 @@ import {
   readSessionLockData,
   removeStaleSessionLock,
 } from "../../session-lock.js";
+import { clearStaleWorkerLock } from "../../crash-recovery.js";
+import { findStaleWorkerForProject } from "../../db/auto-workers.js";
+import { isDbAvailable } from "../../gsd-db.js";
+import { normalizeRealPath } from "../../paths.js";
+import { logWarning } from "../../workflow-logger.js";
 import type { GSDState } from "../../types.js";
 import type { DriftContext, DriftHandler, DriftRecord } from "../types.js";
 
@@ -20,23 +25,46 @@ export function detectStaleWorkerDrift(
   ctx: DriftContext,
 ): StaleWorkerDrift[] {
   const data = readSessionLockData(ctx.basePath);
-  if (!data) return [];
-  if (typeof data.pid !== "number") return [];
-  if (isSessionLockProcessAlive(data)) return [];
-
-  return [
-    {
-      kind: "stale-worker",
-      lockPath: effectiveLockFile(),
-      pid: data.pid,
-    },
-  ];
+  if (data && typeof data.pid === "number") {
+    return isSessionLockProcessAlive(data)
+      ? []
+      : [{ kind: "stale-worker", lockPath: effectiveLockFile(), pid: data.pid }];
+  }
+
+  // The lock file is missing or unparseable. It is not the only source of
+  // truth: a crashed worker can leave a workers row 'active' with held leases
+  // and in-flight dispatches even when its lock file is gone. Fall back to the
+  // DB worker registry so that state is still detected and repaired.
+  if (isDbAvailable()) {
+    try {
+      const stale = findStaleWorkerForProject(normalizeRealPath(ctx.basePath));
+      if (stale && typeof stale.pid === "number") {
+        return [{ kind: "stale-worker", lockPath: effectiveLockFile(), pid: stale.pid }];
+      }
+    } catch (err) {
+      // Best-effort: detection must never throw and abort the reconcile cycle.
+      logWarning(
+        "reconcile",
+        `stale-worker DB fallback detection failed: ${(err as Error).message}`,
+      );
+    }
+  }
+
+  return [];
 }
 
 export function repairStaleWorker(_record: StaleWorkerDrift, ctx: DriftContext): void {
   // removeStaleSessionLock is idempotent: it re-reads lock state and is a
   // no-op when the lock is held by an alive process. Safe under cap=2 retry.
   removeStaleSessionLock(ctx.basePath);
+
+  // Removing the lock file alone leaves the DB-side worker state dangling: the
+  // dead worker's milestone_leases stay 'held' and its unit_dispatches stay
+  // 'running'/'claimed', blocking new claims until the lease TTL expires.
+  // clearStaleWorkerLock cancels those dispatches, releases the leases, and
+  // marks the worker stopping — the same cleanup the startup crash-recovery
+  // path performs. It is DB-gated, idempotent, and best-effort.
+  clearStaleWorkerLock(ctx.basePath);
 }
 
 export const staleWorkerHandler: DriftHandler<StaleWorkerDrift> = {