@smartmemory/compose 0.2.8-beta → 0.2.10-beta

package/lib/build.js

@@ -58,6 +58,8 @@ import { CrossLayerAudit, loadDebugConfig } from './cross-layer-audit.js';
 import { emitCheckpoint } from './bug-checkpoint.js';
 import { appendHypothesisEntry, readHypotheses } from './bug-ledger.js';
 import { tier1CodexReview, tier2FreshAgent } from './bug-escalation.js';
+import { recordTaskStates, writeTimingSidecar } from './gsd-timing.js';
+import { writeGsdTaskDiff } from './gsd-diff-capture.js';
 
 // ---------------------------------------------------------------------------
 // COMP-FIX-HARD T5: per-step retries cap parser
@@ -2958,6 +2960,11 @@ export async function executeParallelDispatchServer(
   // mode invokes this fn with 6 args (opts={}) so every detector branch below
   // is skipped and build behavior stays byte-identical.
   const stuckDetector = opts.stuckDetector ?? null;
+  // COMP-GSD-6: optional per-task heartbeat. ONLY the gsd headless path passes
+  // one; build mode (opts={}) leaves it null so behavior stays byte-identical.
+  // Bumped from the push-event callback so a long task sitting in the dispatch
+  // poll loop keeps state.json's heartbeat fresh (else it would look crashed).
+  const onHeartbeat = typeof opts.onHeartbeat === 'function' ? opts.onHeartbeat : null;
   const startedTasks = new Set();
 
   if (streamWriter) {
@@ -2989,11 +2996,15 @@ export async function executeParallelDispatchServer(
   // state-machine driver. Forward valid events through streamWriter so the
   // bridge rebroadcasts them via SSE under the buildStreamEvent wrapper.
   let unsubscribePush = null;
-  if (typeof stratum.onEvent === 'function' && (streamWriter || stuckDetector)) {
+  if (typeof stratum.onEvent === 'function' && (streamWriter || stuckDetector || onHeartbeat)) {
     unsubscribePush = stratum.onEvent(flowId, stepId, (event) => {
       // Accept all KNOWN_VERSIONS (producer emits 0.2.6); pinning '0.2.5' dropped
       // every push event from the current producer. Client already validated.
       if (!event || !KNOWN_VERSIONS.has(event.schema_version)) return;
+      // COMP-GSD-6: any accepted event proves the run is alive — bump heartbeat.
+      if (onHeartbeat) {
+        try { onHeartbeat(event); } catch (err) { console.error('[build] heartbeat hook failed:', err); }
+      }
       // COMP-GSD-5: feed the per-task stuck detector (no-op in build mode —
       // stuckDetector is null there). record() ignores all but
       // tool_use_summary/tool_result and keys by event.task_id.
@@ -3009,6 +3020,20 @@ export async function executeParallelDispatchServer(
     });
   }
 
+  // COMP-GSD-7: capture poll-observed per-task timing (gsd path only). Stratum's
+  // poll response carries no per-task timing, so we stamp first-sight / first-
+  // terminal from our own poll loop and persist a timing.json sidecar for the
+  // milestone report. Gated on context.gsd so build mode is byte-identical.
+  const captureTiming = context?.gsd === true && !!context?.featureCode;
+  const taskTiming = {};
+
+  // COMP-PAR-MERGE-QUEUE: hold the merge+advance outcome (and its summary) so the
+  // re-dispatch check can run AFTER the finally (unsubscribe) below, and so the
+  // terminal `build_step_done` event fires only for the FINAL attempt — not for an
+  // attempt that is immediately re-dispatched. Terminal stuck/budget returns inside
+  // the try short-circuit and never set these.
+  let _redispatchOutcome = null;
+  let _finalSummary = null;
   try {
     // Poll until outcome is present (NOT can_advance — see design §3)
     let pollResult;
@@ -3021,6 +3046,7 @@ export async function executeParallelDispatchServer(
           `stratum_parallel_poll failed: ${pollResult.error}: ${pollResult.message || ''}`,
         );
       }
+      if (captureTiming) recordTaskStates(taskTiming, pollResult.tasks, new Date().toISOString());
       emitPerTaskProgress(streamWriter, pollResult, emittedStates);
 
       // COMP-GSD-5: real-time stuck detection (gsd path only — null in build).
@@ -3063,6 +3089,13 @@ export async function executeParallelDispatchServer(
       await new Promise((resolve) => setTimeout(resolve, intervalMs));
     }
 
+    // COMP-GSD-7: persist the timing sidecar on every terminal path (complete,
+    // stuck, budget) — best-effort; a write failure must never derail dispatch.
+    if (captureTiming) {
+      try { writeTimingSidecar(baseCwd, context.featureCode, taskTiming); }
+      catch (err) { console.error('[build] gsd timing sidecar write failed:', err.message); }
+    }
+
     // COMP-GSD-5: short-circuit return on a stuck verdict. The cancel outcome is
     // returned verbatim with the verdict attached; the gsd run loop branches on
     // `.stuck`. This bypasses the merge/advance bookkeeping below — there is
@@ -3134,7 +3167,7 @@ export async function executeParallelDispatchServer(
     if (hasServerMerge) {
       if (pollResult.outcome?.status === 'awaiting_consumer_advance') {
         // DEFER PATH: merge locally, report merge_status, let flow advance with truth.
-        const { mergeStatus, conflictedTaskId, conflictError } = applyServerDispatchDiffsCore(
+        const { mergeStatus, conflictedTaskId, conflictError, conflictFiles } = applyServerDispatchDiffsCore(
           dispatchResponse.tasks ?? [],
           pollResult.tasks,
           baseCwd,
@@ -3152,7 +3185,14 @@ export async function executeParallelDispatchServer(
           });
         }
 
-        const advanceResult = await stratum.parallelAdvance(flowId, stepId, mergeStatus);
+        // COMP-PAR-MERGE-QUEUE: on conflict, send a structured payload carrying a
+        // merge_conflict bounce record so Stratum surfaces it on ensure_failed and
+        // Compose's retry prompt can name the task/files (back-compat: 'clean'
+        // stays a bare string).
+        const advancePayload = mergeStatus === 'conflict'
+          ? { status: 'conflict', bounced_tasks: [buildMergeConflictBounce(conflictedTaskId, conflictError, conflictFiles)] }
+          : 'clean';
+        const advanceResult = await stratum.parallelAdvance(flowId, stepId, advancePayload);
         if (advanceResult?.error) {
           throw new Error(
             `stratum_parallel_advance failed: ${advanceResult.error}: ${advanceResult.message || ''}`,
@@ -3184,18 +3224,57 @@ export async function executeParallelDispatchServer(
       }
     }
 
-    if (streamWriter) {
-      streamWriter.write({
-        type: 'build_step_done', stepId,
-        parallel: true,
-        summary: pollResult.summary, flowId,
-      });
-    }
-
-    return pollResult.outcome;
+    _redispatchOutcome = pollResult.outcome;
+    _finalSummary = pollResult.summary;
+    // Fall through to the finally (unsubscribe), then the re-dispatch check below.
+    // build_step_done is emitted AFTER that check so a re-dispatched attempt does
+    // not surface a terminal step event.
   } finally {
     if (unsubscribePush) { try { unsubscribePush(); } catch { /* ignore */ } }
   }
+
+  // COMP-PAR-MERGE-QUEUE: a parallel step that failed its pre-merge gate / require
+  // policy / merge returns `ensure_failed`/`schema_failed` carrying the parallel
+  // surface (tasks). Re-dispatch it as a PARALLEL step rather than leaking the
+  // failure to the outer single-agent retry path (which can't re-run parallel
+  // tasks). Stratum reverted current_idx and persisted the bounce onto each
+  // failed task, so the re-run task prompts carry the failure context. The
+  // server's retry cap bounds this — it returns retries_exhausted/error (a
+  // terminal, non-parallel envelope) when exhausted, ending the recursion.
+  const out = _redispatchOutcome;
+  // Defensive cap so a misbehaving server/stub that keeps returning ensure_failed
+  // can't recurse without bound (Stratum's own retry cap should end it first).
+  const redispatchDepth = (opts._redispatchDepth ?? 0);
+  if (
+    out
+    && (out.status === 'ensure_failed' || out.status === 'schema_failed')
+    && Array.isArray(out.tasks)
+    && out.step_id === stepId
+    && redispatchDepth < 10
+  ) {
+    // The ensure_failed envelope is built from Stratum's parallel *surface*, which
+    // omits the merge-config fields (capture_diff/isolation) that gate the
+    // worktree-merge path (hasServerMerge). Carry them over from THIS dispatch (the
+    // same step) so the re-run still merges its diffs instead of silently skipping.
+    const redispatch = {
+      ...out,
+      isolation: dispatchResponse.isolation ?? out.isolation,
+      capture_diff: dispatchResponse.capture_diff ?? out.capture_diff,
+    };
+    return await executeParallelDispatchServer(
+      redispatch, stratum, context, progress, streamWriter, baseCwd,
+      { ...opts, _redispatchDepth: redispatchDepth + 1 },
+    );
+  }
+  // Terminal attempt — now surface the step-done lifecycle event.
+  if (streamWriter) {
+    streamWriter.write({
+      type: 'build_step_done', stepId,
+      parallel: true,
+      summary: _finalSummary, flowId,
+    });
+  }
+  return out;
 }
 
 /**
@@ -3211,9 +3290,42 @@ export async function executeParallelDispatchServer(
  * @param {string} patchDir — directory to write temporary .patch files
  * @returns {{mergeStatus:'clean'|'conflict', appliedFiles:string[], conflictedTaskId:string|null, conflictError:string|null}}
  */
+/**
+ * COMP-PAR-MERGE-QUEUE: best-effort extraction of the conflicting files from a
+ * `git apply` error message, falling back to the task's declared owned files.
+ * Used to populate a merge_conflict bounce record's `files`.
+ */
+export function extractConflictFiles(conflictError, ownedFiles = []) {
+  const files = new Set();
+  if (conflictError) {
+    // "error: patch failed: path/to/file:LINE"
+    for (const m of conflictError.matchAll(/patch failed:\s+(.+?):\d+/g)) files.add(m[1]);
+    // "error: path/to/file: patch does not apply"
+    for (const m of conflictError.matchAll(/error:\s+(.+?):\s+patch does not apply/g)) files.add(m[1]);
+  }
+  if (files.size === 0) for (const f of (ownedFiles ?? [])) files.add(f);
+  return [...files];
+}
+
+/**
+ * COMP-PAR-MERGE-QUEUE: build a structured merge_conflict bounce record (the
+ * Compose-side counterpart of Stratum's gate_failed bounce). Shape matches
+ * contracts/par-merge-bounce.json.
+ */
+export function buildMergeConflictBounce(taskId, error, files) {
+  return {
+    task_id: taskId,
+    reason: 'merge_conflict',
+    files: files ?? [],
+    command: null,
+    exit_code: null,
+    excerpt: (error ?? '').slice(-2048),
+  };
+}
+
 function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId, patchDir) {
   if (diffMap.size === 0) {
-    return { mergeStatus: 'clean', appliedFiles: [], conflictedTaskId: null, conflictError: null };
+    return { mergeStatus: 'clean', appliedFiles: [], conflictedTaskId: null, conflictError: null, conflictFiles: [] };
   }
 
   // Topological sort on depends_on edges (DFS)
@@ -3246,6 +3358,7 @@ function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId,
   let mergeStatus = 'clean';
   let conflictedTaskId = null;
   let conflictError = null;
+  let conflictFiles = [];
   const appliedFiles = new Set();
 
   for (const taskId of topoOrder) {
@@ -3274,6 +3387,7 @@ function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId,
       mergeStatus = 'conflict';
       conflictedTaskId = taskId;
       conflictError = err.message;
+      conflictFiles = extractConflictFiles(err.message, taskMap.get(taskId)?.files_owned ?? []);
       if (streamWriter) {
         streamWriter.write({
           type: 'build_error',
@@ -3310,6 +3424,7 @@ function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId,
     appliedFiles: [...appliedFiles],
     conflictedTaskId,
     conflictError,
+    conflictFiles,
   };
 }
 
@@ -3340,16 +3455,24 @@ function applyServerDispatchDiffsCore(taskList, pollTasks, baseCwd, streamWriter
       }
       continue;
     }
-    if (ts?.diff != null) diffMap.set(taskId, ts.diff);
+    if (ts?.diff != null) {
+      diffMap.set(taskId, ts.diff);
+      // COMP-GSD-7: snapshot the diff before the worktree is cleaned up, so the
+      // milestone report can inline it. gsd path only (context.gsd); best-effort.
+      if (context?.gsd === true && context?.featureCode) {
+        try { writeGsdTaskDiff(baseCwd, context.featureCode, taskId, ts.diff); }
+        catch (err) { console.error('[build] gsd diff snapshot write failed:', err.message); }
+      }
+    }
   }
 
   if (diffMap.size === 0) {
-    return { mergeStatus: 'clean', conflictedTaskId: null, conflictError: null, appliedFiles: [] };
+    return { mergeStatus: 'clean', conflictedTaskId: null, conflictError: null, appliedFiles: [], conflictFiles: [] };
   }
 
   const patchDir = mkdtempSync(join(tmpdir(), 'compose-server-patch-'));
   try {
-    const { mergeStatus, conflictedTaskId, conflictError, appliedFiles } =
+    const { mergeStatus, conflictedTaskId, conflictError, appliedFiles, conflictFiles } =
       applyTaskDiffsToBaseCwd(taskList, diffMap, baseCwd, streamWriter, stepId, patchDir);
 
     if (mergeStatus !== 'conflict' && appliedFiles.length > 0 && context) {
@@ -3358,7 +3481,7 @@ function applyServerDispatchDiffsCore(taskList, pollTasks, baseCwd, streamWriter
       context.filesChanged = [...set];
     }
 
-    return { mergeStatus, conflictedTaskId, conflictError, appliedFiles };
+    return { mergeStatus, conflictedTaskId, conflictError, appliedFiles, conflictFiles };
   } finally {
     try { rmSync(patchDir, { recursive: true, force: true }); } catch { /* best-effort */ }
   }

package/lib/gsd-diff-capture.js

@@ -0,0 +1,34 @@
+// lib/gsd-diff-capture.js
+//
+// COMP-GSD-7 S3: persist per-task diff snapshots for the milestone report.
+//
+// When a GSD parallel task runs in a worktree with capture_diff:true, Stratum
+// returns the unified diff in the poll payload (ts.diff). build.js consumes it
+// at the merge site and then drops it once the worktree is cleaned up. This
+// helper snapshots that diff to .compose/gsd/<feature>/diffs/<taskId>.diff so
+// the report (lib/gsd-milestone-report.js) can inline it. The path helper is the
+// single source of truth shared with the report reader.
+//
+// Atomic write: tmp+rename, mirrors lib/gsd-state.js:44.
+
+import { writeFileSync, existsSync, mkdirSync, renameSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+
+export function gsdDiffsDir(cwd, featureCode) {
+  return join(cwd, '.compose', 'gsd', featureCode, 'diffs');
+}
+
+export function gsdTaskDiffPath(cwd, featureCode, taskId) {
+  return join(gsdDiffsDir(cwd, featureCode), `${taskId}.diff`);
+}
+
+/** Atomic write of a task's unified diff. Returns the path. */
+export function writeGsdTaskDiff(cwd, featureCode, taskId, diffText) {
+  mkdirSync(gsdDiffsDir(cwd, featureCode), { recursive: true });
+  const target = gsdTaskDiffPath(cwd, featureCode, taskId);
+  const tmp = `${target}.tmp`;
+  if (existsSync(tmp)) { try { unlinkSync(tmp); } catch { /* ignore */ } }
+  writeFileSync(tmp, diffText);
+  renameSync(tmp, target);
+  return target;
+}

package/lib/gsd-events.js

@@ -0,0 +1,61 @@
+// lib/gsd-events.js
+//
+// COMP-GSD-7-EVENTLOG: append-only GSD run-event log at
+// .compose/gsd/<feature>/events.jsonl — one JSON object per line,
+// { ts, kind, ...detail }. GSD otherwise persists only snapshots, so the
+// milestone report's timeline (COMP-GSD-7) had to reconstruct order from
+// whatever artifacts happened to exist. This log records what happened, in
+// order, across resume sessions; the report consumes it (snapshot fallback).
+//
+// All writes are BEST-EFFORT — an event-log failure must never affect the run.
+// The reader tolerates a torn/corrupt final line (crash mid-append) by skipping
+// unparseable lines.
+//
+// Contract: `detail` must NOT contain `ts` or `kind` keys (the event kind would
+// be clobbered). For a `paused` event the stuck/budget discriminator is
+// `pauseKind`, not `kind`.
+
+import { appendFileSync, readFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+function gsdDir(cwd, featureCode) {
+  return join(cwd, '.compose', 'gsd', featureCode);
+}
+
+export function gsdEventsPath(cwd, featureCode) {
+  return join(gsdDir(cwd, featureCode), 'events.jsonl');
+}
+
+export function appendGsdEvent(cwd, featureCode, kind, detail = {}) {
+  try {
+    mkdirSync(gsdDir(cwd, featureCode), { recursive: true });
+    const line = JSON.stringify({ ts: new Date().toISOString(), kind, ...detail }) + '\n';
+    appendFileSync(gsdEventsPath(cwd, featureCode), line);
+  } catch { /* best-effort — never affect the run */ }
+}
+
+export function readGsdEvents(cwd, featureCode) {
+  const p = gsdEventsPath(cwd, featureCode);
+  if (!existsSync(p)) return [];
+  let raw;
+  try { raw = readFileSync(p, 'utf-8'); } catch { return []; }
+  const out = [];
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    let parsed;
+    try { parsed = JSON.parse(line); } catch { continue; /* skip torn/corrupt line */ }
+    // Only event OBJECTS are usable — a parseable scalar/array/null is not an
+    // event (and would throw in the report's label mapping). Treat as "no event".
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) out.push(parsed);
+  }
+  return out;
+}
+
+// COMP-GSD-7-EVENTLOG: fresh-run truncate. Empties the log so a new run's
+// timeline doesn't inherit a prior (abandoned) run's events. Called at the
+// planning checkpoint, AFTER preconditions pass (so a failed fresh invocation
+// doesn't destroy a prior run's history).
+export function clearGsdEvents(cwd, featureCode) {
+  const p = gsdEventsPath(cwd, featureCode);
+  try { if (existsSync(p)) writeFileSync(p, ''); } catch { /* best-effort */ }
+}

package/lib/gsd-headless-config.js

@@ -0,0 +1,110 @@
+// lib/gsd-headless-config.js
+//
+// COMP-GSD-6 S05: headless auto-resume policy. Reads `gsd.headless.*` from
+// .compose/compose.json and merges over conservative defaults. Every pause kind
+// is independently overridable (per the product decision): crash and stuck
+// auto-resume by default, budget does NOT (opting in would defeat the GSD-4
+// ceiling), but a user MAY set budget.enabled:true for a fully-unattended burn.
+//
+// Unset config ⇒ defaults ⇒ behavior is a plain run plus supervision.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+export const HEADLESS_DEFAULTS = Object.freeze({
+  autoResume: {
+    crash: { enabled: true, maxAttempts: 5 },
+    stuck: { enabled: true, maxAttempts: 2 },
+    budget: { enabled: false, maxAttempts: 0 },
+    // COMP-GSD-6-WATCHDOG: a hung child (heartbeat frozen on a live pid) is
+    // killed + resumed like a crash. On by default — that's the whole point of
+    // unattended robustness.
+    hung: { enabled: true, maxAttempts: 3 },
+  },
+  backoff: { baseMs: 2000, factor: 2, maxMs: 60000 },
+  heartbeatStaleMs: 90000,
+  // COMP-GSD-6-WATCHDOG: supervisor poll cadence, SIGTERM→SIGKILL grace, and the
+  // child's independent wall-clock heartbeat cadence (must be < heartbeatStaleMs).
+  watchdogPollMs: 15000,
+  watchdogKillGraceMs: 5000,
+  watchdogHeartbeatMs: 30000,
+});
+
+function num(v, fallback) {
+  return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : fallback;
+}
+function bool(v, fallback) {
+  return typeof v === 'boolean' ? v : fallback;
+}
+
+function mergeKind(override, def) {
+  const o = override ?? {};
+  return {
+    enabled: bool(o.enabled, def.enabled),
+    maxAttempts: num(o.maxAttempts, def.maxAttempts),
+  };
+}
+
+// Merge a raw `gsd.headless` object over HEADLESS_DEFAULTS with per-field type
+// validation (a malformed field falls back to its default, never throws).
+export function resolveHeadlessConfig(raw) {
+  const h = raw && typeof raw === 'object' ? raw : {};
+  const ar = h.autoResume && typeof h.autoResume === 'object' ? h.autoResume : {};
+  const bo = h.backoff && typeof h.backoff === 'object' ? h.backoff : {};
+  const d = HEADLESS_DEFAULTS;
+
+  // COMP-GSD-6-WATCHDOG: enforce the load-bearing invariant
+  // `watchdogHeartbeatMs < heartbeatStaleMs`. The child must restamp its heartbeat
+  // at least twice within the stale window, or a healthy quiet child trips the
+  // watchdog. Two guards:
+  //   - a degenerate stale window (< 2ms, incl. 0) is unusable — there's no room
+  //     for a smaller heartbeat — so it falls back to the default.
+  //   - a heartbeat cadence ≥ the (now positive) stale window is clamped to half
+  //     it (always strictly < stale for stale ≥ 2), rather than honored.
+  let heartbeatStaleMs = num(h.heartbeatStaleMs, d.heartbeatStaleMs);
+  if (heartbeatStaleMs < 2) heartbeatStaleMs = d.heartbeatStaleMs;
+  let watchdogHeartbeatMs = num(h.watchdogHeartbeatMs, d.watchdogHeartbeatMs);
+  if (watchdogHeartbeatMs >= heartbeatStaleMs) {
+    watchdogHeartbeatMs = Math.floor(heartbeatStaleMs / 2);
+  }
+
+  return {
+    autoResume: {
+      crash: mergeKind(ar.crash, d.autoResume.crash),
+      stuck: mergeKind(ar.stuck, d.autoResume.stuck),
+      budget: mergeKind(ar.budget, d.autoResume.budget),
+      hung: mergeKind(ar.hung, d.autoResume.hung),
+    },
+    backoff: {
+      baseMs: num(bo.baseMs, d.backoff.baseMs),
+      factor: num(bo.factor, d.backoff.factor),
+      maxMs: num(bo.maxMs, d.backoff.maxMs),
+    },
+    heartbeatStaleMs,
+    watchdogPollMs: num(h.watchdogPollMs, d.watchdogPollMs),
+    watchdogKillGraceMs: num(h.watchdogKillGraceMs, d.watchdogKillGraceMs),
+    watchdogHeartbeatMs,
+  };
+}
+
+// Read .compose/compose.json gsd.headless.* and resolve against defaults.
+export function readHeadlessConfig(cwd) {
+  const configPath = join(cwd, '.compose', 'compose.json');
+  let raw = {};
+  if (existsSync(configPath)) {
+    try {
+      const cfg = JSON.parse(readFileSync(configPath, 'utf-8'));
+      raw = cfg?.gsd?.headless ?? {};
+    } catch {
+      /* malformed config → defaults */
+    }
+  }
+  return resolveHeadlessConfig(raw);
+}
+
+// Backoff for attempt N (1-based): base * factor^(N-1), capped at maxMs.
+export function backoffMs(cfg, attempt) {
+  const { baseMs, factor, maxMs } = cfg.backoff;
+  const raw = baseMs * Math.pow(factor, Math.max(0, attempt - 1));
+  return Math.min(raw, maxMs);
+}