@smartmemory/compose 0.2.9-beta → 0.2.10-beta

package/contracts/par-merge-bounce.json

@@ -0,0 +1,39 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "par-merge-bounce.json",
+  "_source": "docs/features/COMP-PAR-MERGE-QUEUE/design.md",
+  "_roadmap": "COMP-PAR-MERGE-QUEUE",
+  "title": "ParMergeBounce",
+  "description": "A single bounce record produced by parallel_dispatch's post-dispatch merge gate. Unifies both failure kinds (a failed pre-merge verify gate, or a merge conflict during diff apply) into one structured contract. It is surfaced on the ensure_failed.bounced_tasks[] envelope AND persisted onto the failed task's state so that when the step is re-dispatched, Stratum's ParallelExecutor._render_prompt injects the failure context into the re-run task's prompt (server-side, because the server re-resolves the task list on each re-dispatch). Convention: _source = design doc origin, _roadmap = feature code.",
+  "type": "object",
+  "required": ["task_id", "reason", "files", "excerpt"],
+  "properties": {
+    "task_id": {
+      "type": "string",
+      "description": "Which task to bounce / re-dispatch."
+    },
+    "reason": {
+      "type": "string",
+      "enum": ["gate_failed", "merge_conflict"],
+      "description": "Why the task bounced. 'gate_failed' = a pre_merge_verify command exited non-zero in the task worktree; 'merge_conflict' = git apply --check/apply failed when merging the task diff onto base."
+    },
+    "files": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Files involved. gate_failed: from `git diff --name-only HEAD` in the worktree. merge_conflict: parsed from the git-apply error, falling back to the task's owned files."
+    },
+    "command": {
+      "type": ["string", "null"],
+      "description": "The gate command that failed, e.g. 'pnpm build'. Populated for gate_failed only; null for merge_conflict."
+    },
+    "exit_code": {
+      "type": ["integer", "null"],
+      "description": "Exit code of the failed gate command. Populated for gate_failed only; null for merge_conflict."
+    },
+    "excerpt": {
+      "type": "string",
+      "description": "Bounded (~2KB) tail of the gate stdout+stderr (gate_failed) OR the git-apply error text (merge_conflict). Bounded to avoid unbounded prompt growth and to limit credential leakage; do not echo environment."
+    }
+  },
+  "additionalProperties": true
+}

package/lib/build.js

@@ -3027,6 +3027,13 @@ export async function executeParallelDispatchServer(
   const captureTiming = context?.gsd === true && !!context?.featureCode;
   const taskTiming = {};
 
+  // COMP-PAR-MERGE-QUEUE: hold the merge+advance outcome (and its summary) so the
+  // re-dispatch check can run AFTER the finally (unsubscribe) below, and so the
+  // terminal `build_step_done` event fires only for the FINAL attempt — not for an
+  // attempt that is immediately re-dispatched. Terminal stuck/budget returns inside
+  // the try short-circuit and never set these.
+  let _redispatchOutcome = null;
+  let _finalSummary = null;
   try {
     // Poll until outcome is present (NOT can_advance — see design §3)
     let pollResult;
@@ -3160,7 +3167,7 @@ export async function executeParallelDispatchServer(
     if (hasServerMerge) {
       if (pollResult.outcome?.status === 'awaiting_consumer_advance') {
         // DEFER PATH: merge locally, report merge_status, let flow advance with truth.
-        const { mergeStatus, conflictedTaskId, conflictError } = applyServerDispatchDiffsCore(
+        const { mergeStatus, conflictedTaskId, conflictError, conflictFiles } = applyServerDispatchDiffsCore(
           dispatchResponse.tasks ?? [],
           pollResult.tasks,
           baseCwd,
@@ -3178,7 +3185,14 @@ export async function executeParallelDispatchServer(
           });
         }
 
-        const advanceResult = await stratum.parallelAdvance(flowId, stepId, mergeStatus);
+        // COMP-PAR-MERGE-QUEUE: on conflict, send a structured payload carrying a
+        // merge_conflict bounce record so Stratum surfaces it on ensure_failed and
+        // Compose's retry prompt can name the task/files (back-compat: 'clean'
+        // stays a bare string).
+        const advancePayload = mergeStatus === 'conflict'
+          ? { status: 'conflict', bounced_tasks: [buildMergeConflictBounce(conflictedTaskId, conflictError, conflictFiles)] }
+          : 'clean';
+        const advanceResult = await stratum.parallelAdvance(flowId, stepId, advancePayload);
         if (advanceResult?.error) {
           throw new Error(
             `stratum_parallel_advance failed: ${advanceResult.error}: ${advanceResult.message || ''}`,
@@ -3210,18 +3224,57 @@ export async function executeParallelDispatchServer(
       }
     }
 
-    if (streamWriter) {
-      streamWriter.write({
-        type: 'build_step_done', stepId,
-        parallel: true,
-        summary: pollResult.summary, flowId,
-      });
-    }
-
-    return pollResult.outcome;
+    _redispatchOutcome = pollResult.outcome;
+    _finalSummary = pollResult.summary;
+    // Fall through to the finally (unsubscribe), then the re-dispatch check below.
+    // build_step_done is emitted AFTER that check so a re-dispatched attempt does
+    // not surface a terminal step event.
   } finally {
     if (unsubscribePush) { try { unsubscribePush(); } catch { /* ignore */ } }
   }
+
+  // COMP-PAR-MERGE-QUEUE: a parallel step that failed its pre-merge gate / require
+  // policy / merge returns `ensure_failed`/`schema_failed` carrying the parallel
+  // surface (tasks). Re-dispatch it as a PARALLEL step rather than leaking the
+  // failure to the outer single-agent retry path (which can't re-run parallel
+  // tasks). Stratum reverted current_idx and persisted the bounce onto each
+  // failed task, so the re-run task prompts carry the failure context. The
+  // server's retry cap bounds this — it returns retries_exhausted/error (a
+  // terminal, non-parallel envelope) when exhausted, ending the recursion.
+  const out = _redispatchOutcome;
+  // Defensive cap so a misbehaving server/stub that keeps returning ensure_failed
+  // can't recurse without bound (Stratum's own retry cap should end it first).
+  const redispatchDepth = (opts._redispatchDepth ?? 0);
+  if (
+    out
+    && (out.status === 'ensure_failed' || out.status === 'schema_failed')
+    && Array.isArray(out.tasks)
+    && out.step_id === stepId
+    && redispatchDepth < 10
+  ) {
+    // The ensure_failed envelope is built from Stratum's parallel *surface*, which
+    // omits the merge-config fields (capture_diff/isolation) that gate the
+    // worktree-merge path (hasServerMerge). Carry them over from THIS dispatch (the
+    // same step) so the re-run still merges its diffs instead of silently skipping.
+    const redispatch = {
+      ...out,
+      isolation: dispatchResponse.isolation ?? out.isolation,
+      capture_diff: dispatchResponse.capture_diff ?? out.capture_diff,
+    };
+    return await executeParallelDispatchServer(
+      redispatch, stratum, context, progress, streamWriter, baseCwd,
+      { ...opts, _redispatchDepth: redispatchDepth + 1 },
+    );
+  }
+  // Terminal attempt — now surface the step-done lifecycle event.
+  if (streamWriter) {
+    streamWriter.write({
+      type: 'build_step_done', stepId,
+      parallel: true,
+      summary: _finalSummary, flowId,
+    });
+  }
+  return out;
 }
 
 /**
@@ -3237,9 +3290,42 @@ export async function executeParallelDispatchServer(
  * @param {string} patchDir — directory to write temporary .patch files
  * @returns {{mergeStatus:'clean'|'conflict', appliedFiles:string[], conflictedTaskId:string|null, conflictError:string|null}}
  */
+/**
+ * COMP-PAR-MERGE-QUEUE: best-effort extraction of the conflicting files from a
+ * `git apply` error message, falling back to the task's declared owned files.
+ * Used to populate a merge_conflict bounce record's `files`.
+ */
+export function extractConflictFiles(conflictError, ownedFiles = []) {
+  const files = new Set();
+  if (conflictError) {
+    // "error: patch failed: path/to/file:LINE"
+    for (const m of conflictError.matchAll(/patch failed:\s+(.+?):\d+/g)) files.add(m[1]);
+    // "error: path/to/file: patch does not apply"
+    for (const m of conflictError.matchAll(/error:\s+(.+?):\s+patch does not apply/g)) files.add(m[1]);
+  }
+  if (files.size === 0) for (const f of (ownedFiles ?? [])) files.add(f);
+  return [...files];
+}
+
+/**
+ * COMP-PAR-MERGE-QUEUE: build a structured merge_conflict bounce record (the
+ * Compose-side counterpart of Stratum's gate_failed bounce). Shape matches
+ * contracts/par-merge-bounce.json.
+ */
+export function buildMergeConflictBounce(taskId, error, files) {
+  return {
+    task_id: taskId,
+    reason: 'merge_conflict',
+    files: files ?? [],
+    command: null,
+    exit_code: null,
+    excerpt: (error ?? '').slice(-2048),
+  };
+}
+
 function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId, patchDir) {
   if (diffMap.size === 0) {
-    return { mergeStatus: 'clean', appliedFiles: [], conflictedTaskId: null, conflictError: null };
+    return { mergeStatus: 'clean', appliedFiles: [], conflictedTaskId: null, conflictError: null, conflictFiles: [] };
   }
 
   // Topological sort on depends_on edges (DFS)
@@ -3272,6 +3358,7 @@ function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId,
   let mergeStatus = 'clean';
   let conflictedTaskId = null;
   let conflictError = null;
+  let conflictFiles = [];
   const appliedFiles = new Set();
 
   for (const taskId of topoOrder) {
@@ -3300,6 +3387,7 @@ function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId,
       mergeStatus = 'conflict';
       conflictedTaskId = taskId;
       conflictError = err.message;
+      conflictFiles = extractConflictFiles(err.message, taskMap.get(taskId)?.files_owned ?? []);
       if (streamWriter) {
         streamWriter.write({
           type: 'build_error',
@@ -3336,6 +3424,7 @@ function applyTaskDiffsToBaseCwd(tasks, diffMap, baseCwd, streamWriter, stepId,
     appliedFiles: [...appliedFiles],
     conflictedTaskId,
     conflictError,
+    conflictFiles,
   };
 }
 
@@ -3378,12 +3467,12 @@ function applyServerDispatchDiffsCore(taskList, pollTasks, baseCwd, streamWriter
   }
 
   if (diffMap.size === 0) {
-    return { mergeStatus: 'clean', conflictedTaskId: null, conflictError: null, appliedFiles: [] };
+    return { mergeStatus: 'clean', conflictedTaskId: null, conflictError: null, appliedFiles: [], conflictFiles: [] };
   }
 
   const patchDir = mkdtempSync(join(tmpdir(), 'compose-server-patch-'));
   try {
-    const { mergeStatus, conflictedTaskId, conflictError, appliedFiles } =
+    const { mergeStatus, conflictedTaskId, conflictError, appliedFiles, conflictFiles } =
       applyTaskDiffsToBaseCwd(taskList, diffMap, baseCwd, streamWriter, stepId, patchDir);
 
     if (mergeStatus !== 'conflict' && appliedFiles.length > 0 && context) {
@@ -3392,7 +3481,7 @@ function applyServerDispatchDiffsCore(taskList, pollTasks, baseCwd, streamWriter
       context.filesChanged = [...set];
     }
 
-    return { mergeStatus, conflictedTaskId, conflictError, appliedFiles };
+    return { mergeStatus, conflictedTaskId, conflictError, appliedFiles, conflictFiles };
   } finally {
     try { rmSync(patchDir, { recursive: true, force: true }); } catch { /* best-effort */ }
   }

package/lib/gsd.js

@@ -38,6 +38,11 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 const PACKAGE_ROOT = resolve(__dirname, '..');
 
 const DEFAULT_GATE_COMMANDS = ['pnpm lint', 'pnpm build', 'pnpm test'];
+// COMP-PAR-MERGE-QUEUE: the fast per-task pre-merge gate (lint + build, no full
+// test suite). Enforced in each task's worktree before its diff merges; the full
+// `pnpm test` runs once at ship_gsd. Single-sourced into both the enforced gate
+// (execute.pre_merge_verify) and the instructed gate (task descriptions).
+const DEFAULT_FAST_GATE = ['pnpm lint', 'pnpm build'];
 
 // ---------- Public API ----------
 
@@ -122,6 +127,8 @@ export async function runGsd(featureCode, opts = {}) {
   // 4. Resolve gateCommands. loadProjectConfig() does not merge defaults, so
   // explicit fallback here.
   const gateCommands = resolveGateCommands(cwd, opts.gateCommands);
+  // COMP-PAR-MERGE-QUEUE: the fast per-task pre-merge gate (lint+build).
+  const preMergeGate = resolvePreMergeGate(cwd, opts.preMergeGate);
 
   // 4. Load pipeline spec
   const specPath = join(PACKAGE_ROOT, 'pipelines', 'gsd.stratum.yaml');
@@ -208,7 +215,7 @@ export async function runGsd(featureCode, opts = {}) {
     // can stage them. executeShipStep's default filter only stages feature
     // docs unless context.filesChanged is provided.
     stepCtx = {
-      stratum, cwd, featureCode, blueprintText, gateCommands,
+      stratum, cwd, featureCode, blueprintText, gateCommands, preMergeGate,
       filesChanged: [],
       stuckDetector,
       resumeTaskGraph,
@@ -260,6 +267,7 @@ export async function runGsd(featureCode, opts = {}) {
     let response = await stratum.plan(specYaml, 'gsd', {
       featureCode,
       gateCommands,
+      pre_merge_gate: preMergeGate,
     });
     const flowId = response.flow_id;
     flushState(stepCtx, { flowId, phase: 'decompose' });
@@ -273,11 +281,30 @@ export async function runGsd(featureCode, opts = {}) {
       response.status !== 'complete' &&
       response.status !== 'killed' &&
       response.status !== 'stuck' &&
-      response.status !== 'budget_exhausted'
+      response.status !== 'budget_exhausted' &&
+      response.status !== 'error'   // COMP-PAR-MERGE-QUEUE: terminal step failure (e.g. retries_exhausted)
     ) {
       response = await runOneStep(response, stepCtx);
     }
 
+    // COMP-PAR-MERGE-QUEUE: a step that exhausted its retries (e.g. the execute
+    // step after repeated pre-merge gate failures) surfaces as a terminal `error`
+    // envelope rather than silently advancing to ship. Stop here with the failure
+    // and its bounce context instead of throwing `unknown response status`.
+    if (response.status === 'error') {
+      emitCompletionDeltas(stepCtx);
+      flushState(stepCtx, { status: 'failed' });
+      return {
+        status: 'failed',
+        flowId,
+        stepId: response.step_id ?? stepCtx.lastStepId ?? null,
+        errorType: response.error_type ?? 'step_failed',
+        message: response.message ?? 'GSD step failed',
+        violations: response.violations ?? [],
+        bouncedTasks: response.bounced_tasks ?? [],
+      };
+    }
+
     if (response.status === 'stuck') {
       // Artifacts (stuck.md/json + pause.json) were written by runOneStep.
       // COMP-GSD-7-EVENTLOG: flush any completions that finished before the stuck
@@ -422,8 +449,32 @@ export function resolveGateCommands(cwd, override) {
   return [...DEFAULT_GATE_COMMANDS];
 }
 
+// COMP-PAR-MERGE-QUEUE: resolve the fast per-task pre-merge gate. Mirrors
+// resolveGateCommands but defaults to lint+build (no full test suite). Honors
+// `.compose/compose.json#preMergeGate`, else falls back to the non-test subset
+// of `gateCommands`, else DEFAULT_FAST_GATE.
+export function resolvePreMergeGate(cwd, override) {
+  if (Array.isArray(override) && override.length > 0) return override;
+  const configPath = join(cwd, '.compose', 'compose.json');
+  if (existsSync(configPath)) {
+    try {
+      const cfg = JSON.parse(readFileSync(configPath, 'utf-8'));
+      if (Array.isArray(cfg.preMergeGate) && cfg.preMergeGate.length > 0) {
+        return cfg.preMergeGate;
+      }
+      if (Array.isArray(cfg.gateCommands) && cfg.gateCommands.length > 0) {
+        const fast = cfg.gateCommands.filter((c) => !/\btest\b/.test(c));
+        if (fast.length > 0) return fast;
+      }
+    } catch {
+      /* fall through to default */
+    }
+  }
+  return [...DEFAULT_FAST_GATE];
+}
+
 async function runOneStep(response, ctx) {
-  const { stratum, cwd, featureCode, blueprintText, gateCommands } = ctx;
+  const { stratum, cwd, featureCode, blueprintText, gateCommands, preMergeGate } = ctx;
   const flowId = response.flow_id;
   const stepId = response.step_id;
   const stepType = response.type ?? response.step_type;
@@ -526,7 +577,10 @@ async function runOneStep(response, ctx) {
 
     // T6 step 7: validate decompose_gsd output and repair missing descriptions.
     if (stepId === 'decompose_gsd') {
-      result = validateAndRepairTaskGraph(result, blueprintText, gateCommands);
+      // COMP-PAR-MERGE-QUEUE: single-source the per-task instructed gate to the
+      // fast pre-merge gate (== the enforced execute.pre_merge_verify). Full
+      // `pnpm test` is instructed only at ship_gsd.
+      result = validateAndRepairTaskGraph(result, blueprintText, preMergeGate ?? gateCommands);
       // COMP-GSD-5: remember the ENRICHED graph so a later stuck halt can
       // persist the full task definitions (with descriptions/produces/consumes)
       // into pause.json — resume re-dispatches these without re-enriching.

package/lib/step-prompt.js

@@ -162,6 +162,12 @@ function buildConflictSection(conflicts) {
   return lines.join('\n');
 }
 
+// COMP-PAR-MERGE-QUEUE note: pre-merge bounce context (gate_failed / merge_conflict)
+// is injected into a re-dispatched task's prompt SERVER-SIDE, in Stratum's
+// ParallelExecutor._render_prompt — because the server re-resolves the task list
+// from flow state on each re-dispatch, so a Compose-side prompt edit here would
+// never reach the re-run task. See parallel_exec.py `_format_bounce_for_prompt`.
+
 /**
  * Build a retry prompt when postconditions failed.
  *

package/lib/stratum-mcp-client.js

@@ -444,7 +444,9 @@ export class StratumMcpClient {
    *
    * @param {string} flowId
    * @param {string} stepId
-   * @param {'clean'|'conflict'} mergeStatus
+   * @param {'clean'|'conflict'|{status:'clean'|'conflict',bounced_tasks?:object[]}} mergeStatus
+   *   COMP-PAR-MERGE-QUEUE: either a bare status string (back-compat) or a
+   *   structured object carrying merge_conflict bounce records. Serializes as-is.
    * @returns {Promise<object>}
    */
   async parallelAdvance(flowId, stepId, mergeStatus) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smartmemory/compose",
-  "version": "0.2.9-beta",
+  "version": "0.2.10-beta",
   "description": "Structured AI dev pipeline — goal-to-product orchestration with gates, iteration loops, and feature lifecycle management.",
   "author": "SmartMemory",
   "license": "MIT",

package/pipelines/gsd.stratum.yaml

@@ -18,6 +18,9 @@ workflow:
     gateCommands:
       type: array
       required: true   # injected by runGsd from loadProjectConfig() with default fallback
+    pre_merge_gate:
+      type: array
+      required: false  # COMP-PAR-MERGE-QUEUE: fast per-task enforced gate (lint+build); injected by runGsd
 
 contracts:
   PhaseResult:
@@ -36,6 +39,7 @@ flows:
     input:
       featureCode: {type: string}
       gateCommands: {type: array}
+      pre_merge_gate: {type: array}   # COMP-PAR-MERGE-QUEUE
     output: PhaseResult
     max_rounds: 10
     steps:
@@ -61,7 +65,7 @@ flows:
             - "Upstream tasks (spec-level summary; their code lands at end-of-step
               merge):" + one line per upstream task listing its declared produces
             - "GATES — you MUST run each command and they MUST pass before
-              declaring done:" + each command from the gateCommands array
+              declaring done:" + each command from the pre_merge_gate array
             - "Fix and re-run within this invocation. Do NOT declare done while
               gates are red."
 
@@ -75,10 +79,12 @@ flows:
           has no matching task or its File Plan files are split across
           multiple tasks.
 
-          gateCommands to embed: {input.gateCommands}
+          pre_merge_gate commands to embed (the fast per-task gate; the full
+          test suite runs once at ship): {input.pre_merge_gate}
         inputs:
-          featureCode:  "$.input.featureCode"
-          gateCommands: "$.input.gateCommands"
+          featureCode:    "$.input.featureCode"
+          gateCommands:   "$.input.gateCommands"
+          pre_merge_gate: "$.input.pre_merge_gate"
         output_contract: TaskGraph
         ensure:
           - "no_file_conflicts(result.tasks)"
@@ -93,6 +99,8 @@ flows:
         max_concurrent: 1
         isolation: worktree
         capture_diff: true
+        defer_advance: true   # COMP-PAR-MERGE-QUEUE: consumer merges + reports status, enabling conflict-bounce
+        pre_merge_verify: "$.input.pre_merge_gate"   # COMP-PAR-MERGE-QUEUE: enforced fast gate per task worktree
         require: all
         merge: sequential_apply
         retries: 2