mandrel 1.57.0

package/.agents/scripts/lib/orchestration/retro/phases/gather-signals.js

@@ -0,0 +1,335 @@
+/**
+ * phases/gather-signals.js — retro Phase 1: aggregate retro signals.
+ *
+ * Reads child Stories' `story-perf-summary` comments, descendant labels
+ * (HITL events), the Epic's `parked-follow-ons` structured comment, and
+ * per-Story `signals.ndjson` streams. Composes the routed-proposal
+ * sections via `composeRoutedProposals`.
+ *
+ * Exported as `gatherRetroSignals` for the parent sequencer (and tests).
+ */
+
+import { TYPE_LABELS } from '../../../label-constants.js';
+import { forEachLine } from '../../../observability/signals-writer.js';
+import { concurrentMap } from '../../../util/concurrent-map.js';
+import { composeRoutedProposals } from '../../retro-proposals.js';
+import { parseFencedJsonComment } from '../../structured-comment-parser.js';
+import { findStructuredComment } from '../../ticketing.js';
+
+/**
+ * Default framework repo (the Mandrel mirror) used when the caller does
+ * not supply an override. Consumer projects re-routing framework friction
+ * back to mandrel rely on this constant.
+ */
+export const DEFAULT_FRAMEWORK_REPO = 'dsj1984/mandrel';
+
+const RECUT_BODY_MARKER = /<!--\s*recut-of:\s*#?\d+\s*-->/;
+
+// Story #3347 — bounded fan-out cap for the per-Story `signals.ndjson`
+// reads below. Mirrors the read-concurrency convention used elsewhere in
+// the orchestrator (e.g. SUBTICKET_HYDRATION_CONCURRENCY,
+// CASCADE_SIBLING_READ_CONCURRENCY); keeps a large Epic from opening an
+// unbounded number of concurrent file handles while still collapsing N
+// sequential disk reads into one wall-clock fan-out.
+const SIGNALS_READ_CONCURRENCY = 8;
+
+// Detects #NNN issue references in an Epic body — any match means the Epic
+// likely has child Stories enumerated in the planning artifact, so a
+// zero-descendant walk is suspicious.
+const EPIC_BODY_REFERENCE = /#\d+/;
+
+/**
+ * Pure: aggregate `frictionByCategory` payloads into a single integer.
+ */
+function sumFriction(byCategory) {
+  if (!byCategory || typeof byCategory !== 'object') return 0;
+  let total = 0;
+  for (const v of Object.values(byCategory)) {
+    if (typeof v === 'number' && Number.isFinite(v) && v >= 0) total += v;
+  }
+  return total;
+}
+
+/**
+ * Walk every descendant ticket of `epicId` once. Returns the flat list with
+ * each ticket's labels + body + state — the consumer derives its own
+ * counts from this snapshot. Pure with respect to the provider injection.
+ *
+ * `provider.getSubTickets` is part of the `ITicketingProvider` contract;
+ * the call is **not** optional-chained so a missing method (or a typo'd
+ * test stub) surfaces as a thrown error rather than a silent empty graph.
+ */
+async function collectDescendants(provider, epicId) {
+  const visited = new Set([epicId]);
+  const out = [];
+  // Story #2853 — level-order BFS so every parent at the current depth
+  // fires its getSubTickets call concurrently. Previously this loop
+  // awaited one parent at a time, serializing one network round-trip per
+  // descendant tier (a 2-level Epic with 10 Stories was 11 sequential
+  // calls). Each `getSubTickets` already fans out child hydration
+  // internally with concurrency=8 (issues.js:SUBTICKET_HYDRATION_CONCURRENCY),
+  // so outer parallelism is consistent with the existing design.
+  let frontier = [epicId];
+  while (frontier.length > 0) {
+    const results = await Promise.all(
+      frontier.map((id) => provider.getSubTickets(id)),
+    );
+    const nextFrontier = [];
+    for (const subs of results) {
+      for (const sub of subs ?? []) {
+        const subId = Number(sub?.id ?? sub?.number);
+        if (!Number.isInteger(subId) || visited.has(subId)) continue;
+        visited.add(subId);
+        out.push(sub);
+        nextFrontier.push(subId);
+      }
+    }
+    frontier = nextFrontier;
+  }
+  return out;
+}
+
+/**
+ * Defensive guard: when the descendant walker returned zero, probe the
+ * Epic ticket itself. If the Epic exists and its body references child
+ * issues (e.g., `#123` planning refs), the empty walk is almost certainly
+ * a contract drift in the underlying provider — emit a loud warn so the
+ * silent failure surfaces. Probe errors degrade silently (the guard is
+ * best-effort observability, not a correctness gate).
+ */
+async function warnIfEpicLooksPopulated({ epicId, provider, logger }) {
+  let epic;
+  try {
+    epic = await provider.getTicket?.(epicId);
+  } catch (err) {
+    logger?.warn?.(
+      `[retro-runner] guard: getTicket(#${epicId}) failed (continuing): ${err?.message ?? err}`,
+    );
+    return;
+  }
+  if (!epic) return;
+  const body = typeof epic.body === 'string' ? epic.body : '';
+  if (!EPIC_BODY_REFERENCE.test(body)) return;
+  logger?.warn?.(
+    `[retro-runner] WARNING: Epic #${epicId} body references child issues, ` +
+      'but the descendant walker returned zero — retro will under-report ' +
+      'friction. Possible provider contract drift in getSubTickets.',
+  );
+}
+
+/**
+ * Read raw retro signals from the GitHub graph. Pure with respect to the
+ * provider injection — exported so tests can drive the predicate end-to-end
+ * with a stub provider.
+ *
+ * When `descendants.length === 0`, the function probes `provider.getTicket`
+ * to distinguish "Epic legitimately empty" from "the descendant walker
+ * silently returned nothing under a populated Epic" (the failure mode the
+ * pre-Story #2289 `getSubIssues` contract drift produced). A populated
+ * Epic with zero walked descendants emits a loud warn via the supplied
+ * `logger` so the silent failure becomes visible. Probe failure degrades
+ * gracefully — the function never throws on the guard alone.
+ *
+ * **3-tier ledgers (Story #3151, Story #3200).** Under the 3-tier
+ * hierarchy (Epic → Feature → Story; no Task-tier children), `friction`
+ * / `parked` / `recuts` / `storyPerfSummaries` are all Story-scoped, so
+ * the function continues to produce a non-empty signals report. The
+ * empty-walk guard above is **not** triggered for this shape because
+ * `descendants` is populated (it contains the Stories themselves); the
+ * guard fires only when the walker returned literally nothing under a
+ * body that references children.
+ *
+ * @returns {Promise<{
+ *   stories: Array<{ id: number, body?: string, labels?: string[] }>,
+ *   counts:  { friction: number, parked: number, recuts: number, hitl: number },
+ *   storyPerfSummaries: object[],
+ *   epicPerfReport: object|null,
+ *   parkedFollowOns: { recuts: object[], parked: object[], present: boolean },
+ * }>}
+ */
+export async function gatherRetroSignals({
+  epicId,
+  provider,
+  logger,
+  frameworkRepo,
+  consumerRepo,
+  forEachLineFn = forEachLine,
+  composeRoutedProposalsFn = composeRoutedProposals,
+}) {
+  const descendants = await collectDescendants(provider, epicId);
+  if (descendants.length === 0) {
+    await warnIfEpicLooksPopulated({ epicId, provider, logger });
+  }
+  const stories = descendants.filter((t) =>
+    (t.labels ?? []).includes(TYPE_LABELS.STORY),
+  );
+  // HITL count: distinct descendants that currently or historically carry
+  // `agent::blocked`. We can only see "currently" here without an event
+  // stream — counts undercount but never overcount.
+  const hitl = descendants.filter((t) =>
+    (t.labels ?? []).includes('agent::blocked'),
+  ).length;
+
+  // Aggregate per-Story `story-perf-summary` payloads for friction totals.
+  // Story #2853 — fan out the per-Story `findStructuredComment` lookups
+  // concurrently. Each Story's lookup resolves to a single paginated
+  // `getTicketComments` round-trip (the raw-comments cache at
+  // ticketing/reads.js:381-412 dedupes the per-type lookups within a
+  // ticket), so parallelization adds no extra wire calls — it just
+  // collapses N sequential round-trips into one wall-clock fan-out.
+  // `Promise.all` preserves index order, so the resulting
+  // `storyPerfSummaries` array stays deterministic relative to `stories`.
+  const perStoryComments = await Promise.all(
+    stories.map((story) =>
+      findStructuredComment(
+        provider,
+        story.id ?? story.number,
+        'story-perf-summary',
+      ),
+    ),
+  );
+  const storyPerfSummaries = [];
+  let frictionFromSummaries = 0;
+  for (const comment of perStoryComments) {
+    const parsed = parseFencedJsonComment(comment);
+    if (parsed) {
+      storyPerfSummaries.push(parsed);
+      frictionFromSummaries += sumFriction(parsed.frictionByCategory);
+    }
+  }
+
+  // Epic-level perf report (used by the full retro's "Top hotspots").
+  const epicPerfComment = await findStructuredComment(
+    provider,
+    epicId,
+    'epic-perf-report',
+  );
+  const epicPerfReport = parseFencedJsonComment(epicPerfComment);
+
+  // Parked + recut counts: prefer the structured comment; fall back to body
+  // grep for the recut marker when the comment is absent.
+  const parkedComment = await findStructuredComment(
+    provider,
+    epicId,
+    'parked-follow-ons',
+  );
+  const parkedParsed = parseFencedJsonComment(parkedComment);
+  let parkedFollowOns;
+  if (parkedParsed) {
+    parkedFollowOns = {
+      present: true,
+      recuts: Array.isArray(parkedParsed.recuts) ? parkedParsed.recuts : [],
+      parked: Array.isArray(parkedParsed.parked) ? parkedParsed.parked : [],
+    };
+  } else {
+    const recutsByBody = stories.filter(
+      (s) => typeof s.body === 'string' && RECUT_BODY_MARKER.test(s.body),
+    );
+    parkedFollowOns = {
+      present: false,
+      recuts: recutsByBody.map((s) => ({ storyId: s.id ?? s.number })),
+      parked: [],
+    };
+  }
+
+  const counts = {
+    friction: frictionFromSummaries,
+    parked: parkedFollowOns.parked.length,
+    recuts: parkedFollowOns.recuts.length,
+    hitl,
+  };
+
+  // Story #2558 — read per-Story `signals.ndjson` streams (already
+  // source-tagged by Story #2553's writer) and compose the four routed
+  // proposal sections (framework / consumer / memory / discarded). Read
+  // failures degrade silently — observability MUST NOT take down the
+  // retro path. Empty streams yield empty arrays so the composer
+  // remains backward-compatible.
+  //
+  // Story #3347 — the per-Story reads previously ran one-at-a-time in a
+  // sequential `for` loop, serializing N disk reads. They now fan out via
+  // `concurrentMap` with a bounded cap (`SIGNALS_READ_CONCURRENCY`). Each
+  // Story accumulates into its own local arrays; `concurrentMap` preserves
+  // input order so we concatenate the per-Story results in `stories` order.
+  // That keeps `routedSignals` / `memorablePatterns` — and therefore the
+  // composed `routedProposals` — byte-for-byte identical to the prior
+  // serial behaviour, independent of which read settles first.
+  const perStorySignals = await concurrentMap(
+    stories,
+    async (story) => {
+      const sid = Number(story.id ?? story.number);
+      if (!Number.isInteger(sid) || sid <= 0) {
+        return { routedSignals: [], memorablePatterns: [] };
+      }
+      const localRoutedSignals = [];
+      const localMemorablePatterns = [];
+      try {
+        await forEachLineFn(epicId, sid, (parsed) => {
+          if (parsed === null || typeof parsed !== 'object') return;
+          const record = /** @type {Record<string, unknown>} */ (parsed);
+          const category =
+            typeof record.category === 'string' ? record.category : null;
+          const source =
+            record.source === 'framework' ? 'framework' : 'consumer';
+          if (category) {
+            localRoutedSignals.push({ category, source });
+          }
+          if (record.memorable === true && typeof record.insight === 'string') {
+            localMemorablePatterns.push({
+              category: category ?? 'general',
+              insight: record.insight,
+            });
+          }
+        });
+      } catch (err) {
+        logger?.warn?.(
+          `[retro-runner] forEachLine failed for story #${sid} (continuing): ${
+            err?.message ?? err
+          }`,
+        );
+      }
+      return {
+        routedSignals: localRoutedSignals,
+        memorablePatterns: localMemorablePatterns,
+      };
+    },
+    { concurrency: SIGNALS_READ_CONCURRENCY },
+  );
+  const routedSignals = [];
+  const memorablePatterns = [];
+  for (const perStory of perStorySignals) {
+    routedSignals.push(...perStory.routedSignals);
+    memorablePatterns.push(...perStory.memorablePatterns);
+  }
+
+  // Resolve repos. Caller overrides win; otherwise default the consumer
+  // repo to the project's own `github.owner/repo` (best-effort: the
+  // provider may expose it, but we don't depend on it — empty string
+  // disables that pane in the routed proposals).
+  const resolvedFrameworkRepo =
+    typeof frameworkRepo === 'string' && frameworkRepo.length > 0
+      ? frameworkRepo
+      : DEFAULT_FRAMEWORK_REPO;
+  const resolvedConsumerRepo =
+    typeof consumerRepo === 'string' && consumerRepo.length > 0
+      ? consumerRepo
+      : resolvedFrameworkRepo; // when caller omits, fall back to the framework repo so the command renders without an empty `--repo` flag.
+
+  const routedProposals = composeRoutedProposalsFn({
+    epicId,
+    frameworkRepo: resolvedFrameworkRepo,
+    consumerRepo: resolvedConsumerRepo,
+    signals: routedSignals,
+    unresolvedBlockedEvents: [],
+    memorablePatterns,
+  });
+
+  return {
+    stories,
+    counts,
+    storyPerfSummaries,
+    epicPerfReport,
+    parkedFollowOns,
+    routedProposals,
+  };
+}

package/.agents/scripts/lib/orchestration/retro/phases/post-and-mirror.js

@@ -0,0 +1,133 @@
+/**
+ * phases/post-and-mirror.js — retro Phase 4: compose + upsert + mirror.
+ *
+ * Runs gather → compose → checks → upsert structured comment → mirror to
+ * the per-Epic temp dir, and emits the `retro.end` boundary on the bus
+ * once the write side-effects settle.
+ *
+ * Extracted so `runRetro` can wrap the full body in a try/catch for the
+ * `retro.end` boundary emit on throw without cluttering the happy-path
+ * read.
+ */
+
+import nodeFs from 'node:fs';
+import path from 'node:path';
+
+import { epicRetroMirrorPath } from '../../../config/temp-paths.js';
+import { upsertStructuredComment } from '../../ticketing.js';
+import { appendChecksSection, collectRetroFindings } from './checks.js';
+import {
+  composeRetroBody as defaultComposeRetroBody,
+  normalizeInterventionCount,
+} from './compose-body.js';
+import { gatherRetroSignals as defaultGatherRetroSignals } from './gather-signals.js';
+
+/**
+ * Inner compose-and-post helper. Extracted so `runRetro` can wrap the
+ * full body in a try/catch for the `retro.end` boundary emit without
+ * cluttering the happy-path read.
+ */
+export async function composeAndPostRetro({
+  epicId,
+  provider,
+  logger,
+  forceFull,
+  timestamp,
+  bus,
+  now,
+  manualInterventions,
+  gatherFn = defaultGatherRetroSignals,
+  composeFn = defaultComposeRetroBody,
+  upsertFn = upsertStructuredComment,
+  runChecksFn,
+  assembleStateFn,
+  cwd,
+  fsImpl = nodeFs,
+  startedAt,
+  onMirrorWritten,
+  perfThresholds = null,
+}) {
+  const signals = await gatherFn({ epicId, provider, logger });
+
+  // Best-effort fetch of the Epic title for the heading.
+  let epicTitle;
+  try {
+    const epic = await provider.getTicket?.(epicId);
+    epicTitle = epic?.title;
+  } catch (err) {
+    logger?.warn?.(
+      `[retro-runner] Failed to fetch Epic #${epicId} title (using fallback): ${err?.message ?? err}`,
+    );
+  }
+
+  const interventions = normalizeInterventionCount(manualInterventions);
+  const { body, compact, scorecard } = composeFn({
+    epicId,
+    epicTitle,
+    counts: { ...signals.counts, interventions },
+    storyPerfSummaries: signals.storyPerfSummaries,
+    epicPerfReport: signals.epicPerfReport,
+    parkedFollowOns: signals.parkedFollowOns,
+    routedProposals: signals.routedProposals,
+    timestamp,
+    forceFull,
+    perfThresholds,
+  });
+
+  const findings = await collectRetroFindings({
+    runChecksFn,
+    assembleStateFn,
+    cwd,
+    logger,
+  });
+  const bodyWithChecks = appendChecksSection(body, findings);
+
+  logger?.info?.(
+    `[retro-runner] Posting ${compact ? 'compact' : 'full'} retro on Epic #${epicId}${findings.length > 0 ? ` (${findings.length} finding(s))` : ''}...`,
+  );
+  const result = await upsertFn(provider, epicId, 'retro', bodyWithChecks);
+
+  // Story #2089: also mirror the retro body to the per-Epic temp dir so
+  // operators can read it locally without re-fetching from GitHub. GitHub
+  // remains SSOT — a write failure logs a warn and does not fail the
+  // phase. The path is resolved relative to `cwd` when supplied so that
+  // worktree-scoped invocations land under the worktree's temp tree.
+  let mirrorAbsPath = null;
+  try {
+    const rel = epicRetroMirrorPath(epicId);
+    const absPath = path.isAbsolute(rel)
+      ? rel
+      : path.join(cwd ?? process.cwd(), rel);
+    fsImpl.mkdirSync(path.dirname(absPath), { recursive: true });
+    fsImpl.writeFileSync(absPath, bodyWithChecks, 'utf8');
+    mirrorAbsPath = absPath;
+    onMirrorWritten?.(absPath);
+    logger?.info?.(`[retro-runner] Mirrored retro to ${absPath}`);
+  } catch (err) {
+    logger?.warn?.(
+      `[retro-runner] Failed to write retro mirror (retro.md) for Epic #${epicId} (continuing — GitHub upsert succeeded): ${err?.message ?? err}`,
+    );
+  }
+
+  // Story #2252 — emit `retro.end` after the upsert + mirror settle so
+  // the lifecycle ledger captures the closing boundary with the
+  // posted/compact flags AND the resolved mirror path (when present).
+  const endedAt = typeof now === 'function' ? now() : Date.now();
+  const retroEndPayload = {
+    epicId,
+    posted: true,
+    compact: Boolean(compact),
+    durationMs: Math.max(0, Math.floor(endedAt - startedAt)),
+  };
+  if (mirrorAbsPath) retroEndPayload.retroPath = mirrorAbsPath;
+  await bus.emit('retro.end', retroEndPayload);
+
+  return {
+    posted: true,
+    compact,
+    scorecard,
+    body: bodyWithChecks,
+    findings,
+    commentId: result?.commentId,
+  };
+}

package/.agents/scripts/lib/orchestration/retro-heuristics.js

@@ -0,0 +1,57 @@
+/**
+ * Heuristics that drive which retro path the `epic-retro` helper composes.
+ *
+ * The compact three-section retro fires when the Epic's dispatch manifest
+ * carries zero friction signals. Non-zero on any of the five dimensions
+ * routes back to the full six-section retro. Keeping the predicate here
+ * (pure, no I/O) guards the clean-sprint definition from drifting as the
+ * retro helper markdown evolves.
+ *
+ * Epic #1030 Story #1046 — the `friction` count is now sourced from the
+ * aggregated `story-perf-summary.frictionByCategory` totals across the
+ * Epic's descendants (the unified summary comment posted by
+ * `analyze-execution.js`), not the legacy per-Task `friction` structured
+ * comment fan-out. The predicate signature stays unchanged — the input
+ * is still a single integer count — so callers only need to swap their
+ * data source.
+ *
+ * Story #2289 — added `interventions`, sourced from the
+ * `manualInterventions` array on the `epic-run-state` snapshot read
+ * via `epic-run-state-store.read`. The same list disqualifies the
+ * Epic from auto-merge, so the retro shape and the auto-merge gate
+ * now agree on what "clean" means.
+ */
+
+/**
+ * Evaluate whether an Epic's dispatch manifest counts qualify for the
+ * compact retro path.
+ *
+ * All five signals must be zero. Any non-zero value (including negatives,
+ * which are invalid but treated defensively as "non-clean") returns false.
+ * Non-number inputs are treated as missing and default to zero so callers
+ * can omit dimensions they have not yet gathered — a call with no arguments
+ * returns true.
+ *
+ * @param {Object} [counts]
+ * @param {number} [counts.friction=0]      Aggregate friction-event count, summed from `story-perf-summary.frictionByCategory` across the Epic's descendants (Story #1046).
+ * @param {number} [counts.parked=0]        Count of parked follow-on Stories (no manifest lineage).
+ * @param {number} [counts.recuts=0]        Count of Stories carrying a `<!-- recut-of: #N -->` marker.
+ * @param {number} [counts.hitl=0]          Count of tickets that raised an `agent::blocked` event mid-sprint (the runtime HITL pause point).
+ * @param {number} [counts.interventions=0] Count of recorded manual interventions (the same `manualInterventions` list the auto-merge predicate consults — Story #2289).
+ * @returns {boolean}
+ */
+export function isCleanManifest(counts = {}) {
+  const { friction, parked, recuts, hitl, interventions } = counts;
+  return (
+    normalize(friction) === 0 &&
+    normalize(parked) === 0 &&
+    normalize(recuts) === 0 &&
+    normalize(hitl) === 0 &&
+    normalize(interventions) === 0
+  );
+}
+
+function normalize(value) {
+  if (typeof value !== 'number' || !Number.isFinite(value)) return 0;
+  return value;
+}