mandrel 2.20.0 → 2.22.0

package/.agents/scripts/lib/orchestration/story-deliver-terminal.js

@@ -28,6 +28,10 @@
  * pre-#4543 pipeline collapsed by treating budget exhaustion as a block.
  */
 
+import nodeFs from 'node:fs';
+import path from 'node:path';
+import { storyTerminalEnvelopePath } from '../config/temp-paths.js';
+import { resolveConfig } from '../config-resolver.js';
 import { validateTerminalEnvelope } from './story-deliver-terminal-schema.js';
 
 // Re-exported so the schema split stays an implementation detail: every
@@ -327,7 +331,95 @@ export const TERMINAL_BEGIN_MARKER = '--- STORY DELIVER TERMINAL ---';
 export const TERMINAL_END_MARKER = '--- END TERMINAL ---';
 
 /**
- * Write a terminal envelope to stdout, between its markers.
+ * Resolve the repo config, or `undefined` when it cannot be read.
+ *
+ * An unreadable `.agentrc.json` must not cost the run its envelope copy: the
+ * path helpers fall back to the framework-default temp root, which is still a
+ * far better outcome than no artifact at all.
+ *
+ * @param {typeof resolveConfig} resolveConfigImpl
+ * @returns {object|undefined}
+ */
+function tolerantConfig(resolveConfigImpl) {
+  try {
+    return resolveConfigImpl();
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Persist a terminal envelope beside its Story's gate log (Story #4816).
+ *
+ * Stdout is a channel with exactly one reader — the turn that launched the
+ * close — and that reader is not always still listening. A `story-worker`
+ * that reports progress and ends its turn while the close it started is
+ * mid-gate-chain is behaving reasonably, but the envelope it never relayed is
+ * gone: the router then has to reconstruct the Story's state from labels, and
+ * `deliver-recover.js` used to answer that reconstruction with
+ * "Implementation never finished" — false, and its re-init suggestion can put
+ * a second close on one PR. Observed four times across three workers in one
+ * consumer run. The file is the second channel, and it outlives the turn.
+ *
+ * **Best-effort by construction.** Every failure path returns `null`: a
+ * delivery must never turn a landed PR into a crash because a temp directory
+ * was unwritable. The stdout envelope above is the contract; this is a copy.
+ *
+ * **Atomic by construction.** The payload is written to a pid-scoped
+ * temporary name and renamed into place, because the reader that matters most
+ * is a router polling during a live close — a half-written file would hand it
+ * a parse error at exactly the moment it is trying to avoid guessing.
+ *
+ * A null `storyId` (the `escalated` terminal, which by construction never
+ * authored a Story) has nowhere to be filed and writes nothing.
+ *
+ * `config` is optional and resolved lazily when absent. Two of the emit sites
+ * are `catch` blocks that crashed before any config was resolved, and those
+ * are precisely the runs whose envelope is most worth keeping — so the temp
+ * root is looked up here rather than left at the framework default, which
+ * would file the artifact where a consumer's router never looks (and where
+ * the retention purge would never reap it).
+ *
+ * @param {object} envelope A validated terminal envelope.
+ * @param {{
+ *   config?: object,
+ *   fsImpl?: typeof nodeFs,
+ *   resolveConfigImpl?: typeof resolveConfig,
+ * }} [deps]
+ * @returns {string|null} The path written, or `null` when nothing was.
+ */
+export function persistTerminalEnvelope(
+  envelope,
+  { config, fsImpl = nodeFs, resolveConfigImpl = resolveConfig } = {},
+) {
+  const storyId = envelope?.storyId;
+  if (!Number.isInteger(storyId) || storyId <= 0) return null;
+  let tmpPath = null;
+  try {
+    const resolved = config ?? tolerantConfig(resolveConfigImpl);
+    const target = storyTerminalEnvelopePath(storyId, resolved);
+    fsImpl.mkdirSync(path.dirname(target), { recursive: true });
+    tmpPath = `${target}.${process.pid}.tmp`;
+    fsImpl.writeFileSync(tmpPath, `${JSON.stringify(envelope)}\n`, 'utf8');
+    fsImpl.renameSync(tmpPath, target);
+    return target;
+  } catch {
+    // A rename that never ran leaves the scratch file behind; drop it rather
+    // than accumulating one per failed close.
+    if (tmpPath) {
+      try {
+        fsImpl.rmSync(tmpPath, { force: true });
+      } catch {
+        // Nothing left to try — this whole path is already best-effort.
+      }
+    }
+    return null;
+  }
+}
+
+/**
+ * Write a terminal envelope to stdout, between its markers, and persist a
+ * copy to disk.
  *
  * **Deliberately not `Logger.info`.** The envelope is this CLI's
  * machine-readable contract — every invocation emits exactly ONE, and a
@@ -340,16 +432,37 @@ export const TERMINAL_END_MARKER = '--- END TERMINAL ---';
  *
  * Single home for the marker format so the four emit sites (the runner's
  * terminal, the close CLI's failed-terminal catch, and both confirm-CLI
- * paths) cannot drift apart.
+ * paths) cannot drift apart — and, since Story #4816, the single home for the
+ * on-disk copy too, so no emit path can persist and another forget.
+ * {@link persistTerminalEnvelope} runs **first**: a caller that has read the
+ * markers off stdout can then rely on the file already being there.
  *
  * @param {object} envelope
- * @param {{ write?: (s: string) => void }} [opts] `write` is a test seam.
+ * @param {{
+ *   write?: (s: string) => void,
+ *   config?: object,
+ *   persist?: typeof persistTerminalEnvelope,
+ * }} [opts] `write` and `persist` are test seams; `config` resolves the
+ *   artifact's temp root and is threaded from whichever emit site holds one.
  * @returns {void}
  */
 export function emitTerminalEnvelope(
   envelope,
-  { write = (s) => process.stdout.write(s) } = {},
+  {
+    write = (s) => process.stdout.write(s),
+    config,
+    persist = persistTerminalEnvelope,
+  } = {},
 ) {
+  // Belt and braces around a copy: `persistTerminalEnvelope` already swallows
+  // its own failures, but the stdout envelope is the CONTRACT and the disk
+  // copy is a convenience. Nothing in the secondary path — including a future
+  // injected `persist` — may cost the caller the primary one.
+  try {
+    persist(envelope, { config });
+  } catch {
+    // Intentionally silent: see above.
+  }
   // Story #4685 — compact (not 2-space pretty) JSON. The envelope is a
   // machine contract callers recover with `JSON.parse`, so pretty-printing
   // only adds turn-resident bytes without helping any consumer.

package/.agents/scripts/lib/orchestration/story-follow-ups.js

@@ -9,11 +9,15 @@
  * @module lib/orchestration/story-follow-ups
  */
 
+import { signalsFile } from '../config/temp-paths.js';
 import { graduateRetroProposals } from '../feedback-loop/retro-proposals-graduator.js';
 import { DEFAULT_FRAMEWORK_REPO } from '../github/framework-repo.js';
 import { Logger } from '../Logger.js';
 import { normalizeGatheredSignal } from '../observability/runtime-friction.js';
-import { forEachLine } from '../observability/signals-writer.js';
+import {
+  forEachLine,
+  forEachSignalStreamLine,
+} from '../observability/signals-writer.js';
 import {
   composeRoutedProposals,
   deriveUnresolvedBlockedEvents,
@@ -83,8 +87,55 @@ export async function gatherStoryFrictionSignals(storyId, config) {
 }
 
 /**
- * Gather friction signals across every Story in a run, for the run-scoped
- * roll-up.
+ * Identity of one physical signal row, for de-duplication.
+ *
+ * `eventId` is minted by every producer (`diagnose-friction.js` and
+ * `runtime-friction.js` both `crypto.randomUUID()` it), so it is the primary
+ * key. A row predating the field falls back to its physical `file:line`,
+ * which is equally stable — the same row read through two passes over the
+ * same tree yields the same coordinates.
+ *
+ * @param {unknown} parsed
+ * @param {string} file
+ * @param {number} lineNumber
+ * @returns {string}
+ */
+function signalIdentity(parsed, file, lineNumber) {
+  const eventId =
+    parsed !== null &&
+    typeof parsed === 'object' &&
+    typeof (/** @type {Record<string, unknown>} */ (parsed).eventId) ===
+      'string'
+      ? /** @type {string} */ (
+          /** @type {Record<string, unknown>} */ (parsed).eventId
+        ).trim()
+      : '';
+  return eventId.length > 0 ? `event:${eventId}` : `row:${file}:${lineNumber}`;
+}
+
+/**
+ * Gather friction signals for the run-scoped roll-up, over the **whole
+ * surviving recurrence window** rather than the run's own Stories
+ * (Story #4824).
+ *
+ * The recurrence threshold in `retro-proposals.js` is ≥ 2 occurrences, and
+ * the window it was measured over was one run's Story ids. A defect that
+ * fires exactly **once per Story** — which is what a systemic framework
+ * defect looks like — therefore scored 1 on every Story and was discarded as
+ * a singleton, forever. Eighteen consecutive Stories filed nothing.
+ *
+ * So the gather reduces over every `signals.ndjson` still present under the
+ * configured temp root: `<tempRoot>/standalone/stories/story-<sid>/` and
+ * `<tempRoot>/run-<eid>/stories/story-<sid>/`. Temp-tree auto-purge
+ * shortening that window is acceptable — a short window under-counts, and
+ * therefore fails toward *not* filing, which is the safe direction.
+ *
+ * The run's own Stories are still gathered explicitly first. The discovery
+ * walk resolves through the identical path helpers, so it is provably a
+ * superset; the explicit pass makes "never fewer signals than before" a
+ * property of the code rather than of an argument about path resolution.
+ * {@link signalIdentity} de-duplicates the overlap, so one event can never be
+ * counted twice and inflate a singleton into a fabricated recurrence.
  *
  * Homed beside {@link gatherStoryFrictionSignals} on purpose: the two used to
  * be independent copies of the same loop in two modules, and they drifted in
@@ -94,17 +145,39 @@ export async function gatherStoryFrictionSignals(storyId, config) {
  * Unusable ids are skipped rather than throwing — a roll-up must not fail the
  * epilogue over one malformed entry.
  *
- * @param {Array<number|string>} storyIds
+ * @param {Array<number|string>} storyIds The run's own Stories.
  * @param {object} [config]
  * @returns {Promise<Array<{ category: string, source: 'framework'|'consumer', storyId: number, details: object }>>}
  */
 export async function gatherRunFrictionSignals(storyIds, config) {
   const signals = [];
+  const seen = new Set();
+  const take = (parsed, fallbackStoryId, identity) => {
+    if (seen.has(identity)) return;
+    seen.add(identity);
+    const signal = normalizeGatheredSignal(parsed, fallbackStoryId);
+    if (signal) signals.push(signal);
+  };
+
   for (const raw of Array.isArray(storyIds) ? storyIds : []) {
     const sid = Number(raw);
     if (!Number.isInteger(sid) || sid <= 0) continue;
-    signals.push(...(await gatherStoryFrictionSignals(sid, config)));
+    const file = signalsFile(null, sid, config);
+    await forEachLine(
+      null,
+      sid,
+      (parsed, lineNumber) =>
+        take(parsed, sid, signalIdentity(parsed, file, lineNumber)),
+      config,
+    );
   }
+
+  await forEachSignalStreamLine(
+    (parsed, { storyId, file, lineNumber }) =>
+      take(parsed, storyId, signalIdentity(parsed, file, lineNumber)),
+    config,
+  );
+
   return signals;
 }
 
@@ -151,14 +224,187 @@ function renderEmptyRollupLines(storyCount) {
   ];
 }
 
+/**
+ * Skip reasons that are a deliberate outcome rather than a broken loop. A
+ * roll-up whose every proposal was skipped for one of these filed nothing
+ * *on purpose*; anything else is the loop failing quietly.
+ */
+const BENIGN_SKIP_REASONS = new Set([
+  'already-filed',
+  'toggle-disabled',
+  'cross-repo-deferred',
+  'cap-reached',
+  'no-actionable-proposals',
+]);
+
+/**
+ * Summarize a category corpus for the "name what you saw" lines. Pure.
+ *
+ * @param {Array<{ category?: string }>} signals
+ * @returns {Array<{ category: string, occurrences: number }>}
+ */
+export function summarizeSignalCategories(signals) {
+  const counts = new Map();
+  for (const sig of Array.isArray(signals) ? signals : []) {
+    if (sig === null || typeof sig !== 'object') continue;
+    const category =
+      typeof sig.category === 'string' ? sig.category.trim() : '';
+    if (category.length === 0) continue;
+    counts.set(category, (counts.get(category) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .map(([category, occurrences]) => ({ category, occurrences }))
+    .sort((a, b) => a.category.localeCompare(b.category));
+}
+
+/**
+ * Classify what a roll-up's own numbers say about it — the reporting-layer
+ * assertion Story #4828 adds. Pure, so the run epilogue's step result and the
+ * rendered comment cannot disagree about whether a roll-up succeeded.
+ *
+ * Two suspect shapes, both of which previously rendered as success:
+ *
+ *   - `zeroProposals` — signals were gathered and **nothing** came out, not
+ *     even a below-threshold row. That is the third instance of the failure
+ *     mode Story #4578 fixed for the zero-signal case and Story #4824 for the
+ *     all-discarded case: an all-empty routed result is silence, and a routing
+ *     regression is exactly what it looks like.
+ *   - `unfiledProposals` — proposals cleared the threshold and none were
+ *     filed for a reason that is not a deliberate one. This is what actually
+ *     happened in Story #4828: `gh issue create` rejected every call over an
+ *     absent label, the error landed in a bucket nobody rendered, and the
+ *     roll-up reported `filed: 0`.
+ *
+ * @param {object} args
+ * @param {number} args.signalCount
+ * @param {number} args.proposalCount   framework + consumer
+ * @param {number} args.discardedCount
+ * @param {number} args.filedCount
+ * @param {string[]} [args.filingErrors]
+ * @param {Array<{ reason?: string }>} [args.filingSkipped]
+ * @returns {{ zeroProposals: boolean, unfiledProposals: boolean, blockingSkipReasons: string[] }}
+ */
+export function assessRollupOutcome({
+  signalCount,
+  proposalCount,
+  discardedCount,
+  filedCount,
+  filingErrors = [],
+  filingSkipped = [],
+}) {
+  const blockingSkipReasons = [
+    ...new Set(
+      (Array.isArray(filingSkipped) ? filingSkipped : [])
+        .map((entry) => (typeof entry?.reason === 'string' ? entry.reason : ''))
+        .filter((reason) => reason && !BENIGN_SKIP_REASONS.has(reason)),
+    ),
+  ].sort();
+  const errors = Array.isArray(filingErrors) ? filingErrors : [];
+  return {
+    zeroProposals:
+      signalCount > 0 && proposalCount === 0 && discardedCount === 0,
+    unfiledProposals:
+      proposalCount > 0 &&
+      filedCount === 0 &&
+      (errors.length > 0 || blockingSkipReasons.length > 0),
+    blockingSkipReasons,
+  };
+}
+
+/**
+ * Render the "N signals in, zero proposals out" warning (Story #4828).
+ *
+ * @param {number} signalCount
+ * @param {Array<{ category: string, occurrences: number }>} categories
+ * @returns {string[]}
+ */
+function renderZeroProposalLines(signalCount, categories) {
+  const named =
+    categories.length > 0
+      ? categories.map((c) => `\`${c.category}\` ×${c.occurrences}`).join(', ')
+      : '_none — every gathered signal carried an unusable category_';
+  return [
+    `> ⚠️ **${signalCount} friction signals gathered, 0 proposals produced — a routing outcome, not a clean run.**`,
+    `> Categories seen: ${named}.`,
+    '> Nothing cleared the actionable threshold AND nothing was recorded below',
+    '> it, so every signal was netted out as recovered or dropped in routing.',
+    '> A regression in routing renders byte-identically to this, which is why',
+    '> the roll-up states it rather than rendering silence.',
+  ];
+}
+
+/**
+ * Render the "proposals cleared the threshold but none were filed" warning
+ * (Story #4828).
+ *
+ * @param {number} proposalCount
+ * @param {string[]} filingErrors
+ * @param {string[]} blockingSkipReasons
+ * @returns {string[]}
+ */
+function renderUnfiledProposalLines(
+  proposalCount,
+  filingErrors,
+  blockingSkipReasons,
+) {
+  const lines = [
+    `> ⚠️ **${proposalCount} actionable proposal(s) reached the filer and none were filed.**`,
+    '> Auto-file is on, so this is the feedback loop failing, not declining.',
+  ];
+  if (blockingSkipReasons.length > 0) {
+    lines.push(`> Skipped: ${blockingSkipReasons.join(', ')}.`);
+  }
+  for (const error of filingErrors) {
+    lines.push(`> ${error}`);
+  }
+  return lines;
+}
+
+/**
+ * Render one discarded (below-threshold) roll-up row (Story #4824).
+ *
+ * The pre-#4824 row was `` `category` ×N `` and nothing else. That is exactly
+ * how a defect firing once per Story stayed invisible for eighteen
+ * consecutive Stories: an operator reading "×1" cannot tell a one-off from a
+ * systemic defect whose window was too narrow to see it recur. The row now
+ * names the emitting tools, the bucket fingerprint, and the number of
+ * distinct Stories it spans — the cross-run count the widened recurrence
+ * window produces.
+ *
+ * Every added field is optional so a caller passing a hand-built proposals
+ * object (or an older persisted one) still renders.
+ *
+ * @param {{ category: string, occurrences: number, tools?: string[], fingerprint?: string, storyCount?: number }} item
+ * @returns {string}
+ */
+function renderDiscardedItem(item) {
+  const parts = [`\`${item.category}\` ×${item.occurrences}`];
+  if (Number.isInteger(item.storyCount) && item.storyCount > 0) {
+    const plural = item.storyCount === 1 ? 'Story' : 'Stories';
+    parts.push(`across ${item.storyCount} ${plural}`);
+  }
+  if (Array.isArray(item.tools) && item.tools.length > 0) {
+    parts.push(`via ${item.tools.map((t) => `\`${t}\``).join(', ')}`);
+  }
+  if (typeof item.fingerprint === 'string' && item.fingerprint.length > 0) {
+    parts.push(`fingerprint \`${item.fingerprint}\``);
+  }
+  return parts.join(' — ');
+}
+
 /**
  * @param {{
  *   storyId: number,
  *   proposals: object,
  *   graduated: object,
  *   storyCount?: number,
+ *   signalCount?: number,
+ *   categories?: Array<{ category: string, occurrences: number }>,
  * }} args - `storyCount` (default 1) is how many Stories the roll-up spans;
  *   it decides whether an empty result reads as quiet or as a flagged claim.
+ *   `signalCount` / `categories` (Story #4828) are what the roll-up actually
+ *   gathered, so a zero-proposal or zero-filed outcome can name its own
+ *   corpus instead of rendering as a clean run.
  * @returns {string}
  */
 export function buildFollowUpsCommentBody({
@@ -166,17 +412,37 @@ export function buildFollowUpsCommentBody({
   proposals,
   graduated,
   storyCount = 1,
+  signalCount = 0,
+  categories = [],
 }) {
   const filed = Array.isArray(graduated?.filed) ? graduated.filed : [];
   const framework = proposals?.framework ?? [];
   const consumer = proposals?.consumer ?? [];
   const discarded = proposals?.discarded ?? [];
+  const outcome = assessRollupOutcome({
+    signalCount,
+    proposalCount: framework.length + consumer.length,
+    discardedCount: discarded.length,
+    filedCount: filed.length,
+    filingErrors: graduated?.errors,
+    filingSkipped: graduated?.skipped,
+  });
   const lines = [
     '### follow-ups',
     '',
     `Actionable follow-ups captured from Story #${storyId} after merge.`,
     '',
   ];
+  if (outcome.unfiledProposals) {
+    lines.push(
+      ...renderUnfiledProposalLines(
+        framework.length + consumer.length,
+        Array.isArray(graduated?.errors) ? graduated.errors : [],
+        outcome.blockingSkipReasons,
+      ),
+      '',
+    );
+  }
   if (filed.length > 0) {
     lines.push('**Filed**');
     for (const item of filed) {
@@ -198,9 +464,9 @@ export function buildFollowUpsCommentBody({
     lines.push('');
   }
   if (discarded.length > 0) {
-    lines.push('**Single-occurrence (not filed)**');
+    lines.push('**Below threshold (not filed)**');
     for (const item of discarded) {
-      lines.push(`- ${item.source}: \`${item.category}\` ×${item.occurrences}`);
+      lines.push(`- ${item.source}: ${renderDiscardedItem(item)}`);
     }
     lines.push('');
   }
@@ -210,7 +476,14 @@ export function buildFollowUpsCommentBody({
     consumer.length === 0 &&
     discarded.length === 0
   ) {
-    lines.push(...renderEmptyRollupLines(storyCount));
+    // Story #4828 — "no proposals" has two readings, and only one of them is
+    // a quiet run. Signals gathered but nothing routed is the third instance
+    // of the silence Stories #4578 and #4824 each fixed once.
+    lines.push(
+      ...(outcome.zeroProposals
+        ? renderZeroProposalLines(signalCount, categories)
+        : renderEmptyRollupLines(storyCount)),
+    );
     lines.push('');
   }
   lines.push('```json');
@@ -219,9 +492,23 @@ export function buildFollowUpsCommentBody({
       {
         storyId,
         storyCount,
+        // Story #4828 — the corpus the roll-up actually read. Without it a
+        // reader cannot tell "0 proposals because nothing recurred" from
+        // "0 proposals because routing broke".
+        signalCount,
+        categories,
         framework: framework.map((i) => i.category),
         consumer: consumer.map((i) => i.category),
-        discarded: discarded.map((i) => i.category),
+        // Story #4824 — the machine-readable twin of the row above. A bare
+        // category list could not distinguish a genuine one-off from a
+        // recurrence the window was too narrow to see, so the count, the
+        // cross-Story span, and the shape fingerprint ride along.
+        discarded: discarded.map((i) => ({
+          category: i.category,
+          occurrences: i.occurrences,
+          storyCount: i.storyCount ?? null,
+          fingerprint: i.fingerprint ?? null,
+        })),
         filed: filed.map((i) => ({
           category: i.category,
           url: i.url ?? null,
@@ -234,7 +521,13 @@ export function buildFollowUpsCommentBody({
           filed.length === 0 &&
           framework.length === 0 &&
           consumer.length === 0 &&
-          discarded.length === 0,
+          discarded.length === 0 &&
+          signalCount === 0,
+        // Story #4828 — the two remaining shapes that used to render as
+        // success. Machine-readable twins of the warning prose above.
+        zeroProposalSuspect: outcome.zeroProposals,
+        unfiledProposalSuspect: outcome.unfiledProposals,
+        filingErrors: Array.isArray(graduated?.errors) ? graduated.errors : [],
       },
       null,
       2,
@@ -308,6 +601,8 @@ export async function captureStoryFollowUps({
       storyId: sid,
       proposals,
       graduated,
+      signalCount: signals.length,
+      categories: summarizeSignalCategories(signals),
     });
     await upsertStructuredComment(provider, sid, FOLLOW_UPS_COMMENT_TYPE, body);
     progress?.(