mandrel 2.24.0 → 2.26.0

package/.agents/scripts/lib/orchestration/plan-persist/run-plan-persist.js

@@ -38,11 +38,14 @@
  */
 
 import { rm } from 'node:fs/promises';
-import path from 'node:path';
 import { getLimits, PROJECT_ROOT } from '../../config-resolver.js';
 import { gitSpawn } from '../../git-utils.js';
 import { Logger } from '../../Logger.js';
 import { sweepTempRetention } from '../../temp-retention.js';
+import {
+  concurrentMap,
+  FANOUT_CONCURRENCY,
+} from '../../util/concurrent-map.js';
 import {
   deriveStoryShape,
   LITE_ROUTE_LABEL,
@@ -410,6 +413,198 @@ export async function reapStalePlanDirs({
   return { reaped: result.purged.map((entry) => entry.path) };
 }
 
+/**
+ * Fail closed on a payload that cannot be persisted, and warn on an
+ * explicitly-authorized over-budget one. Extracted from `runPlanPersist`
+ * (Story #4926) so the entry point carries the flow, not the guards.
+ *
+ * @param {unknown} rawStories
+ * @param {{ maxTickets: number, allowOverBudget: boolean }} limits
+ * @returns {void}
+ * @throws {Error} On an empty payload or an unauthorized over-budget one.
+ */
+function assertPersistablePlan(rawStories, { maxTickets, allowOverBudget }) {
+  if (!Array.isArray(rawStories) || rawStories.length === 0) {
+    throw new Error(
+      '[plan-persist] stories payload must be a non-empty array ' +
+        '(--stories <file>). Default is one Story.',
+    );
+  }
+  if (rawStories.length <= maxTickets) return;
+  if (!allowOverBudget) {
+    throw new Error(
+      `[plan-persist] Stories (${rawStories.length}) exceed the reviewability ` +
+        `budget (${maxTickets}). Re-scope, or rerun with --allow-over-budget.`,
+    );
+  }
+  Logger.warn(
+    `[plan-persist] Persisting an over-budget plan: ${rawStories.length} ` +
+      `Stories vs. budget ${maxTickets} (--allow-over-budget).`,
+  );
+}
+
+/**
+ * Enforce the draft-reachability critic: orphans throw, a skip is ledgered.
+ *
+ * @param {{ status: string, reasons: string[], orphans?: object[] }} reachability
+ * @param {object} config
+ * @returns {Promise<void>}
+ * @throws {Error} `PLAN_REACHABILITY_ORPHANS` when the draft orphans a Story.
+ */
+async function enforceReachability(reachability, config) {
+  if (reachability.status === 'orphans') {
+    const err = new Error(renderReachabilityOrphans(reachability));
+    err.code = 'PLAN_REACHABILITY_ORPHANS';
+    err.orphans = reachability.orphans;
+    throw err;
+  }
+  Logger.info(`[plan-persist] reachability: ${reachability.reasons[0]}`);
+  if (reachability.status === 'skipped') {
+    await appendCriticSkip(
+      {
+        critic: 'reachability',
+        reasons: reachability.reasons,
+        cli: 'plan-persist',
+      },
+      config,
+    );
+  }
+}
+
+/**
+ * Write the per-Story checkpoint, upsert the plan-summary comment, and flip
+ * every created Story to `agent::ready`. Terminal ordering is load-bearing:
+ * `agent::ready` lands last so it can honestly mean "fully persisted"
+ * (Story #4541). A dry run performs none of it.
+ *
+ * **The checkpoints fan out; the phase boundary does not** (Story #4952). The
+ * per-Story upserts are independent of one another and run under bounded
+ * concurrency, but the `await` on that whole fan-out is what keeps the
+ * Story #4541 invariant intact: *every* checkpoint is on its ticket before the
+ * first `agent::ready` flip is issued, so `ready` still means "fully
+ * persisted" and a `/deliver` that picks a Story up cannot read a null
+ * checkpoint. Concurrency inside the phase is safe; overlapping the phases is
+ * the race this ordering exists to close.
+ *
+ * @param {object} args
+ * @returns {Promise<void>}
+ */
+async function persistStoryArtifacts({
+  provider,
+  created,
+  primary,
+  route,
+  summaryBody,
+}) {
+  const cohort = created.map((createdStory) => ({
+    slug: createdStory.slug,
+    id: createdStory.id,
+  }));
+  await concurrentMap(
+    created,
+    (story) =>
+      writeCheckpointV2(provider, story.id, {
+        persist: {
+          completedAt: new Date().toISOString(),
+          storyCount: created.length,
+          primaryStoryId: primary.id,
+          stories: cohort,
+        },
+        // Ledger the authored route verdict — the recorded reason and the
+        // per-Story shape evidence, including a shape-refused claim — on plan
+        // state (Story #4722). No authored verdict writes no block: absence
+        // is the standard full path.
+        ...(route ? { route } : {}),
+      }),
+    // The per-Story checkpoint upserts (Story #4952): each targets a
+    // different issue and reads nothing another writes, so this loop was
+    // serial only by construction — but see {@link persistStoryArtifacts}
+    // for the phase ordering that is *not* incidental.
+    { concurrency: FANOUT_CONCURRENCY },
+  );
+  await upsertStructuredComment(
+    provider,
+    primary.id,
+    PLAN_SUMMARY_COMMENT_TYPE,
+    summaryBody,
+  );
+  await markStoriesReady({ provider, created });
+}
+
+/**
+ * Remove this run's plan directory on terminal success, then sweep the
+ * abandoned ones terminal-success cleanup can never reach (Story #4541).
+ *
+ * @param {{ config: object, planDir: string|null, skipCleanup: boolean }} args
+ * @returns {Promise<void>}
+ */
+async function cleanupPlanDirs({ config, planDir, skipCleanup }) {
+  if (!skipCleanup && planDir) {
+    try {
+      await rm(planDir, { recursive: true, force: true });
+    } catch (err) {
+      Logger.warn(`[plan-persist] temp cleanup skipped: ${err.message}`);
+    }
+  }
+  await reapStalePlanDirs({ config, keepDir: skipCleanup ? planDir : null });
+}
+
+/**
+ * Log the effective complexity route (Story #4722). Lite is upheld by the
+ * shape backstop; anything else reports why it fell back to full.
+ *
+ * @param {object|null} route
+ * @param {boolean} isLiteRoute
+ * @returns {void}
+ */
+function logEffectiveRoute(route, isLiteRoute) {
+  if (isLiteRoute) {
+    Logger.info(
+      `[plan-persist] ceremony-lite route upheld by the shape backstop: ` +
+        `created Stories carry the ${LITE_ROUTE_LABEL} hint ` +
+        `(recorded reason: ${route.authored.reason}). /deliver re-derives ` +
+        'the route from each Story body — the label is never the control signal.',
+    );
+    return;
+  }
+  if (route) {
+    Logger.warn(
+      `[plan-persist] ${route.reasons.join('; ')} — persisting as full ` +
+        '(no route hint label).',
+    );
+  }
+}
+
+/**
+ * Log the operator-facing persist epilogue: adoption, the ready primary, the
+ * deliver command, and the cohort grouping label.
+ *
+ * @param {{ created: object[], primary: object, planRunLabel: string }} args
+ * @returns {void}
+ */
+function logPersistEpilogue({ created, primary, planRunLabel }) {
+  const adopted = created.filter((story) => story.adopted);
+  if (adopted.length > 0) {
+    Logger.info(
+      `[plan-persist] resumed ${adopted.length} of ${created.length} Story(ies) ` +
+        `from a previous persist: ${adopted.map((s2) => `#${s2.id}`).join(', ')}.`,
+    );
+  }
+  Logger.info(
+    `[plan-persist] Persisted ${created.length} Story(ies)` +
+      `; primary #${primary.id} is agent::ready.`,
+  );
+  Logger.info(
+    `[plan-persist] Deliver with: /deliver ${created.map((s2) => s2.id).join(' ')}`,
+  );
+  // Metadata only — a GitHub filter for the cohort this run authored, never
+  // a delivery-resolution input (/deliver stays ids-only, Story #4540).
+  Logger.info(
+    `[plan-persist] Cohort grouping label: ${planRunLabel} — filter with ` +
+      `label:${planRunLabel}`,
+  );
+}
+
 /**
  * Execute the flat Story persist end to end.
  *
@@ -451,6 +646,7 @@ export async function runPlanPersist({
     stories: rawStories = null,
     techSpecContent = null,
     planAcceptance = null,
+    planContextEnvelope = null,
   } = artifacts ?? {};
   const {
     forceReview = false,
@@ -474,26 +670,10 @@ export async function runPlanPersist({
   // through the shared standalone ledger (Story #4541).
   const runStartedAt = opts.metricsSince ?? new Date().toISOString();
 
-  if (!Array.isArray(rawStories) || rawStories.length === 0) {
-    throw new Error(
-      '[plan-persist] stories payload must be a non-empty array ' +
-        '(--stories <file>). Default is one Story.',
-    );
-  }
-
-  const maxTickets = getLimits(config).maxTickets;
-  if (rawStories.length > maxTickets && !allowOverBudget) {
-    throw new Error(
-      `[plan-persist] Stories (${rawStories.length}) exceed the reviewability ` +
-        `budget (${maxTickets}). Re-scope, or rerun with --allow-over-budget.`,
-    );
-  }
-  if (rawStories.length > maxTickets && allowOverBudget) {
-    Logger.warn(
-      `[plan-persist] Persisting an over-budget plan: ${rawStories.length} ` +
-        `Stories vs. budget ${maxTickets} (--allow-over-budget).`,
-    );
-  }
+  assertPersistablePlan(rawStories, {
+    maxTickets: getLimits(config).maxTickets,
+    allowOverBudget,
+  });
 
   Logger.info(
     `[plan-persist] Running cross-validation on ${rawStories.length} Story ticket(s)...`,
@@ -515,29 +695,19 @@ export async function runPlanPersist({
     tickets: rawStories,
     config,
   });
-  if (reachability.status === 'orphans') {
-    const err = new Error(renderReachabilityOrphans(reachability));
-    err.code = 'PLAN_REACHABILITY_ORPHANS';
-    err.orphans = reachability.orphans;
-    throw err;
-  }
-  Logger.info(`[plan-persist] reachability: ${reachability.reasons[0]}`);
-  if (reachability.status === 'skipped') {
-    await appendCriticSkip(
-      {
-        critic: 'reachability',
-        reasons: reachability.reasons,
-        cli: 'plan-persist',
-      },
-      config,
-    );
-  }
+  await enforceReachability(reachability, config);
 
   // Split policy + inline Spec fold (over-budget Specs fail closed — no docs/).
   const { stories } = assemblePlanStories(rawStories, {
     sharedSpec: techSpecContent,
     planAcceptance: planAcceptance ?? undefined,
     sourceTicketIds,
+    // The seed this plan was authored from is the provenance source: an audit
+    // sweep's Single-plan seed carries the `audit-fingerprints` /
+    // `audit-semantic-keys` footers, which assembly copies into every persisted
+    // Story body so the next sweep recognises what it already planned
+    // (Story #4877). Empty for a `--tickets` run, which is a no-op.
+    provenanceSource: planContextEnvelope?.seed?.content ?? '',
   });
 
   // Effective complexity route (Story #4722): the planner's authored lite
@@ -552,19 +722,7 @@ export async function runPlanPersist({
     injectedRules,
   });
   const isLiteRoute = route?.route === 'lite';
-  if (isLiteRoute) {
-    Logger.info(
-      `[plan-persist] ceremony-lite route upheld by the shape backstop: ` +
-        `created Stories carry the ${LITE_ROUTE_LABEL} hint ` +
-        `(recorded reason: ${route.authored.reason}). /deliver re-derives ` +
-        'the route from each Story body — the label is never the control signal.',
-    );
-  } else if (route) {
-    Logger.warn(
-      `[plan-persist] ${route.reasons.join('; ')} — persisting as full ` +
-        '(no route hint label).',
-    );
-  }
+  logEffectiveRoute(route, isLiteRoute);
 
   const { created, planRunLabel } = await createStoryIssues({
     provider,
@@ -609,35 +767,13 @@ export async function runPlanPersist({
   });
 
   if (!dryRun) {
-    for (const story of created) {
-      await writeCheckpointV2(provider, story.id, {
-        persist: {
-          completedAt: new Date().toISOString(),
-          storyCount: created.length,
-          primaryStoryId: primary.id,
-          stories: created.map((createdStory) => ({
-            slug: createdStory.slug,
-            id: createdStory.id,
-          })),
-        },
-        // Ledger the authored route verdict — the recorded reason and the
-        // per-Story shape evidence, including a shape-refused claim — on plan
-        // state (Story #4722). No authored verdict writes no block: absence
-        // is the standard full path.
-        ...(route ? { route } : {}),
-      });
-    }
-    await upsertStructuredComment(
+    await persistStoryArtifacts({
       provider,
-      primary.id,
-      PLAN_SUMMARY_COMMENT_TYPE,
+      created,
+      primary,
+      route,
       summaryBody,
-    );
-
-    // Terminal step: every checkpoint above is now on every Story, so
-    // `agent::ready` can honestly mean "fully persisted" (Story #4541).
-    // Anything that picks a Story up from here reads a real checkpoint.
-    await markStoriesReady({ provider, created });
+    });
   }
 
   const supersede = await runSupersedePhase({
@@ -653,37 +789,8 @@ export async function runPlanPersist({
   // any) rather than reading as a clean no-op — Story #4554.
   supersede.sourceTicketOrigin = sourceTicketOrigin;
 
-  if (!skipCleanup && planDir) {
-    try {
-      await rm(planDir, { recursive: true, force: true });
-    } catch (err) {
-      Logger.warn(`[plan-persist] temp cleanup skipped: ${err.message}`);
-    }
-  }
-  // Terminal-success cleanup only ever removes *this* run's planDir, so
-  // abandoned ones accumulated forever. Sweep them (Story #4541).
-  await reapStalePlanDirs({ config, keepDir: skipCleanup ? planDir : null });
-
-  const adopted = created.filter((story) => story.adopted);
-  if (adopted.length > 0) {
-    Logger.info(
-      `[plan-persist] resumed ${adopted.length} of ${created.length} Story(ies) ` +
-        `from a previous persist: ${adopted.map((s2) => `#${s2.id}`).join(', ')}.`,
-    );
-  }
-  Logger.info(
-    `[plan-persist] Persisted ${created.length} Story(ies)` +
-      `; primary #${primary.id} is agent::ready.`,
-  );
-  Logger.info(
-    `[plan-persist] Deliver with: /deliver ${created.map((s2) => s2.id).join(' ')}`,
-  );
-  // Metadata only — a GitHub filter for the cohort this run authored, never
-  // a delivery-resolution input (/deliver stays ids-only, Story #4540).
-  Logger.info(
-    `[plan-persist] Cohort grouping label: ${planRunLabel} — filter with ` +
-      `label:${planRunLabel}`,
-  );
+  await cleanupPlanDirs({ config, planDir, skipCleanup });
+  logPersistEpilogue({ created, primary, planRunLabel });
 
   return {
     stories: created,

package/.agents/scripts/lib/orchestration/plan-persist/story-ops.js

@@ -17,12 +17,17 @@
 
 import { createHash } from 'node:crypto';
 import { applyBlockedByDependencies } from '../../../providers/github/blocked-by-add.js';
+import { carryProvenanceFooters } from '../../findings/route-finding.js';
 import { Logger } from '../../Logger.js';
 import { AGENT_LABELS, TYPE_LABELS } from '../../label-constants.js';
 import {
   parse as parseStoryBody,
   serialize as serializeStoryBody,
 } from '../../story-body/story-body.js';
+import {
+  concurrentMap,
+  FANOUT_CONCURRENCY,
+} from '../../util/concurrent-map.js';
 import { assertSpecWithinBudget } from '../spec-spill.js';
 import { assertAcceptancePartition } from '../split-policy-validator.js';
 import {
@@ -392,7 +397,18 @@ function assembleOnePlanStory(ticket, opts) {
   });
   // Body first: the fingerprint is an identity over the *assembled* content,
   // so it cannot be computed until that content exists.
-  const body = serializeStoryBody({ ...folded, depends_on });
+  const serialized = serializeStoryBody({ ...folded, depends_on });
+  // Carry audit dedup provenance out of the seed this plan was authored from
+  // (Story #4877). The audit sweep's Single-plan path stamps the
+  // `audit-fingerprints` / `audit-semantic-keys` footers into the seed it hands
+  // `/plan`; without this the persisted Story carries no provenance and the
+  // next sweep re-files work it already planned. Mechanical on purpose — the
+  // authoring agent is not asked to notice HTML comments in a one-pager. A
+  // non-audit seed carries no footers, so this is a no-op there.
+  const { body } = carryProvenanceFooters({
+    from: opts.provenanceSource ?? '',
+    into: serialized,
+  });
   const fingerprint = planStoryFingerprint({ slug, title, body });
   return {
     story: {
@@ -783,6 +799,14 @@ async function ensurePersistLabel({
  * (see `ensureCohortLabel`). `/deliver` never reads it — delivery stays
  * ids-only over live state (Story #4540's actual point).
  *
+ * **The create loop stays serial and dependency-ordered** — deliberately, and
+ * unlike every other per-Story loop on this path (Story #4952). It is not an
+ * independent fan-out: `renderStoryBodyForCreate(story, idBySlug)` resolves a
+ * Story's `depends_on` slugs to the real issue ids its siblings were just
+ * minted with, and `idBySlug` is filled *in loop order* by the POSTs
+ * themselves. Running it concurrently would render `#undefined` dependency
+ * refs for any Story whose dependency had not yet returned an id.
+ *
  * **Sibling order is mirrored into native GitHub `blocked_by` edges** once
  * every id is known (Story #4544), so plan-created order stops depending on
  * prose. That pass is non-fatal — see `mirrorNativeDependencyEdges`.
@@ -948,6 +972,16 @@ export async function createStoryIssues({ provider, stories, opts = {} }) {
  * Fails closed: an un-flipped Story is invisible to `/deliver`, which is the
  * safe direction — the operator is told exactly which ids need the label.
  *
+ * **Collect failures, never fast-fail** (preserved verbatim under the
+ * Story #4952 concurrency conversion). Every Story is attempted even when an
+ * earlier one's PATCH rejects, because the whole value of the closing error is
+ * naming the *complete* set of ids that still need the label by hand. The
+ * mapper below therefore absorbs its own rejection into a per-Story outcome
+ * rather than letting `concurrentMap`'s first-rejection-wins policy abandon
+ * the remaining flips. `concurrentMap` preserves input order, so `readied[]`
+ * and the reported failures come back in `created[]` order exactly as the
+ * serial loop produced them.
+ *
  * @param {object} args
  * @param {object} args.provider
  * @param {Array<{ id: number, slug: string }>} args.created
@@ -960,18 +994,32 @@ export async function markStoriesReady({ provider, created }) {
         'Stories to agent::ready.',
     );
   }
-  const readied = [];
-  const failed = [];
-  for (const story of created) {
-    try {
-      await provider.updateTicket(story.id, {
-        labels: { add: [AGENT_LABELS.READY] },
-      });
-      readied.push(story.id);
-    } catch (err) {
-      failed.push(`#${story.id} (${story.slug}): ${err.message}`);
-    }
-  }
+  const outcomes = await concurrentMap(
+    created,
+    async (story) => {
+      try {
+        await provider.updateTicket(story.id, {
+          labels: { add: [AGENT_LABELS.READY] },
+        });
+        return { id: story.id, failure: null };
+      } catch (err) {
+        return {
+          id: story.id,
+          failure: `#${story.id} (${story.slug}): ${err.message}`,
+        };
+      }
+    },
+    // The terminal per-Story `agent::ready` PATCHes (Story #4952): each flip
+    // is an independent single-issue write, so the loop was serial only by
+    // construction. A latency fix, not a throughput one.
+    { concurrency: FANOUT_CONCURRENCY },
+  );
+  const readied = outcomes
+    .filter((outcome) => outcome.failure === null)
+    .map((outcome) => outcome.id);
+  const failed = outcomes
+    .filter((outcome) => outcome.failure !== null)
+    .map((outcome) => outcome.failure);
   if (failed.length > 0) {
     throw new Error(
       `[plan-persist] ${failed.length} Story(ies) were created with their ` +

package/.agents/scripts/lib/orchestration/plan-persist/supersede-ops.js

@@ -36,6 +36,10 @@
  */
 
 import { Logger } from '../../Logger.js';
+import {
+  concurrentMap,
+  FANOUT_CONCURRENCY,
+} from '../../util/concurrent-map.js';
 import { upsertStructuredComment } from '../ticketing.js';
 
 /** Structured-comment type marking a source issue as superseded. */
@@ -440,41 +444,94 @@ export async function closeSupersededTickets({
   const createdBySlug = new Map(
     (created ?? []).map((story) => [story.slug, story]),
   );
-
-  const report = emptyReport({ enabled: true, dryRun });
-
-  for (const story of stories ?? []) {
-    const createdStory = createdBySlug.get(story.slug);
-    for (const { id, note } of story.supersedes ?? []) {
+  const units = collectSupersedeUnits(stories, createdBySlug);
+
+  // Story #4952 — one bounded fan-out across distinct source tickets. The
+  // mapper inherits `closeOneSupersededTicket`'s never-throw contract (and
+  // resolves the two pre-write outcomes itself), so `concurrentMap`'s
+  // first-rejection-wins policy can never fire and no unit is abandoned
+  // because a sibling failed. Input order is preserved, so the report arrays
+  // below read exactly as the serial nested loop wrote them.
+  const outcomes = await concurrentMap(
+    units,
+    ({ id, note, createdStory }) => {
       if (!createdStory) {
-        report.skipped.push({ ticket: id, reason: 'story-not-created' });
-        continue;
-      }
-      if (dryRun) {
-        report.planned.push({ ticket: id, storySlug: createdStory.slug });
-        continue;
+        return { outcome: 'skipped', reason: 'story-not-created' };
       }
-      const result = await closeOneSupersededTicket({
+      if (dryRun) return { outcome: 'planned' };
+      return closeOneSupersededTicket({
         provider,
         id,
         note,
         story: createdStory,
         sourceTicketIds: sources,
       });
-      if (result.outcome === 'closed') {
-        report.closed.push(id);
-      } else if (result.outcome === 'skipped') {
-        report.skipped.push({ ticket: id, reason: result.reason });
-      } else {
-        report.failed.push({ ticket: id, reason: result.reason });
-      }
-    }
-  }
+    },
+    // The per-source-ticket close (Story #4952) fans out across **distinct**
+    // tickets — `assertSupersedePartition` has already failed the run closed
+    // if two Stories claim the same id, so no two units in flight can touch
+    // the same issue. Within one unit the probe → comment → close sequence
+    // stays strictly ordered: commenting on an issue the probe reported
+    // closed, or closing one the comment never landed on, is the whole
+    // failure mode this phase is careful about.
+    { concurrency: FANOUT_CONCURRENCY },
+  );
+
+  const report = emptyReport({ enabled: true, dryRun });
+  outcomes.forEach((result, index) => {
+    recordSupersedeOutcome(report, units[index], result);
+  });
 
   logSupersedeReport(report);
   return report;
 }
 
+/**
+ * Flatten the per-Story `supersedes[]` maps into one list of close units, in
+ * the nested iteration order the report arrays are expected to follow.
+ *
+ * @param {Array<{ slug: string, supersedes?: Array<{ id: number, note: string|null }> }>|undefined} stories
+ * @param {Map<string, { slug: string, id: number, title: string }>} createdBySlug
+ * @returns {Array<{ id: number, note: string|null, createdStory: object|undefined }>}
+ */
+function collectSupersedeUnits(stories, createdBySlug) {
+  const units = [];
+  for (const story of stories ?? []) {
+    const createdStory = createdBySlug.get(story.slug);
+    for (const { id, note } of story.supersedes ?? []) {
+      units.push({ id, note, createdStory });
+    }
+  }
+  return units;
+}
+
+/**
+ * File one unit's outcome onto the report.
+ *
+ * @param {SupersedeReport} report
+ * @param {{ id: number, createdStory: object|undefined }} unit
+ * @param {{ outcome: string, reason?: string }} result
+ * @returns {void}
+ */
+function recordSupersedeOutcome(report, unit, result) {
+  if (result.outcome === 'closed') {
+    report.closed.push(unit.id);
+    return;
+  }
+  if (result.outcome === 'planned') {
+    report.planned.push({
+      ticket: unit.id,
+      storySlug: unit.createdStory.slug,
+    });
+    return;
+  }
+  if (result.outcome === 'skipped') {
+    report.skipped.push({ ticket: unit.id, reason: result.reason });
+    return;
+  }
+  report.failed.push({ ticket: unit.id, reason: result.reason });
+}
+
 /**
  * Surface the supersede outcome on the console so a partial failure is
  * visible without reading the JSON envelope.