agent-horde 0.1.6 → 0.2.1

package/src/run/launch.ts

@@ -1,7 +1,9 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
 import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
 import type { DashboardOption, HordeRunArgs } from "../args.js";
 import { resolveDashboardPort, startDashboard } from "../dashboard.js";
-import { errorMessage, oneLine } from "../format.js";
+import { envNumber, errorMessage, oneLine } from "../format.js";
 import { initHordeDir } from "../ledger/files.js";
 import { resetMemberState } from "../member.js";
 import type { MemberState } from "../member.js";
@@ -9,17 +11,26 @@ import { hardStopMembers, requestCleanStop } from "./controls.js";
 import { finalizeRun, stopRunTimers } from "./finalize.js";
 import { recordLifecycle, recordPhase } from "./lifecycle.js";
 import { createRun, seedRun } from "./setup.js";
+import type { RunSpec } from "./setup.js";
 import { drainRepairs } from "./repair.js";
 import { startCollaborationScheduler } from "./schedule.js";
 import { startSolvedWatch } from "./solved.js";
 import { spawnOne, startStallWatch, superviseMember } from "./supervise.js";
 import type { RunLaunch } from "./supervise.js";
-import type { HordeRun, HordeState } from "./types.js";
+import { allocateSwarm, dealLayers, siblingHordes, swarmMarkdown } from "./swarm.js";
+import type { SwarmLayout } from "./swarm.js";
+import { noteRunFinalized, noteRunReported, writeSwarmRecord } from "./swarm-finalize.js";
+import { runPlanner } from "./synthesis.js";
+import { activeRuns, finalizingRuns, hordeLabel } from "./types.js";
+import type { HordeRun, HordeState, Swarm } from "./types.js";
 import { mountWidget, safeNotify, safeNotifyUi, unmountWidget } from "./ui.js";
 
 /** How long the canary preflight waits for the first token before declaring failure. */
 const CANARY_TIMEOUT_MS = Number(process.env.PI_HORDE_CANARY_MS) || 60_000;
 
+/** Pause between one horde's launch and the next, so H canaries do not hit the provider at once. */
+const SWARM_STAGGER_MS = envNumber("PI_HORDE_SWARM_STAGGER_MS", 5_000);
+
 type CanaryOutcome = "output" | "exit" | "timeout";
 
 function sleep(ms: number): Promise<void> {
@@ -53,7 +64,15 @@ async function ensureDashboard(
   try {
     state.dashboard = await startDashboard({
       port: resolveDashboardPort(option.port),
-      source: { hordeDir, getActive: () => state.active, getFinalizing: () => state.finalizing },
+      source: {
+        hordeDir,
+        getActive: () => (state.active ? activeRuns(state.active) : []),
+        getFinalizing: () => [
+          ...(state.active ? finalizingRuns(state.active) : []),
+          ...(state.finalizing ? finalizingRuns(state.finalizing) : []),
+        ],
+        getSwarm: () => state.active ?? state.finalizing,
+      },
       onError: (error) => {
         state.dashboard = undefined;
         safeNotifyUi(ctx, `dashboard error: ${error.message}; the run continues`, "error");
@@ -143,13 +162,18 @@ async function runCanary(launch: RunLaunch, spawns: Promise<void>[]): Promise<Ca
  * One retry distinguishes a flaky process from a dead provider.
  */
 async function preflightAndFanOut(launch: RunLaunch): Promise<void> {
-  const { state, run, ctx } = launch;
+  const { state, swarm, run, ctx } = launch;
+  if (run.stopped) {
+    // A stop arrived while this horde waited its turn: nothing to spawn, finalize now.
+    await finalizeRun(state, swarm, run);
+    return;
+  }
   const spawns: Promise<void>[] = [];
   const outcome = await runCanary(launch, spawns);
   if (run.stopped) {
     // A stop arrived during preflight: do not fan out, finalize now.
     await Promise.all(spawns);
-    await finalizeRun(state, run);
+    await finalizeRun(state, swarm, run);
     return;
   }
   if (outcome === "output") {
@@ -175,48 +199,169 @@ async function preflightAndFanOut(launch: RunLaunch): Promise<void> {
   if (run.scheduleTimer) clearInterval(run.scheduleTimer);
   run.scheduleTimer = undefined;
   await drainRepairs(launch);
-  await finalizeRun(state, run);
+  await finalizeRun(state, swarm, run);
+}
+
+/** The hordes' workstreams and plans: dealt layers, the planner's slices, or nothing for one horde. */
+interface Partition {
+  slices: string[] | undefined;
+  /** Per-horde plan layers, indexed by horde - 1. */
+  plans: Array<string[] | undefined>;
+}
+
+/** Note the planner's failure where the dashboard and the operator can find it. */
+function writeNotStarted(layout: SwarmLayout, reason: string): void {
+  if (!layout.dir) return;
+  fs.writeFileSync(path.join(layout.dir, "SWARM.md"), `# Swarm ${layout.id}\n\nNot started: ${reason}\n`, "utf-8");
+}
+
+/**
+ * Decide what each horde works on. `plan` layers go round-robin to the hordes and each horde
+ * runs its share as its own plan; a plain prompt is split by the planner helper, and a planner
+ * that cannot produce H distinct workstreams stops the launch rather than running H hordes on
+ * one slice.
+ */
+async function partitionTask(
+  state: HordeState,
+  ctx: ExtensionCommandContext,
+  parsed: HordeRunArgs,
+  layout: SwarmLayout,
+): Promise<Partition | undefined> {
+  if (parsed.hordes === 1) return { slices: undefined, plans: [parsed.plan] };
+  if (parsed.plan) {
+    const dealt = dealLayers(parsed.plan, parsed.hordes);
+    return { slices: dealt.map((layers) => layers.join("; ")), plans: dealt };
+  }
+  ctx.ui.notify(`swarm ${layout.id}: planner is partitioning the task into ${parsed.hordes} workstreams`, "info");
+  const model = ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : undefined;
+  const result = await runPlanner(state, { dir: layout.dir!, cwd: ctx.cwd, model, prompt: parsed.prompt, hordes: parsed.hordes });
+  if ("error" in result) {
+    writeNotStarted(layout, result.error);
+    ctx.ui.notify(`swarm ${layout.id} not started: ${result.error}`, "error");
+    return undefined;
+  }
+  return { slices: result.slices, plans: Array.from({ length: parsed.hordes }, () => undefined) };
+}
+
+function buildSwarm(
+  ctx: ExtensionCommandContext,
+  parsed: HordeRunArgs,
+  layout: SwarmLayout,
+  partition: Partition,
+  dashboardUrl: string | undefined,
+): { swarm: Swarm; launches: Array<{ run: HordeRun; roles: string[] }> } {
+  const launches = layout.runIds.map((id, index) => {
+    const spec: RunSpec = {
+      id,
+      hordeDir: layout.hordeDirs[index]!,
+      runDir: layout.runDirs[index]!,
+      horde: index + 1,
+      hordes: parsed.hordes,
+      swarmId: layout.id,
+      slice: partition.slices?.[index],
+      plan: partition.plans[index],
+      siblings: siblingHordes(layout, partition.slices, index + 1),
+    };
+    return createRun(ctx, parsed, spec, dashboardUrl);
+  });
+  const swarm: Swarm = {
+    id: layout.id,
+    dir: layout.dir,
+    hordeDir: layout.hordeDirs.length === 1 ? layout.hordeDirs[0]! : path.resolve(layout.dir!, "..", ".."),
+    cwd: ctx.cwd,
+    prompt: parsed.prompt,
+    hordes: parsed.hordes,
+    count: parsed.count,
+    startedAt: Date.now(),
+    runs: launches.map((launch) => launch.run),
+    plan: parsed.plan,
+    partition: partition.slices,
+    untilStop: parsed.untilStop === true ? true : undefined,
+    dashboardUrl,
+  };
+  return { swarm, launches };
+}
+
+/** Seed one horde's ledgers and record its start; throws on a ledger failure. */
+async function seedHorde(swarm: Swarm, run: HordeRun, roles: string[]): Promise<void> {
+  await seedRun(run);
+  await recordLifecycle(run, "run-start", {
+    count: run.count, model: run.model, verifier: run.verifier, plan: run.plan, untilStop: run.untilStop,
+    roles: roles.map((role) => oneLine(role, 120)), dashboardUrl: run.dashboardUrl,
+    ...(swarm.hordes > 1 ? { swarm: swarm.id, horde: run.horde, hordes: swarm.hordes, slice: run.slice } : {}),
+  });
+  await recordPhase(run, "discovery", "start");
+}
+
+function announceStart(ctx: ExtensionCommandContext, swarm: Swarm): void {
+  if (swarm.hordes === 1) {
+    const run = swarm.runs[0]!;
+    const plural = run.count === 1 ? "" : "s";
+    ctx.ui.notify(`horde ${run.id} started with ${run.count} member${plural} in ${run.hordeDir}`, "info");
+    if (run.dashboardUrl) ctx.ui.notify(`horde ${run.id} dashboard: ${run.dashboardUrl}runs/${run.id}`, "info");
+    return;
+  }
+  ctx.ui.notify(
+    `swarm ${swarm.id} started: ${swarm.hordes} hordes of ${swarm.count} members in ${swarm.dir}, ` +
+      `${Math.round(SWARM_STAGGER_MS / 1000)}s apart`,
+    "info",
+  );
+  for (const [index, slice] of (swarm.partition ?? []).entries()) {
+    ctx.ui.notify(`${hordeLabel(index + 1)}: ${oneLine(slice, 160)}`, "info");
+  }
+  if (swarm.dashboardUrl) ctx.ui.notify(`swarm ${swarm.id} dashboard: ${swarm.dashboardUrl}swarms/${swarm.id}`, "info");
+}
+
+/** Start the hordes' preflights a few seconds apart; a stop during the wait is honoured. */
+async function staggerLaunches(state: HordeState, launches: RunLaunch[]): Promise<void> {
+  for (const [index, launch] of launches.entries()) {
+    if (index > 0 && !state.disposed) await sleep(SWARM_STAGGER_MS);
+    if (state.disposed) return;
+    void preflightAndFanOut(launch).catch((error) => {
+      safeNotify(launch.run, `horde ${launch.run.id} failed: ${errorMessage(error)}`, "error");
+      // The horde is down for good; let the swarm account for it so the slot and widget free.
+      noteRunReported(state, launch.swarm, launch.run);
+      void noteRunFinalized(state, launch.swarm, launch.run);
+    });
+  }
 }
 
-async function launchRun(
+async function launchSwarm(
   state: HordeState,
   ctx: ExtensionCommandContext,
   parsed: HordeRunArgs,
   hordeDir: string,
 ): Promise<void> {
-  const { run, roles } = createRun(ctx, parsed, hordeDir, state.dashboard?.url);
-  // Claim the run synchronously, before the first await, so a second
+  const layout = allocateSwarm(hordeDir, parsed.hordes, Date.now());
+  const partition = await partitionTask(state, ctx, parsed, layout);
+  if (!partition || state.disposed) return;
+  const { swarm, launches } = buildSwarm(ctx, parsed, layout, partition, state.dashboard?.url);
+  // Claim the slot synchronously, before the first await, so a second
   // /agent-horde invocation in the same tick sees `active` and is refused.
-  state.active = run;
-  if (ctx.hasUI) mountWidget(run, ctx.ui);
-  startDeadline(run, ctx);
-  startSolvedWatch(run, ctx);
+  state.active = swarm;
+  if (ctx.hasUI) mountWidget(swarm, ctx.ui);
+  for (const run of swarm.runs) {
+    startDeadline(run, ctx);
+    startSolvedWatch(run, ctx);
+  }
   try {
-    await seedRun(run);
-    await recordLifecycle(run, "run-start", {
-      count: run.count, model: run.model, verifier: run.verifier, plan: run.plan, untilStop: run.untilStop,
-      roles: roles.map((role) => oneLine(role, 120)), dashboardUrl: run.dashboardUrl,
-    });
-    await recordPhase(run, "discovery", "start");
+    if (swarm.dir) {
+      fs.writeFileSync(path.join(swarm.dir, "SWARM.md"), swarmMarkdown(swarm), "utf-8");
+      writeSwarmRecord(swarm);
+    }
+    for (const { run, roles } of launches) await seedHorde(swarm, run, roles);
   } catch (error) {
-    stopRunTimers(run);
-    unmountWidget(run);
-    if (state.active === run) state.active = undefined;
-    ctx.ui.notify(`horde ${run.id} failed to start: ${errorMessage(error)}`, "error");
+    for (const run of swarm.runs) stopRunTimers(run);
+    unmountWidget(swarm);
+    if (state.active === swarm) state.active = undefined;
+    ctx.ui.notify(`horde ${swarm.id} failed to start: ${errorMessage(error)}`, "error");
     return;
   }
-  void preflightAndFanOut({ state, run, ctx, roles }).catch((error) => {
-    safeNotify(run, `horde ${run.id} failed: ${errorMessage(error)}`, "error");
-    if (state.active === run) state.active = undefined;
-  });
-  const plural = run.count === 1 ? "" : "s";
-  ctx.ui.notify(`horde ${run.id} started with ${run.count} member${plural} in ${hordeDir}`, "info");
-  if (run.dashboardUrl) {
-    ctx.ui.notify(`horde ${run.id} dashboard: ${run.dashboardUrl}runs/${run.id}`, "info");
-  }
+  void staggerLaunches(state, launches.map(({ run, roles }) => ({ state, swarm, run, ctx, roles })));
+  announceStart(ctx, swarm);
 }
 
-export async function startRun(
+export async function startSwarm(
   state: HordeState,
   ctx: ExtensionCommandContext,
   parsed: HordeRunArgs,
@@ -225,7 +370,7 @@ export async function startRun(
   try {
     const hordeDir = initHordeDir(ctx.cwd);
     if (!(await ensureDashboard(state, ctx, hordeDir, parsed.dashboard))) return;
-    await launchRun(state, ctx, parsed, hordeDir);
+    await launchSwarm(state, ctx, parsed, hordeDir);
   } finally {
     state.starting = false;
   }

package/src/run/repair.ts

@@ -4,6 +4,7 @@ import { randomUUID } from "node:crypto";
 import { inboxFor } from "../tool/inbox.js";
 import type { PeerMessage } from "../ledger/collaboration.js";
 import { recordEvent } from "../ledger/events.js";
+import type { EventContext } from "../ledger/events.js";
 import { recordPhase } from "./lifecycle.js";
 import { envNumber } from "../format.js";
 import { resetMemberState } from "../member.js";
@@ -24,11 +25,94 @@ function repairPriority(message: PeerMessage): number {
   return 3;
 }
 
-function queueFor(launch: RunLaunch, member: MemberState): PeerMessage[] {
-  return inboxFor({ hordeDir: launch.run.hordeDir, runId: launch.run.id, actor: member.name })
+/** The repair phase's scheduling state, shared by every round. */
+interface RepairPhase {
+  launch: RunLaunch;
+  floors: RepairFloors;
+  /**
+   * Threads a member's completed attempts left exactly as they found them, by member then
+   * message id: the signature the thread was stuck at and how many consecutive attempts left it
+   * there. The thread is held back from that member once the count reaches `UNMOVED_ATTEMPTS`,
+   * until its signature changes, which only another member's action or a new response can cause.
+   */
+  stalled: Map<string, Map<string, UnmovedThread>>;
+}
+
+interface UnmovedThread {
+  signature: string;
+  attempts: number;
+}
+
+/**
+ * Completed attempts that may leave a thread untouched before it is held. One miss is routine: a
+ * member that answered the rest of its queue, or re-answered the task, often clears the thread on
+ * its next attempt within seconds. Two misses on the same signature are the loop this guards.
+ */
+const UNMOVED_ATTEMPTS = 2;
+
+function memberContext(phase: RepairPhase, member: MemberState): EventContext {
+  return { hordeDir: phase.launch.run.hordeDir, runId: phase.launch.run.id, actor: member.name };
+}
+
+/**
+ * What an attempt can change on a thread: its state and the revision it was answered with.
+ * Delivery receipts and a re-worded response on the same revision are not progress, so neither
+ * the inbox read of a fresh attempt nor a recipient repeating itself re-qualifies a thread.
+ */
+function threadSignature(m: PeerMessage): string {
+  return `${m.state}:${m.responseRevision ?? ""}:${m.responseTarget ?? ""}`;
+}
+
+function isStalled(phase: RepairPhase, member: MemberState, m: PeerMessage): boolean {
+  const unmoved = phase.stalled.get(member.name)?.get(m.id);
+  return unmoved !== undefined && unmoved.signature === threadSignature(m) && unmoved.attempts >= UNMOVED_ATTEMPTS;
+}
+
+/** The threads currently held back from the member, in inbox order. */
+function heldThreads(phase: RepairPhase, member: MemberState): string[] {
+  return inboxFor(memberContext(phase, member)).filter((m) => isStalled(phase, member, m)).map((m) => m.id);
+}
+
+/** The threads an attempt for the member would be handed: its inbox minus the threads held back. */
+function queueFor(phase: RepairPhase, member: MemberState): PeerMessage[] {
+  return inboxFor(memberContext(phase, member)).filter((m) => !isStalled(phase, member, m))
     .sort((a, b) => repairPriority(a) - repairPriority(b) || a.createdAt.localeCompare(b.createdAt));
 }
 
+/**
+ * Counts, per thread, the consecutive completed attempts that left it as they found it; a thread
+ * the attempt moved starts over. A killed attempt counts nothing: it may have been about to act.
+ * Returns the ids this attempt left unchanged.
+ */
+function noteUnchanged(phase: RepairPhase, member: MemberState, queue: PeerMessage[]): string[] {
+  if (member.endCause !== "completed") return [];
+  const after = new Map(inboxFor(memberContext(phase, member)).map((m) => [m.id, threadSignature(m)]));
+  const unmoved = phase.stalled.get(member.name) ?? new Map<string, UnmovedThread>();
+  const unchanged: string[] = [];
+  for (const m of queue) {
+    const signature = threadSignature(m);
+    if (after.get(m.id) !== signature) { unmoved.delete(m.id); continue; }
+    const previous = unmoved.get(m.id);
+    unmoved.set(m.id, { signature, attempts: previous?.signature === signature ? previous.attempts + 1 : 1 });
+    unchanged.push(m.id);
+  }
+  phase.stalled.set(member.name, unmoved);
+  return unchanged;
+}
+
+/**
+ * Once every thread in the member's inbox is one its own completed attempt could not move, later
+ * rounds skip the member and its threads drop out of the demand that sizes the shares, until
+ * another member changes one. Recorded so the post-mortem sees why the member stopped appearing.
+ */
+async function recordStalled(phase: RepairPhase, member: MemberState): Promise<void> {
+  const held = heldThreads(phase, member);
+  if (!held.length || queueFor(phase, member).length) return;
+  const { run } = phase.launch;
+  run.repairStalled = [...new Set([...(run.repairStalled ?? []), member.name])];
+  await recordEvent({ ...memberContext(phase, member), actor: "parent", attempt: "scheduler" }, "repair-skipped", { payload: { reason: "no-progress", members: [{ member: member.name, messages: held }] } });
+}
+
 /**
  * What a queue costs in floors: one cold session covers the administrative threads in it, and
  * each review request or blocking challenge is an inspect/attach/review cycle of about a floor
@@ -114,10 +198,10 @@ export function repairBudget(demands: number[], floors: RepairFloors, rounds: nu
 
 type SkipReason = "insufficient-budget" | "insufficient-time";
 
-async function recordSkipped(launch: RunLaunch, pending: MemberState[], reason: SkipReason, detail: Record<string, number>): Promise<void> {
-  const { run } = launch;
+async function recordSkipped(phase: RepairPhase, pending: MemberState[], reason: SkipReason, detail: Record<string, number>): Promise<void> {
+  const { run } = phase.launch;
   const ctx = { hordeDir: run.hordeDir, runId: run.id, actor: "parent", attempt: "scheduler" };
-  const members = pending.map((member) => ({ member: member.name, messages: queueFor(launch, member).map((m) => m.id) }))
+  const members = pending.map((member) => ({ member: member.name, messages: queueFor(phase, member).map((m) => m.id) }))
     .filter((entry) => entry.messages.length);
   if (!members.length) return;
   run.repairSkipped = [...new Set([...(run.repairSkipped ?? []), ...members.map((entry) => entry.member)])];
@@ -143,10 +227,11 @@ interface FirstTurnCost {
  * share is the allowance recorded and the deadline is derived from it once, so the record cannot
  * read a millisecond under the floor the share was raised to.
  */
-async function repairMember(launch: RunLaunch, member: MemberState, ms: number, limits: RepairLimits, floors: RepairFloors): Promise<void> {
+async function repairMember(phase: RepairPhase, member: MemberState, ms: number, limits: RepairLimits): Promise<void> {
+  const { launch, floors } = phase;
   const { run } = launch;
-  const ctx = { hordeDir: run.hordeDir, runId: run.id, actor: member.name };
-  const queue = queueFor(launch, member);
+  const ctx = memberContext(phase, member);
+  const queue = queueFor(phase, member);
   if (!queue.length || stopReason(launch)) return;
   const prior = { turns: member.turns, toolCalls: member.toolCalls, usage: { ...member.usage }, startedAt: member.startedAt };
   resetMemberState(member);
@@ -181,7 +266,8 @@ async function repairMember(launch: RunLaunch, member: MemberState, ms: number,
   const timeout = setTimeout(() => halt("time"), Math.max(1, deadline - Date.now()));
   const watchdog = setInterval(check, 100);
   try {
-    await recordEvent({ ...ctx, attempt: member.attempt }, "repair-start", { payload: { messages: queue.map((m) => m.id), demand: repairDemand(queue), prior, allowance, floors } });
+    const held = heldThreads(phase, member);
+    await recordEvent({ ...ctx, attempt: member.attempt }, "repair-start", { payload: { messages: queue.map((m) => m.id), demand: repairDemand(queue), prior, allowance, floors, ...(held.length ? { held } : {}) } });
     if (stopReason(launch) || Date.now() >= deadline) {
       halt(stopReason(launch) ?? "time");
       member.status = "stopped";
@@ -191,10 +277,12 @@ async function repairMember(launch: RunLaunch, member: MemberState, ms: number,
       // Enforce a share on every usage update, rather than waiting for the next poll.
       await spawnOne(launch, member, undefined, check);
     }
+    const unchanged = noteUnchanged(phase, member, queue);
     await recordEvent({ ...ctx, attempt: member.attempt }, "repair-end", { payload: {
       cause: member.endCause, stopReason: member.repairStopReason, allowance, usage: member.usage, turns: member.turns, firstTurn,
-      messages: queue.map((m) => m.id),
+      messages: queue.map((m) => m.id), unchanged,
     } });
+    if (unchanged.length) await recordStalled(phase, member);
   } finally {
     clearTimeout(timeout);
     clearInterval(watchdog);
@@ -223,19 +311,42 @@ export async function drainRepairs(launch: RunLaunch): Promise<void> {
   }
 }
 
-function eligibleMembers(launch: RunLaunch): Array<{ member: MemberState; queue: PeerMessage[] }> {
-  return launch.run.members.map((member) => ({ member, queue: queueFor(launch, member) })).filter((entry) => entry.queue.length)
+function eligibleMembers(phase: RepairPhase): Array<{ member: MemberState; queue: PeerMessage[] }> {
+  return phase.launch.run.members.map((member) => ({ member, queue: queueFor(phase, member) })).filter((entry) => entry.queue.length)
     .sort((a, b) => repairPriority(a.queue[0]!) - repairPriority(b.queue[0]!) ||
       (a.member.repairs ?? 0) - (b.member.repairs ?? 0) || a.queue[0]!.createdAt.localeCompare(b.queue[0]!.createdAt));
 }
 
+/** The queue a member was handed this round, by message id, at the signature it was handed. */
+type HandedQueues = Map<string, Map<string, string>>;
+
+/**
+ * True when every blocking thread left belongs to a member already served this round, and one of
+ * them is new or moved since that member was handed its queue. A correction answered mid-round
+ * leaves its requester to recheck it; finishing the round on administrative rechecks first spent
+ * the clock that recheck needed, so when a later round exists it starts now. A blocker the member
+ * left as it was handed is no reason to serve it again early; the last round serves everyone.
+ */
+export function blockedOnServed(eligible: Array<{ member: MemberState; queue: PeerMessage[] }>, handed: HandedQueues): boolean {
+  let moved = false;
+  for (const { member, queue } of eligible) {
+    const seen = handed.get(member.name);
+    for (const m of queue) {
+      if (repairPriority(m) !== 0) continue;
+      if (!seen) return false;
+      if (seen.get(m.id) !== threadSignature(m)) moved = true;
+    }
+  }
+  return moved;
+}
+
 async function runRepairRounds(launch: RunLaunch, begin: (budget: RepairBudget) => Promise<void>): Promise<void> {
   const { run } = launch;
   const rounds = Math.max(0, Math.min(6, envNumber("PI_HORDE_REPAIR_ROUNDS", 4)));
   if (rounds === 0 || stopReason(launch)) return;
-  const floors = repairFloors();
+  const phase: RepairPhase = { launch, floors: repairFloors(), stalled: new Map() };
   await scheduleCollaboration(launch, { settleMs: 0, liveOnly: false });
-  const budget = repairBudget(eligibleMembers(launch).map((entry) => repairDemand(entry.queue)), floors, rounds);
+  const budget = repairBudget(eligibleMembers(phase).map((entry) => repairDemand(entry.queue)), phase.floors, rounds);
   if (budget.queued === 0) return;
   const deadline = Date.now() + budget.ms;
   const exhausted = () => stopReason(launch) || Date.now() >= deadline || (run.repairTokens ?? 0) >= budget.tokens;
@@ -245,34 +356,35 @@ async function runRepairRounds(launch: RunLaunch, begin: (budget: RepairBudget)
     // One attempt per member per round, picked in priority order re-read before each attempt:
     // a thread that became blocking during an attempt moves its recipient ahead of the
     // administrative queues that were ordered first at round start.
-    const served = new Set<string>();
+    const handed: HandedQueues = new Map();
     for (;;) {
       if (exhausted()) return;
-      const pending = eligibleMembers(launch).filter((entry) => !served.has(entry.member.name));
-      if (!pending.length) break;
+      const eligible = eligibleMembers(phase);
+      const pending = eligible.filter((entry) => !handed.has(entry.member.name));
+      if (!pending.length || (round < rounds - 1 && blockedOnServed(eligible, handed))) break;
       const next = pending[0]!;
       const pendingUnits = pending.reduce((sum, entry) => sum + repairDemand(entry.queue), 0);
       const units = repairDemand(next.queue);
       const unserved = pending.map((entry) => entry.member);
       const remaining = budget.tokens - (run.repairTokens ?? 0);
       const laterRounds = rounds - round - 1;
-      const tokens = repairAllowance(repairShare(remaining, pendingUnits, laterRounds, units), remaining, floors.tokens);
+      const tokens = repairAllowance(repairShare(remaining, pendingUnits, laterRounds, units), remaining, phase.floors.tokens);
       if (tokens === undefined) {
         await begin(budget);
-        await recordSkipped(launch, unserved, "insufficient-budget", { remaining, floor: floors.tokens });
+        await recordSkipped(phase, unserved, "insufficient-budget", { remaining, floor: phase.floors.tokens });
         return;
       }
       const remainingMs = Math.max(0, deadline - Date.now());
-      const fundable = Math.floor(remaining / Math.max(1, floors.tokens));
-      const ms = repairAllowance(repairTimeShare(remainingMs, pendingUnits, laterRounds, fundable, units), remainingMs, floors.ms);
+      const fundable = Math.floor(remaining / Math.max(1, phase.floors.tokens));
+      const ms = repairAllowance(repairTimeShare(remainingMs, pendingUnits, laterRounds, fundable, units), remainingMs, phase.floors.ms);
       if (ms === undefined || ms < 1) {
         await begin(budget);
-        await recordSkipped(launch, unserved, "insufficient-time", { remainingMs, floorMs: floors.ms });
+        await recordSkipped(phase, unserved, "insufficient-time", { remainingMs, floorMs: phase.floors.ms });
         return;
       }
       await begin(budget);
-      served.add(next.member.name);
-      await repairMember(launch, next.member, ms, { tokens, turns: budget.turns }, floors);
+      handed.set(next.member.name, new Map(next.queue.map((m) => [m.id, threadSignature(m)])));
+      await repairMember(phase, next.member, ms, { tokens, turns: budget.turns });
     }
   }
 }

package/src/run/setup.ts

@@ -4,15 +4,31 @@ import { createHash } from "node:crypto";
 import { fileURLToPath } from "node:url";
 import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
 import type { HordeRunArgs } from "../args.js";
-import { formatRunId } from "../format.js";
 import { appendRunHeading } from "../ledger/board.js";
 import { listFacts } from "../ledger/facts.js";
 import { ensureFindingIds } from "../ledger/findings.js";
 import { claimIdea, listIdeas, seedPlan } from "../ledger/ideas.js";
 import { newMemberState } from "../member.js";
+import type { SiblingHorde } from "../member.js";
 import { VERIFIER_ROLE, memberSystemPrompt, taskMarkdown } from "../prompt.js";
 import type { HordeRun } from "./types.js";
 
+/** Where one horde of a swarm runs and what it was given. */
+export interface RunSpec {
+  id: string;
+  hordeDir: string;
+  runDir: string;
+  /** 1-based horde index in the swarm. */
+  horde: number;
+  hordes: number;
+  swarmId: string;
+  /** The horde's workstream when the swarm partitioned the task. */
+  slice?: string | undefined;
+  /** The `plan` layers this horde runs as its own plan. */
+  plan?: string[] | undefined;
+  siblings: SiblingHorde[];
+}
+
 function planMarkdown(id: string, layers: string[], count: number): string {
   return [
     `# Plan ${id}`,
@@ -55,32 +71,17 @@ function defaultRoles(count: number, noVerifier: boolean): string[] {
   return roles;
 }
 
-function allocateRunDir(hordeDir: string, startedAt: number): { id: string; runDir: string } {
-  const base = formatRunId(new Date(startedAt));
-  for (let attempt = 0; ; attempt++) {
-    const id = attempt === 0 ? base : `${base}-${attempt}`;
-    const runDir = path.join(hordeDir, "runs", id);
-    try {
-      fs.mkdirSync(runDir);
-    } catch (error) {
-      if ((error as NodeJS.ErrnoException).code === "EEXIST") continue;
-      throw error;
-    }
-    fs.mkdirSync(path.join(runDir, "sessions"), { recursive: true });
-    return { id, runDir };
-  }
-}
-
-/** Allocate the run directory, write TASK.md and SYSTEM.md, and build the run record. */
+/** Write TASK.md and SYSTEM.md into the allocated run directory and build the run record. */
 export function createRun(
   ctx: ExtensionCommandContext,
   parsed: HordeRunArgs,
-  hordeDir: string,
+  spec: RunSpec,
   dashboardUrl: string | undefined,
 ): { run: HordeRun; roles: string[] } {
-  const { count, prompt, plan } = parsed;
+  const { count, prompt } = parsed;
+  const { id, hordeDir, runDir, plan } = spec;
   const startedAt = Date.now();
-  const { id, runDir } = allocateRunDir(hordeDir, startedAt);
+  fs.mkdirSync(path.join(runDir, "sessions"), { recursive: true });
   const model = ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : undefined;
   const roles =
     plan && plan.length > 0
@@ -88,18 +89,27 @@ export function createRun(
       : defaultRoles(count, parsed.noVerifier === true);
   const verifierIndex = roles.indexOf(VERIFIER_ROLE);
   const verifier = verifierIndex === -1 ? undefined : `member-${verifierIndex + 1}`;
+  const swarm = spec.hordes > 1 ? { id: spec.swarmId, horde: spec.horde, hordes: spec.hordes, slice: spec.slice } : undefined;
   fs.writeFileSync(
     path.join(runDir, "TASK.md"),
     taskMarkdown({
-      id, prompt, count, model, thinkingLevel: ctx.thinkingLevel, startedAt, verifier, untilStop: parsed.untilStop,
+      id, prompt, count, model, thinkingLevel: ctx.thinkingLevel, startedAt, verifier, untilStop: parsed.untilStop, swarm,
     }),
     "utf-8",
   );
-  fs.writeFileSync(path.join(runDir, "SYSTEM.md"), memberSystemPrompt({ hordeDir, count }), "utf-8");
+  fs.writeFileSync(
+    path.join(runDir, "SYSTEM.md"),
+    memberSystemPrompt({ hordeDir, count, siblings: spec.siblings }),
+    "utf-8",
+  );
   fs.mkdirSync(path.join(runDir, "artifacts"), { recursive: true });
-  writeManifest(runDir, { model, thinking: ctx.thinkingLevel, roles, prompt });
+  writeManifest(runDir, { model, thinking: ctx.thinkingLevel, roles, prompt, swarm });
   const run: HordeRun = {
     id, hordeDir, runDir, prompt, count, startedAt,
+    horde: spec.horde,
+    hordes: spec.hordes,
+    slice: spec.slice,
+    siblings: spec.siblings,
     members: Array.from({ length: count }, (_, i) => newMemberState(i + 1)),
     controller: new AbortController(),
     controllers: new Map(),
@@ -131,7 +141,7 @@ function writeManifest(runDir: string, settings: Record<string, unknown>): void
     }
   };
   visit(root);
-  const config = Object.fromEntries(Object.entries(process.env).filter(([key]) => /^PI_HORDE_(?:REPAIR|REMIND|STALL|RESET|MAX|ALLOW|SOLVED|INBOX|CANARY|DELIVERABLE|POSTMORTEM|CLEANUP)/.test(key)));
+  const config = Object.fromEntries(Object.entries(process.env).filter(([key]) => /^PI_HORDE_(?:REPAIR|REMIND|STALL|RESET|MAX|ALLOW|SOLVED|INBOX|CANARY|DELIVERABLE|POSTMORTEM|CLEANUP|SWARM|PLANNER)/.test(key)));
   fs.writeFileSync(path.join(runDir, "MANIFEST.json"), JSON.stringify({ protocol: 1, codeHash: code.digest("hex"),
     promptHash: createHash("sha256").update(String(settings.prompt)).digest("hex"),
     systemHash: createHash("sha256").update(fs.readFileSync(path.join(runDir, "SYSTEM.md"))).digest("hex"), settings, config }, null, 2));