@pify/swarm 0.7.7 → 0.9.0

package/README.md

@@ -18,7 +18,7 @@ The catch is that "independent" is usually a small lie — the items do not depe
 
 | Parameter | Type | Notes |
 |---|---|---|
-| `items` | string[], 1–12 | One task per item; four run at a time, the rest queue |
+| `items` | array, 1–12 | A plain string per task, or `{task, id, needs}` to declare a dependency |
 | `context` | string, optional | Prepended to every item, so shared constraints are written once |
 | `agent` | string, optional | Force one agent type for all items instead of routing |
 | `isolation` | `"worktree"`, optional | Give each item its own git worktree — use it when items write |
@@ -27,6 +27,24 @@ The catch is that "independent" is usually a small lie — the items do not depe
 
 Blocking by default: returns `N done, M error` plus a per-item report.
 
+### Dependencies: `needs`
+
+An item can be a plain string (independent, as before) or an object that declares what it depends on:
+
+```json
+{
+  "items": [
+    { "id": "mig",     "task": "write the DB migration for the users table" },
+    { "id": "callers", "task": "update every caller of the old schema", "needs": ["mig"] },
+    { "id": "tests",   "task": "run the suite and fix what broke",       "needs": ["callers"] }
+  ]
+}
+```
+
+An item starts only once its `needs` have finished, and each upstream item's output is prepended to it as a `## Output of <id>` block — so the thing the coordinator used to forget, passing X to the step that needs it, happens by construction. Independent items still run in parallel up to the concurrency cap; a chain runs in order; a diamond joins after both branches. Ids default to `t1`, `t2`, … when you omit them.
+
+The whole graph is checked **before anything spawns**: a cycle, a self-edge, a duplicate id, or a reference to an unknown id is rejected outright, so a bad graph costs nothing. A flat list of strings has no edges and behaves exactly as it always did.
+
 ### `swarm_status`
 
 | Parameter | Type | Notes |
@@ -77,7 +95,7 @@ The catalog is the same `.pi/agents/*.md` one [`@pify/subagent`](https://github.
 
 ## Where this sits in the suite
 
-[`@pify/subagent`](https://github.com/pifydev/subagent) is one child and one task. `@pify/swarm` is many independent items at once. [`@pify/workflow`](https://github.com/pifydev/workflow) is deterministic scripted orchestration for when the steps genuinely depend on each other. Pick the smallest one that fits.
+[`@pify/subagent`](https://github.com/pifydev/subagent) is one child and one task. `@pify/swarm` is many items at once — independent, or wired together with a declarative `needs` graph (fan-out, chains, joins). [`@pify/workflow`](https://github.com/pifydev/workflow) is for when orchestration needs real control flow — loops, conditionals, retries, fan-out computed at run time — that a static graph can't express. Pick the smallest one that fits.
 
 ## License
 

package/extensions/swarm.ts

@@ -26,6 +26,7 @@ import { Type } from "typebox";
 
 import { BUILTIN_AGENTS } from "../src/builtin.ts";
 import { withUiLock } from "../src/ui-lock.ts";
+import { LoopGuard } from "../src/loop-guard.ts";
 import {
   consentQuestion,
   decideConsent,
@@ -41,6 +42,8 @@ import { formatInbox, mailboxDir, mailboxPrompt, postMessage, readInbox } from "
 import { parseAgentFile } from "../src/frontmatter.ts";
 import { buildReport, buildStatusLine } from "../src/report.ts";
 import { routeItem } from "../src/routing.ts";
+import { normalizeItems } from "../src/graph.ts";
+import { runGraph } from "../src/schedule.ts";
 import { buildWidgetLines } from "../src/widget.ts";
 import {
   DEFAULT_CONCURRENCY,
@@ -183,6 +186,7 @@ export default function swarm(pi: ExtensionAPI) {
     let session: AgentSession | null = null;
     let unsubscribe: (() => void) | null = null;
     let releaseLive: (() => void) | null = null;
+    let stallReason: string | null = null;
     try {
       let model = ctx.model ?? null;
       if (def.model) {
@@ -230,11 +234,37 @@ export default function swarm(pi: ExtensionAPI) {
       session = created.session;
       releaseLive = live.register(runId, session);
 
+      const guard = new LoopGuard();
       unsubscribe = session.subscribe((event) => {
-        if (event.type === "message_end" && (event as { message?: { role?: string } }).message?.role === "assistant") {
+        const message = (
+          event as {
+            message?: {
+              role?: string;
+              usage?: { totalTokens?: number };
+              content?: Array<{ type?: string; text?: string }>;
+            };
+          }
+        ).message;
+        if (event.type === "message_end" && message?.role === "assistant") {
           item.turns++;
-          const usage = (event as { message?: { usage?: { totalTokens?: number } } }).message?.usage;
+          const usage = message.usage;
           if (usage && typeof usage.totalTokens === "number") item.tokens += usage.totalTokens;
+
+          // Stop an item that is spinning — restating itself without acting —
+          // rather than letting it run to the turn cap. See loop-guard.ts.
+          if (!stallReason && Array.isArray(message.content)) {
+            const usedTool = message.content.some((c) => c.type === "toolCall");
+            const turnText = message.content
+              .filter((c) => c.type === "text" && typeof c.text === "string")
+              .map((c) => c.text)
+              .join("\n");
+            const verdict = guard.observe({ text: turnText, usedTool });
+            if (verdict.stalled) {
+              stallReason = verdict.reason ?? "no progress";
+              void session?.abort().catch(() => {});
+            }
+          }
+
           renderWidget();
           if (item.turns >= def.maxTurns) void session?.abort().catch(() => {});
         }
@@ -255,9 +285,18 @@ export default function swarm(pi: ExtensionAPI) {
         .join("\n")
         .trim();
 
-      item.result = text || null;
-      item.status =
-        last?.stopReason === "aborted" ? "aborted" : last?.stopReason === "error" ? "error" : "done";
+      // An item the loop guard stopped gave up rather than concluded — mark it
+      // so the aggregated report does not read it as a finished answer.
+      item.result = stallReason
+        ? `${text ? `${text}\n\n` : ""}[stopped: no progress — the agent ${stallReason}]`
+        : text || null;
+      item.status = stallReason
+        ? "aborted"
+        : last?.stopReason === "aborted"
+          ? "aborted"
+          : last?.stopReason === "error"
+            ? "error"
+            : "done";
       if (item.status === "error") item.error = text || "child session error";
       // A child that stopped cleanly and said nothing has not answered — the
       // fix subagent already carries and this executor never received. Left
@@ -291,7 +330,11 @@ export default function swarm(pi: ExtensionAPI) {
     }
   }
 
-  /** Pool executor: at most DEFAULT_CONCURRENCY items in flight. */
+  /**
+   * Readiness scheduler: run each item as soon as its `needs` are done, up to
+   * DEFAULT_CONCURRENCY at once. A flat run (no needs anywhere) makes every
+   * item ready immediately, so this is identical to the old parallel pool.
+   */
   async function executeRun(
     ctx: UiContext,
     run: SwarmRun,
@@ -302,25 +345,33 @@ export default function swarm(pi: ExtensionAPI) {
   ): Promise<void> {
     // One shared log per run; only created when the caller asked for it.
     const mailbox = useMailbox ? mailboxDir(getAgentDir(), run.runId) : undefined;
-    const queue = [...run.items];
-    const workers = Array.from({ length: Math.min(DEFAULT_CONCURRENCY, queue.length) }, async () => {
-      for (;;) {
-        // A cancelled run stops taking new items; the ones already in flight
-        // were aborted by cancelRun.
-        if (run.status === "cancelled") return;
-        const item = queue.shift();
-        if (!item) return;
+
+    // A dependent structurally receives each upstream's output — the thing a
+    // hand-sequenced coordinator forgets. Prepended to the shared preamble.
+    const contextFor = (upstream: ItemState[]): string => {
+      if (upstream.length === 0) return context;
+      const blocks = upstream.map((up) => {
+        const body = up.result ?? (up.error ? `(failed: ${up.error})` : "(no output)");
+        return `## Output of ${up.id}\n${body}`;
+      });
+      return [context, ...blocks].filter((s) => s && s.trim()).join("\n\n");
+    };
+
+    // Run each item once its needs finish, up to the concurrency cap; a flat
+    // run (no needs) has everything ready at once, exactly like the old pool.
+    await runGraph(
+      run.items,
+      DEFAULT_CONCURRENCY,
+      async (item, upstream) => {
         const def = routeItem(item.item, defs, fixed);
         item.agent = def.name;
+        const itemContext = contextFor(upstream);
         if (isolate) {
           try {
             const iso = createIsolationWorktree(ctx.cwd, run.runId + "-i" + (item.index + 1));
-            await runItem(ctx, run.runId, def, item, context, iso.path, mailbox);
-            // Remove the worktree when the item changed nothing — the cleanup
-            // the README promised but the code never performed (removeIfUnchanged
-            // was imported and never called, leaking a worktree + branch per
-            // read-only item). Kept when there is work to merge, and only then
-            // is the merge note worth showing.
+            await runItem(ctx, run.runId, def, item, itemContext, iso.path, mailbox);
+            // Remove the worktree when the item changed nothing (the leak
+            // removeIfUnchanged fixes); keep it when there is work to merge.
             const removed = removeIfUnchanged(ctx.cwd, iso);
             if (item.result !== null) {
               item.result = `${item.result}\n\n${removed ? CLEAN_WORKTREE_NOTE : isolationNote(iso)}`;
@@ -330,11 +381,12 @@ export default function swarm(pi: ExtensionAPI) {
             item.error = err instanceof Error ? err.message : String(err);
           }
         } else {
-          await runItem(ctx, run.runId, def, item, context, undefined, mailbox);
+          await runItem(ctx, run.runId, def, item, itemContext, undefined, mailbox);
         }
-      }
-    });
-    await Promise.all(workers);
+      },
+      () => run.status === "cancelled",
+    );
+
     if (run.status !== "cancelled") run.status = "done";
     run.finishedAt = Date.now();
     pi.appendEntry(RUN_ENTRY, run);
@@ -374,9 +426,24 @@ export default function swarm(pi: ExtensionAPI) {
       "Write each item as a self-contained brief — children see nothing else. For MUTATING items set " +
       "isolation=worktree: each item gets its own git worktree and branch; reports say how to merge. " +
       "mailbox=true adds swarm_post/swarm_inbox so agents can warn each other about shared files and " +
-      "conventions instead of silently conflicting.",
+      "conventions instead of silently conflicting. " +
+      "An item can be a plain string (independent) OR an object {task, id, needs:[ids]} to declare a " +
+      "dependency: a needed item's output is prepended to the dependent automatically, and the dependent " +
+      "starts only once its needs finish. A cycle, a self-edge, or an unknown id is rejected before anything runs.",
     parameters: Type.Object({
-      items: Type.Array(Type.String(), { minItems: 1, maxItems: MAX_ITEMS }),
+      items: Type.Array(
+        Type.Union([
+          Type.String({ description: "A self-contained task brief (independent item)" }),
+          Type.Object({
+            task: Type.String({ description: "A self-contained task brief" }),
+            id: Type.Optional(Type.String({ description: "Stable id other items can reference in needs (default t1, t2, …)" })),
+            needs: Type.Optional(
+              Type.Array(Type.String(), { description: "Ids of items that must finish first; their output is prepended to this item" }),
+            ),
+          }),
+        ]),
+        { minItems: 1, maxItems: MAX_ITEMS },
+      ),
       context: Type.Optional(Type.String({ description: "Shared preamble for every item" })),
       agent: Type.Optional(Type.String({ description: "Force one agent type for all items" })),
       isolation: Type.Optional(Type.String({ description: "Set to worktree to give each item its own git worktree (for mutating items)" })),
@@ -388,7 +455,7 @@ export default function swarm(pi: ExtensionAPI) {
     async execute(
       _id,
       params: {
-        items: string[];
+        items: Array<string | { task: string; id?: string; needs?: string[] }>;
         context?: string;
         agent?: string;
         background?: boolean;
@@ -400,8 +467,11 @@ export default function swarm(pi: ExtensionAPI) {
       ctx,
     ) {
       const uiCtx = ctx as UiContext;
-      const items = params.items.map((s) => s.trim()).filter(Boolean);
-      if (items.length === 0) throw new Error("swarm_run requires at least one non-empty item.");
+      // Normalize strings/objects into graph nodes and reject a bad graph
+      // (cycle, self-edge, unknown or duplicate id) BEFORE spawning anything.
+      const { nodes, error } = normalizeItems(params.items ?? []);
+      if (error) throw new Error(`swarm_run: ${error}`);
+      if (nodes.length === 0) throw new Error("swarm_run requires at least one non-empty item.");
       if (params.agent && !defs.has(params.agent.toLowerCase())) {
         throw new Error(`Unknown agent type "${params.agent}". Available: ${[...defs.keys()].sort().join(", ")}`);
       }
@@ -416,9 +486,11 @@ export default function swarm(pi: ExtensionAPI) {
         status: "running",
         startedAt: Date.now(),
         finishedAt: null,
-        items: items.map((item, index) => ({
+        items: nodes.map((node, index) => ({
           index,
-          item,
+          id: node.id,
+          item: node.task,
+          needs: node.needs,
           agent: params.agent?.toLowerCase() ?? "?",
           status: "queued",
           turns: 0,
@@ -469,7 +541,7 @@ export default function swarm(pi: ExtensionAPI) {
           });
         return {
           content: [
-            { type: "text", text: `Swarm ${run.runId} started (${items.length} items). Poll swarm_status runId="${run.runId}".` },
+            { type: "text", text: `Swarm ${run.runId} started (${run.items.length} items). Poll swarm_status runId="${run.runId}".` },
           ],
           details: { runId: run.runId },
         };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pify/swarm",
-  "version": "0.7.7",
+  "version": "0.9.0",
   "description": "Coordinate multiple pi agents in parallel: swarm_run fan-out with per-item auto-routing, concurrency queue, aggregated reports",
   "keywords": [
     "pi-package",

package/src/graph.ts

@@ -0,0 +1,89 @@
+/**
+ * Turning a list of items — some depending on others — into a runnable graph.
+ *
+ * swarm's flat mode fans N independent items out in parallel. But real work is
+ * rarely that flat: "write the migration, THEN update the callers, THEN run the
+ * tests" is a chain, and "gather these three, then summarize" is a join. Left
+ * to the coordinator to sequence by hand, the dependency is a thing it can
+ * forget — and the classic failure is forgetting to pass the upstream's output
+ * to the downstream item at all.
+ *
+ * So an item may declare `needs: [id, …]`. This module is the pure half: it
+ * normalizes items (string or object) into nodes with stable ids, and rejects a
+ * bad graph — a self-edge, a reference to an unknown id, a duplicate id, or a
+ * cycle — BEFORE anything is spawned, because a bad graph should cost nothing.
+ * The scheduler (in the extension) then runs each item as soon as its needs are
+ * done and prepends every upstream's output to it.
+ *
+ * A flat list (no `needs` anywhere) validates trivially and every node is ready
+ * at once — identical to the old parallel fan-out. Zero dependencies.
+ */
+
+export interface RawItem {
+  task: string;
+  id?: string;
+  needs?: string[];
+}
+
+export interface GraphNode {
+  id: string;
+  task: string;
+  needs: string[];
+}
+
+export interface NormalizeResult {
+  nodes: GraphNode[];
+  error: string | null;
+}
+
+/** Accept a plain string or an object; give every item a stable id and clean needs. */
+export function normalizeItems(items: Array<string | RawItem>): NormalizeResult {
+  const nodes: GraphNode[] = [];
+  for (let i = 0; i < items.length; i++) {
+    const raw = items[i]!;
+    const isString = typeof raw === "string";
+    const task = (isString ? raw : raw.task ?? "").trim();
+    if (!task) return { nodes: [], error: `item ${i + 1} has no task text` };
+    const id = !isString && typeof raw.id === "string" && raw.id.trim() ? raw.id.trim() : `t${i + 1}`;
+    const needs =
+      !isString && Array.isArray(raw.needs)
+        ? raw.needs.filter((n): n is string => typeof n === "string" && n.trim() !== "").map((n) => n.trim())
+        : [];
+    nodes.push({ id, task, needs });
+  }
+  const error = validateGraph(nodes);
+  return { nodes, error };
+}
+
+/** Reject a self-edge, an unknown reference, a duplicate id, or a cycle. */
+export function validateGraph(nodes: GraphNode[]): string | null {
+  const ids = new Set<string>();
+  for (const n of nodes) {
+    if (ids.has(n.id)) return `duplicate item id "${n.id}"`;
+    ids.add(n.id);
+  }
+  for (const n of nodes) {
+    for (const dep of n.needs) {
+      if (dep === n.id) return `item "${n.id}" cannot depend on itself`;
+      if (!ids.has(dep)) return `item "${n.id}" needs unknown item "${dep}"`;
+    }
+  }
+  const byId = new Map(nodes.map((n) => [n.id, n]));
+  const state = new Map<string, 0 | 1 | 2>(); // 0 unseen, 1 on stack, 2 done
+  const hasCycle = (id: string): boolean => {
+    const s = state.get(id) ?? 0;
+    if (s === 1) return true;
+    if (s === 2) return false;
+    state.set(id, 1);
+    for (const dep of byId.get(id)!.needs) if (hasCycle(dep)) return true;
+    state.set(id, 2);
+    return false;
+  };
+  for (const n of nodes) if (hasCycle(n.id)) return `dependency cycle involving "${n.id}"`;
+  return null;
+}
+
+/** Does any node declare a dependency? If not, the run is a plain flat fan-out. */
+export function hasEdges(nodes: GraphNode[]): boolean {
+  return nodes.some((n) => n.needs.length > 0);
+}

package/src/loop-guard.ts

@@ -0,0 +1,90 @@
+/**
+ * Stop a child that has stopped making progress.
+ *
+ * A turn cap bounds what a runaway child can cost; it does not notice the
+ * characteristic autonomous-agent failure, which is cheaper per turn and just
+ * as stuck: restating the same plan every turn without calling a tool, or
+ * oscillating between two states forever. pi's own loop will happily let that
+ * run to the cap.
+ *
+ * So fingerprint each assistant turn and watch for two shapes. A turn that
+ * called a tool is progress and clears the history — the guard only fires on
+ * turns that did nothing but talk. `repeat` identical tool-free turns is a
+ * child spinning in place; an A-B-A-B… run of `cycle` cycles is one bouncing
+ * between two dead ends. Both are advisory signals the caller acts on (a
+ * spawned child is unattended, so acting means aborting it).
+ *
+ * Zero dependencies — node:crypto for the hash. Deterministic and pure given
+ * the sequence of turns, so it is unit-testable without a live child.
+ */
+import { createHash } from "node:crypto";
+
+export interface LoopGuardConfig {
+  /** Consecutive identical tool-free turns before flagging a stall (min 2, default 3). */
+  repeat?: number;
+  /** Repeats of a two-turn A-B cycle before flagging (min 2, default 3). */
+  cycle?: number;
+}
+
+export interface Turn {
+  /** The visible assistant text of the turn. */
+  text: string;
+  /** Did the turn invoke at least one tool? A tool call is progress. */
+  usedTool: boolean;
+}
+
+export interface LoopVerdict {
+  stalled: boolean;
+  reason?: string;
+}
+
+/** Fold away cosmetic differences so "the same thought" hashes the same. */
+function fingerprint(text: string): string {
+  const norm = text.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim();
+  return createHash("sha256").update(norm).digest("hex");
+}
+
+export class LoopGuard {
+  private readonly repeat: number;
+  private readonly cycle: number;
+  /** Recent tool-free fingerprints; a tool call clears this. */
+  private readonly recent: string[] = [];
+  private static readonly MAX = 16;
+
+  constructor(cfg: LoopGuardConfig = {}) {
+    this.repeat = Math.max(2, cfg.repeat ?? 3);
+    this.cycle = Math.max(2, cfg.cycle ?? 3);
+  }
+
+  observe(turn: Turn): LoopVerdict {
+    // A tool call is forward motion: forget the stall history entirely.
+    if (turn.usedTool) {
+      this.recent.length = 0;
+      return { stalled: false };
+    }
+    // A silent turn (no text, no tool) is not evidence of a loop by itself.
+    if (turn.text.trim() === "") return { stalled: false };
+
+    const fp = fingerprint(turn.text);
+    this.recent.push(fp);
+    if (this.recent.length > LoopGuard.MAX) this.recent.shift();
+
+    // Spinning in place: the last `repeat` tool-free turns are identical.
+    if (this.recent.length >= this.repeat && this.recent.slice(-this.repeat).every((f) => f === fp)) {
+      return { stalled: true, reason: `repeated the same output for ${this.repeat} turns without acting` };
+    }
+
+    // Oscillating: the last 2×cycle turns are a strict A-B-A-B… alternation.
+    const need = 2 * this.cycle;
+    if (this.recent.length >= need) {
+      const tail = this.recent.slice(-need);
+      const a = tail[0]!;
+      const b = tail[1]!;
+      if (a !== b && tail.every((f, i) => f === (i % 2 === 0 ? a : b))) {
+        return { stalled: true, reason: `oscillated between two states for ${this.cycle} cycles without acting` };
+      }
+    }
+
+    return { stalled: false };
+  }
+}

package/src/schedule.ts

@@ -0,0 +1,60 @@
+/**
+ * The readiness scheduler, pure and testable.
+ *
+ * Given nodes that may depend on one another, run each one as soon as its
+ * `needs` have finished, never more than `concurrency` at a time. The caller's
+ * `run(node, upstream)` does the actual work and is handed the finished
+ * upstream nodes (in `needs` order) so it can thread their output into the
+ * dependent. A node whose run rejects still counts as done, so a failure
+ * unblocks its dependents rather than wedging the whole graph. `cancelled()` is
+ * polled between launches to stop taking on new work.
+ *
+ * A flat set (no `needs`) makes every node ready at once, so this collapses to
+ * a plain concurrency-capped fan-out. Zero dependencies; the graph itself must
+ * already be validated (see graph.ts) — this assumes no cycle.
+ */
+
+export interface Schedulable {
+  id: string;
+  needs: string[];
+}
+
+export async function runGraph<T extends Schedulable>(
+  nodes: readonly T[],
+  concurrency: number,
+  run: (node: T, upstream: T[]) => void | Promise<void>,
+  cancelled: () => boolean = () => false,
+): Promise<void> {
+  const byId = new Map(nodes.map((n) => [n.id, n]));
+  const done = new Set<string>();
+  const started = new Set<string>();
+  const inFlight = new Map<string, Promise<void>>();
+  const cap = Math.max(1, concurrency);
+
+  const isReady = (n: T) => !started.has(n.id) && n.needs.every((d) => done.has(d));
+
+  while (!cancelled()) {
+    for (const n of nodes) {
+      if (inFlight.size >= cap) break;
+      if (!isReady(n)) continue;
+      started.add(n.id);
+      const upstream = n.needs.map((id) => byId.get(id)).filter((x): x is T => x !== undefined);
+      const p = Promise.resolve()
+        .then(() => run(n, upstream))
+        .catch(() => {
+          // The caller records its own failure; here a rejected node must still
+          // settle so dependents unblock instead of the graph hanging.
+        })
+        .then(() => {
+          done.add(n.id);
+          inFlight.delete(n.id);
+        });
+      inFlight.set(n.id, p);
+    }
+    if (inFlight.size === 0) break; // nothing running and nothing newly ready
+    await Promise.race(inFlight.values());
+  }
+  // A cancel can leave work in flight; let it settle so the caller's record is
+  // complete rather than half-written.
+  await Promise.all(inFlight.values());
+}

package/src/types.ts

@@ -52,7 +52,11 @@ export type ItemStatus = "queued" | "running" | "done" | "error" | "aborted";
 
 export interface ItemState {
   index: number;
+  /** Stable id used for dependency references, e.g. "t1" or a caller-given id. */
+  id: string;
   item: string;
+  /** Ids of items that must finish before this one starts (empty = flat). */
+  needs: string[];
   agent: string;
   status: ItemStatus;
   turns: number;