@pify/swarm 0.8.0 → 0.9.1

package/README.md

@@ -18,7 +18,7 @@ The catch is that "independent" is usually a small lie — the items do not depe
 
 | Parameter | Type | Notes |
 |---|---|---|
-| `items` | string[], 1–12 | One task per item; four run at a time, the rest queue |
+| `items` | array, 1–12 | A plain string per task, or `{task, id, needs}` to declare a dependency |
 | `context` | string, optional | Prepended to every item, so shared constraints are written once |
 | `agent` | string, optional | Force one agent type for all items instead of routing |
 | `isolation` | `"worktree"`, optional | Give each item its own git worktree — use it when items write |
@@ -27,6 +27,24 @@ The catch is that "independent" is usually a small lie — the items do not depe
 
 Blocking by default: returns `N done, M error` plus a per-item report.
 
+### Dependencies: `needs`
+
+An item can be a plain string (independent, as before) or an object that declares what it depends on:
+
+```json
+{
+  "items": [
+    { "id": "mig",     "task": "write the DB migration for the users table" },
+    { "id": "callers", "task": "update every caller of the old schema", "needs": ["mig"] },
+    { "id": "tests",   "task": "run the suite and fix what broke",       "needs": ["callers"] }
+  ]
+}
+```
+
+An item starts only once its `needs` have finished, and each upstream item's output is prepended to it as a `## Output of <id>` block — so the thing the coordinator used to forget, passing X to the step that needs it, happens by construction. Independent items still run in parallel up to the concurrency cap; a chain runs in order; a diamond joins after both branches. Ids default to `t1`, `t2`, … when you omit them.
+
+The whole graph is checked **before anything spawns**: a cycle, a self-edge, a duplicate id, or a reference to an unknown id is rejected outright, so a bad graph costs nothing. A flat list of strings has no edges and behaves exactly as it always did.
+
 ### `swarm_status`
 
 | Parameter | Type | Notes |
@@ -77,7 +95,7 @@ The catalog is the same `.pi/agents/*.md` one [`@pify/subagent`](https://github.
 
 ## Where this sits in the suite
 
-[`@pify/subagent`](https://github.com/pifydev/subagent) is one child and one task. `@pify/swarm` is many independent items at once. [`@pify/workflow`](https://github.com/pifydev/workflow) is deterministic scripted orchestration for when the steps genuinely depend on each other. Pick the smallest one that fits.
+[`@pify/subagent`](https://github.com/pifydev/subagent) is one child and one task. `@pify/swarm` is many items at once — independent, or wired together with a declarative `needs` graph (fan-out, chains, joins). [`@pify/workflow`](https://github.com/pifydev/workflow) is for when orchestration needs real control flow — loops, conditionals, retries, fan-out computed at run time — that a static graph can't express. Pick the smallest one that fits.
 
 ## License
 

package/extensions/swarm.ts

@@ -32,8 +32,8 @@ import {
   decideConsent,
   envConsent,
   parseConsent,
+  persistConsent,
   readConsent,
-  writeConsent,
 } from "../src/consent.ts";
 import { LiveChildren, cancelNote, type CancelReason } from "../src/cancel.ts";
 import { DELIVERY_TYPE, deliveryMessage, pendingResult } from "../src/pending.ts";
@@ -42,6 +42,8 @@ import { formatInbox, mailboxDir, mailboxPrompt, postMessage, readInbox } from "
 import { parseAgentFile } from "../src/frontmatter.ts";
 import { buildReport, buildStatusLine } from "../src/report.ts";
 import { routeItem } from "../src/routing.ts";
+import { normalizeItems } from "../src/graph.ts";
+import { runGraph } from "../src/schedule.ts";
 import { buildWidgetLines } from "../src/widget.ts";
 import {
   DEFAULT_CONCURRENCY,
@@ -328,7 +330,11 @@ export default function swarm(pi: ExtensionAPI) {
     }
   }
 
-  /** Pool executor: at most DEFAULT_CONCURRENCY items in flight. */
+  /**
+   * Readiness scheduler: run each item as soon as its `needs` are done, up to
+   * DEFAULT_CONCURRENCY at once. A flat run (no needs anywhere) makes every
+   * item ready immediately, so this is identical to the old parallel pool.
+   */
   async function executeRun(
     ctx: UiContext,
     run: SwarmRun,
@@ -339,25 +345,33 @@ export default function swarm(pi: ExtensionAPI) {
   ): Promise<void> {
     // One shared log per run; only created when the caller asked for it.
     const mailbox = useMailbox ? mailboxDir(getAgentDir(), run.runId) : undefined;
-    const queue = [...run.items];
-    const workers = Array.from({ length: Math.min(DEFAULT_CONCURRENCY, queue.length) }, async () => {
-      for (;;) {
-        // A cancelled run stops taking new items; the ones already in flight
-        // were aborted by cancelRun.
-        if (run.status === "cancelled") return;
-        const item = queue.shift();
-        if (!item) return;
+
+    // A dependent structurally receives each upstream's output — the thing a
+    // hand-sequenced coordinator forgets. Prepended to the shared preamble.
+    const contextFor = (upstream: ItemState[]): string => {
+      if (upstream.length === 0) return context;
+      const blocks = upstream.map((up) => {
+        const body = up.result ?? (up.error ? `(failed: ${up.error})` : "(no output)");
+        return `## Output of ${up.id}\n${body}`;
+      });
+      return [context, ...blocks].filter((s) => s && s.trim()).join("\n\n");
+    };
+
+    // Run each item once its needs finish, up to the concurrency cap; a flat
+    // run (no needs) has everything ready at once, exactly like the old pool.
+    await runGraph(
+      run.items,
+      DEFAULT_CONCURRENCY,
+      async (item, upstream) => {
         const def = routeItem(item.item, defs, fixed);
         item.agent = def.name;
+        const itemContext = contextFor(upstream);
         if (isolate) {
           try {
             const iso = createIsolationWorktree(ctx.cwd, run.runId + "-i" + (item.index + 1));
-            await runItem(ctx, run.runId, def, item, context, iso.path, mailbox);
-            // Remove the worktree when the item changed nothing — the cleanup
-            // the README promised but the code never performed (removeIfUnchanged
-            // was imported and never called, leaking a worktree + branch per
-            // read-only item). Kept when there is work to merge, and only then
-            // is the merge note worth showing.
+            await runItem(ctx, run.runId, def, item, itemContext, iso.path, mailbox);
+            // Remove the worktree when the item changed nothing (the leak
+            // removeIfUnchanged fixes); keep it when there is work to merge.
             const removed = removeIfUnchanged(ctx.cwd, iso);
             if (item.result !== null) {
               item.result = `${item.result}\n\n${removed ? CLEAN_WORKTREE_NOTE : isolationNote(iso)}`;
@@ -367,11 +381,12 @@ export default function swarm(pi: ExtensionAPI) {
             item.error = err instanceof Error ? err.message : String(err);
           }
         } else {
-          await runItem(ctx, run.runId, def, item, context, undefined, mailbox);
+          await runItem(ctx, run.runId, def, item, itemContext, undefined, mailbox);
         }
-      }
-    });
-    await Promise.all(workers);
+      },
+      () => run.status === "cancelled",
+    );
+
     if (run.status !== "cancelled") run.status = "done";
     run.finishedAt = Date.now();
     pi.appendEntry(RUN_ENTRY, run);
@@ -411,9 +426,24 @@ export default function swarm(pi: ExtensionAPI) {
       "Write each item as a self-contained brief — children see nothing else. For MUTATING items set " +
       "isolation=worktree: each item gets its own git worktree and branch; reports say how to merge. " +
       "mailbox=true adds swarm_post/swarm_inbox so agents can warn each other about shared files and " +
-      "conventions instead of silently conflicting.",
+      "conventions instead of silently conflicting. " +
+      "An item can be a plain string (independent) OR an object {task, id, needs:[ids]} to declare a " +
+      "dependency: a needed item's output is prepended to the dependent automatically, and the dependent " +
+      "starts only once its needs finish. A cycle, a self-edge, or an unknown id is rejected before anything runs.",
     parameters: Type.Object({
-      items: Type.Array(Type.String(), { minItems: 1, maxItems: MAX_ITEMS }),
+      items: Type.Array(
+        Type.Union([
+          Type.String({ description: "A self-contained task brief (independent item)" }),
+          Type.Object({
+            task: Type.String({ description: "A self-contained task brief" }),
+            id: Type.Optional(Type.String({ description: "Stable id other items can reference in needs (default t1, t2, …)" })),
+            needs: Type.Optional(
+              Type.Array(Type.String(), { description: "Ids of items that must finish first; their output is prepended to this item" }),
+            ),
+          }),
+        ]),
+        { minItems: 1, maxItems: MAX_ITEMS },
+      ),
       context: Type.Optional(Type.String({ description: "Shared preamble for every item" })),
       agent: Type.Optional(Type.String({ description: "Force one agent type for all items" })),
       isolation: Type.Optional(Type.String({ description: "Set to worktree to give each item its own git worktree (for mutating items)" })),
@@ -425,7 +455,7 @@ export default function swarm(pi: ExtensionAPI) {
     async execute(
       _id,
       params: {
-        items: string[];
+        items: Array<string | { task: string; id?: string; needs?: string[] }>;
         context?: string;
         agent?: string;
         background?: boolean;
@@ -437,8 +467,11 @@ export default function swarm(pi: ExtensionAPI) {
       ctx,
     ) {
       const uiCtx = ctx as UiContext;
-      const items = params.items.map((s) => s.trim()).filter(Boolean);
-      if (items.length === 0) throw new Error("swarm_run requires at least one non-empty item.");
+      // Normalize strings/objects into graph nodes and reject a bad graph
+      // (cycle, self-edge, unknown or duplicate id) BEFORE spawning anything.
+      const { nodes, error } = normalizeItems(params.items ?? []);
+      if (error) throw new Error(`swarm_run: ${error}`);
+      if (nodes.length === 0) throw new Error("swarm_run requires at least one non-empty item.");
       if (params.agent && !defs.has(params.agent.toLowerCase())) {
         throw new Error(`Unknown agent type "${params.agent}". Available: ${[...defs.keys()].sort().join(", ")}`);
       }
@@ -453,9 +486,11 @@ export default function swarm(pi: ExtensionAPI) {
         status: "running",
         startedAt: Date.now(),
         finishedAt: null,
-        items: items.map((item, index) => ({
+        items: nodes.map((node, index) => ({
           index,
-          item,
+          id: node.id,
+          item: node.task,
+          needs: node.needs,
           agent: params.agent?.toLowerCase() ?? "?",
           status: "queued",
           turns: 0,
@@ -506,7 +541,7 @@ export default function swarm(pi: ExtensionAPI) {
           });
         return {
           content: [
-            { type: "text", text: `Swarm ${run.runId} started (${items.length} items). Poll swarm_status runId="${run.runId}".` },
+            { type: "text", text: `Swarm ${run.runId} started (${run.items.length} items). Poll swarm_status runId="${run.runId}".` },
           ],
           details: { runId: run.runId },
         };
@@ -595,8 +630,7 @@ export default function swarm(pi: ExtensionAPI) {
       ),
     );
     try {
-      writeFileSync(file, `${JSON.stringify(writeConsent(store, ctx.cwd, "agents", approved), null, 2)}
-`);
+      persistConsent(file, ctx.cwd, "agents", approved);
     } catch {
       // An unwritable consent file costs us the memory of the answer, not the answer.
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pify/swarm",
-  "version": "0.8.0",
+  "version": "0.9.1",
   "description": "Coordinate multiple pi agents in parallel: swarm_run fan-out with per-item auto-routing, concurrency queue, aggregated reports",
   "keywords": [
     "pi-package",

package/src/consent.ts

@@ -16,6 +16,8 @@
  * around the first.
  */
 
+import { readFileSync, writeFileSync } from "node:fs";
+
 /** What to do with a project-supplied file this session. */
 export type ConsentVerdict = "allow" | "refuse" | "ask";
 
@@ -80,6 +82,30 @@ export function writeConsent(
   return { ...file, [key]: { ...(file[key] ?? {}), [scope]: allowed } };
 }
 
+/**
+ * Record a consent answer to disk without losing a concurrent writer's scope.
+ *
+ * Several @pify packages share this one file (keyed by cwd → {scope: bool}), and
+ * each asks the user with an `await` between reading the file and writing it
+ * back. If two flows both read the old file, then both write, the second write
+ * drops the scope the first added. So re-read and re-parse the file HERE, right
+ * before writing, with no `await` in between — the merge always starts from the
+ * freshest on-disk state, and within a single (single-threaded) process the
+ * read-modify-write can no longer interleave. Cross-process races remain
+ * theoretically possible but fail safe: the worst case is a re-prompt, never a
+ * silently-granted consent.
+ */
+export function persistConsent(file: string, cwd: string, scope: string, allowed: boolean): void {
+  let raw: string | null = null;
+  try {
+    raw = readFileSync(file, "utf8");
+  } catch {
+    raw = null;
+  }
+  const next = writeConsent(parseConsent(raw), cwd, scope, allowed);
+  writeFileSync(file, `${JSON.stringify(next, null, 2)}\n`);
+}
+
 /** Tolerate anything on disk: a corrupt consent file means "never asked". */
 export function parseConsent(raw: string | null): ConsentFile {
   if (!raw) return {};

package/src/graph.ts

@@ -0,0 +1,89 @@
+/**
+ * Turning a list of items — some depending on others — into a runnable graph.
+ *
+ * swarm's flat mode fans N independent items out in parallel. But real work is
+ * rarely that flat: "write the migration, THEN update the callers, THEN run the
+ * tests" is a chain, and "gather these three, then summarize" is a join. Left
+ * to the coordinator to sequence by hand, the dependency is a thing it can
+ * forget — and the classic failure is forgetting to pass the upstream's output
+ * to the downstream item at all.
+ *
+ * So an item may declare `needs: [id, …]`. This module is the pure half: it
+ * normalizes items (string or object) into nodes with stable ids, and rejects a
+ * bad graph — a self-edge, a reference to an unknown id, a duplicate id, or a
+ * cycle — BEFORE anything is spawned, because a bad graph should cost nothing.
+ * The scheduler (in the extension) then runs each item as soon as its needs are
+ * done and prepends every upstream's output to it.
+ *
+ * A flat list (no `needs` anywhere) validates trivially and every node is ready
+ * at once — identical to the old parallel fan-out. Zero dependencies.
+ */
+
+export interface RawItem {
+  task: string;
+  id?: string;
+  needs?: string[];
+}
+
+export interface GraphNode {
+  id: string;
+  task: string;
+  needs: string[];
+}
+
+export interface NormalizeResult {
+  nodes: GraphNode[];
+  error: string | null;
+}
+
+/** Accept a plain string or an object; give every item a stable id and clean needs. */
+export function normalizeItems(items: Array<string | RawItem>): NormalizeResult {
+  const nodes: GraphNode[] = [];
+  for (let i = 0; i < items.length; i++) {
+    const raw = items[i]!;
+    const isString = typeof raw === "string";
+    const task = (isString ? raw : raw.task ?? "").trim();
+    if (!task) return { nodes: [], error: `item ${i + 1} has no task text` };
+    const id = !isString && typeof raw.id === "string" && raw.id.trim() ? raw.id.trim() : `t${i + 1}`;
+    const needs =
+      !isString && Array.isArray(raw.needs)
+        ? raw.needs.filter((n): n is string => typeof n === "string" && n.trim() !== "").map((n) => n.trim())
+        : [];
+    nodes.push({ id, task, needs });
+  }
+  const error = validateGraph(nodes);
+  return { nodes, error };
+}
+
+/** Reject a self-edge, an unknown reference, a duplicate id, or a cycle. */
+export function validateGraph(nodes: GraphNode[]): string | null {
+  const ids = new Set<string>();
+  for (const n of nodes) {
+    if (ids.has(n.id)) return `duplicate item id "${n.id}"`;
+    ids.add(n.id);
+  }
+  for (const n of nodes) {
+    for (const dep of n.needs) {
+      if (dep === n.id) return `item "${n.id}" cannot depend on itself`;
+      if (!ids.has(dep)) return `item "${n.id}" needs unknown item "${dep}"`;
+    }
+  }
+  const byId = new Map(nodes.map((n) => [n.id, n]));
+  const state = new Map<string, 0 | 1 | 2>(); // 0 unseen, 1 on stack, 2 done
+  const hasCycle = (id: string): boolean => {
+    const s = state.get(id) ?? 0;
+    if (s === 1) return true;
+    if (s === 2) return false;
+    state.set(id, 1);
+    for (const dep of byId.get(id)!.needs) if (hasCycle(dep)) return true;
+    state.set(id, 2);
+    return false;
+  };
+  for (const n of nodes) if (hasCycle(n.id)) return `dependency cycle involving "${n.id}"`;
+  return null;
+}
+
+/** Does any node declare a dependency? If not, the run is a plain flat fan-out. */
+export function hasEdges(nodes: GraphNode[]): boolean {
+  return nodes.some((n) => n.needs.length > 0);
+}

package/src/schedule.ts

@@ -0,0 +1,60 @@
+/**
+ * The readiness scheduler, pure and testable.
+ *
+ * Given nodes that may depend on one another, run each one as soon as its
+ * `needs` have finished, never more than `concurrency` at a time. The caller's
+ * `run(node, upstream)` does the actual work and is handed the finished
+ * upstream nodes (in `needs` order) so it can thread their output into the
+ * dependent. A node whose run rejects still counts as done, so a failure
+ * unblocks its dependents rather than wedging the whole graph. `cancelled()` is
+ * polled between launches to stop taking on new work.
+ *
+ * A flat set (no `needs`) makes every node ready at once, so this collapses to
+ * a plain concurrency-capped fan-out. Zero dependencies; the graph itself must
+ * already be validated (see graph.ts) — this assumes no cycle.
+ */
+
+export interface Schedulable {
+  id: string;
+  needs: string[];
+}
+
+export async function runGraph<T extends Schedulable>(
+  nodes: readonly T[],
+  concurrency: number,
+  run: (node: T, upstream: T[]) => void | Promise<void>,
+  cancelled: () => boolean = () => false,
+): Promise<void> {
+  const byId = new Map(nodes.map((n) => [n.id, n]));
+  const done = new Set<string>();
+  const started = new Set<string>();
+  const inFlight = new Map<string, Promise<void>>();
+  const cap = Math.max(1, concurrency);
+
+  const isReady = (n: T) => !started.has(n.id) && n.needs.every((d) => done.has(d));
+
+  while (!cancelled()) {
+    for (const n of nodes) {
+      if (inFlight.size >= cap) break;
+      if (!isReady(n)) continue;
+      started.add(n.id);
+      const upstream = n.needs.map((id) => byId.get(id)).filter((x): x is T => x !== undefined);
+      const p = Promise.resolve()
+        .then(() => run(n, upstream))
+        .catch(() => {
+          // The caller records its own failure; here a rejected node must still
+          // settle so dependents unblock instead of the graph hanging.
+        })
+        .then(() => {
+          done.add(n.id);
+          inFlight.delete(n.id);
+        });
+      inFlight.set(n.id, p);
+    }
+    if (inFlight.size === 0) break; // nothing running and nothing newly ready
+    await Promise.race(inFlight.values());
+  }
+  // A cancel can leave work in flight; let it settle so the caller's record is
+  // complete rather than half-written.
+  await Promise.all(inFlight.values());
+}

package/src/types.ts

@@ -52,7 +52,11 @@ export type ItemStatus = "queued" | "running" | "done" | "error" | "aborted";
 
 export interface ItemState {
   index: number;
+  /** Stable id used for dependency references, e.g. "t1" or a caller-given id. */
+  id: string;
   item: string;
+  /** Ids of items that must finish before this one starts (empty = flat). */
+  needs: string[];
   agent: string;
   status: ItemStatus;
   turns: number;