agents-pod 0.1.0 → 0.2.0

package/src/sources.ts

@@ -0,0 +1,278 @@
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { BadInboxLine, InboxItem } from "./inbox.ts";
+import type { ListenerRecord } from "./listeners.ts";
+import { sourcesFile } from "./paths.ts";
+
+/** A Source outlives its Listeners; its opaque cursor lives beside them (ADR 0024). */
+export interface SourceRecord {
+  name: string;
+  cmd: string;
+  timeoutMs: number;
+  cursor?: string;
+  /** The previous successful drain is the claim-retention horizon (ADR 0024). */
+  lastDrainAt?: number;
+}
+
+export interface SourceDrain {
+  name: string;
+  items: InboxItem[];
+  bad: BadInboxLine[];
+  cursor?: string;
+  /** Successful drains carry their generation and the preceding one's boundary. */
+  drainedAt?: number;
+  previousDrainAt?: number;
+  /** A failed drain contributes nothing and leaves the cursor untouched. */
+  error?: string;
+}
+
+/** One MiB is the whole successful stdout budget (ADR 0024). */
+export const DRAIN_STDOUT_CAP = 1024 * 1024;
+
+export async function readSources(): Promise<SourceRecord[]> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(sourcesFile(), "utf8"));
+    return Array.isArray(parsed) ? (parsed as SourceRecord[]) : [];
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+    throw error;
+  }
+}
+
+/** The cursor's own atomic write, separate from listeners.json (ADR 0024). */
+export async function writeSources(sources: SourceRecord[]): Promise<void> {
+  const file = sourcesFile();
+  await mkdir(dirname(file), { recursive: true });
+  const tmp = `${file}.${randomBytes(4).toString("hex")}.tmp`;
+  await writeFile(tmp, `${JSON.stringify(sources, null, 2)}\n`);
+  await rename(tmp, file);
+}
+
+function routingField(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined;
+  const trimmed = value.trim();
+  return trimmed === "" ? undefined : trimmed;
+}
+
+/** A stream item is complete when its line parses; no two-pass guard applies. */
+export function parseDrainOutput(name: string, text: string): SourceDrain {
+  const items: InboxItem[] = [];
+  const bad: BadInboxLine[] = [];
+  let cursor: string | undefined;
+  const lines = text.split("\n")
+    .map((raw, index) => ({ raw, line: index + 1 }))
+    .filter(({ raw }) => raw.trim() !== "");
+
+  for (let index = 0; index < lines.length; index += 1) {
+    const { raw, line } = lines[index]!;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      bad.push({ line, reason: "invalid JSON" });
+      continue;
+    }
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      bad.push({ line, reason: "invalid JSON (not an object)" });
+      continue;
+    }
+    const record = parsed as {
+      id?: unknown;
+      body?: unknown;
+      title?: unknown;
+      cursor?: unknown;
+      chat_id?: unknown;
+      root_id?: unknown;
+      thread_id?: unknown;
+    };
+    if (
+      index === lines.length - 1 &&
+      "cursor" in record &&
+      record.id === undefined &&
+      record.body === undefined
+    ) {
+      if (typeof record.cursor !== "string") {
+        bad.push({ line, reason: "invalid cursor" });
+      } else {
+        cursor = record.cursor;
+      }
+      continue;
+    }
+    if (typeof record.id !== "string" || record.id.trim() === "") {
+      bad.push({ line, reason: "no id" });
+      continue;
+    }
+    if (typeof record.body !== "string") {
+      bad.push({ line, reason: "no body" });
+      continue;
+    }
+    const title =
+      typeof record.title === "string" && record.title.trim() !== ""
+        ? record.title
+        : record.body.split("\n", 1)[0]!.trim();
+    const chatId = routingField(record.chat_id);
+    const rootId = routingField(record.root_id);
+    const threadId = routingField(record.thread_id);
+    items.push({
+      id: record.id,
+      title,
+      body: record.body,
+      path: "",
+      mtimeMs: 0,
+      size: 0,
+      ...(chatId === undefined ? {} : { chatId }),
+      ...(rootId === undefined ? {} : { rootId }),
+      ...(threadId === undefined ? {} : { threadId }),
+    });
+  }
+  return { name, items, bad, ...(cursor === undefined ? {} : { cursor }) };
+}
+
+async function drainOne(source: SourceRecord, project: string): Promise<SourceDrain> {
+  const empty = { name: source.name, items: [], bad: [] };
+  const result = await new Promise<{ stdout?: string; error?: string }>((resolve) => {
+    const child = spawn(source.cmd, [], {
+      cwd: project,
+      env: {
+        ...process.env,
+        AGENTS_POD_SOURCE_ID: source.name,
+        AGENTS_POD_CURSOR: source.cursor ?? "",
+      },
+      // A shell drain can leave a child holding stdout after the shell dies.
+      // Its own process group makes the timeout and output cap real bounds.
+      detached: true,
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    const kill = (): void => {
+      if (child.pid === undefined) {
+        child.kill("SIGKILL");
+        return;
+      }
+      try {
+        process.kill(-child.pid, "SIGKILL");
+      } catch {
+        child.kill("SIGKILL");
+      }
+    };
+    const chunks: Buffer[] = [];
+    let bytes = 0;
+    let oversized = false;
+    let timedOut = false;
+    let spawnError: string | undefined;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      kill();
+    }, source.timeoutMs);
+
+    child.stdout.on("data", (chunk: Buffer) => {
+      bytes += chunk.length;
+      if (bytes > DRAIN_STDOUT_CAP) {
+        if (!oversized) {
+          oversized = true;
+          kill();
+        }
+        return;
+      }
+      chunks.push(chunk);
+    });
+    child.on("error", (error) => {
+      spawnError = error.message;
+    });
+    child.on("close", (code, signal) => {
+      clearTimeout(timer);
+      if (oversized) return resolve({ error: "stdout exceeded 1 MiB" });
+      if (timedOut) return resolve({ error: `timed out after ${source.timeoutMs / 1000}s` });
+      if (spawnError !== undefined) return resolve({ error: spawnError });
+      if (code !== 0) {
+        return resolve({
+          error: signal === null ? `exit ${code ?? "unknown"}` : `terminated by ${signal}`,
+        });
+      }
+      resolve({ stdout: Buffer.concat(chunks).toString("utf8") });
+    });
+  });
+  if (result.error !== undefined) return { ...empty, error: result.error };
+  // Distinct generations need distinct boundaries even when two local drains
+  // finish within one wall-clock millisecond.
+  const drainedAt = Math.max(Date.now(), (source.lastDrainAt ?? 0) + 1);
+  return {
+    ...parseDrainOutput(source.name, result.stdout ?? ""),
+    drainedAt,
+    ...(source.lastDrainAt === undefined ? {} : { previousDrainAt: source.lastDrainAt }),
+  };
+}
+
+/** Drain each distinct Source once, before the registry lock or any Pod work (ADR 0024). */
+export async function drainSources(
+  project: string,
+  listeners: ListenerRecord[],
+): Promise<SourceDrain[]> {
+  const sources = await readSources();
+  const names = [...new Set(listeners.flatMap((listener) => listener.source ?? []))];
+  const drained: SourceDrain[] = [];
+  for (const name of names) {
+    const source = sources.find((candidate) => candidate.name === name);
+    drained.push(
+      source === undefined
+        ? { name, items: [], bad: [], error: "source record is missing" }
+        : await drainOne(source, project),
+    );
+  }
+  return drained;
+}
+
+/**
+ * Two generations are enough: after this drain, anything older than the prior
+ * successful one is behind the committed cursor and cannot arrive again. The
+ * store is transformed without I/O; the caller writes it inside the registry
+ * lock before advancing the Source records (ADR 0024). Legacy claims without a
+ * timestamp are retained because their generation cannot be proven.
+ */
+export function pruneSourceClaims(
+  listeners: ListenerRecord[],
+  drained: SourceDrain[],
+): { listeners: ListenerRecord[]; changed: boolean } {
+  const horizons = new Map<string, number>();
+  for (const drain of drained) {
+    if (drain.error === undefined && drain.previousDrainAt !== undefined) {
+      horizons.set(`source:${drain.name}`, drain.previousDrainAt);
+    }
+  }
+  if (horizons.size === 0) return { listeners, changed: false };
+
+  let changed = false;
+  const pruned = listeners.map((listener) => {
+    const claimed = listener.claimed.filter((claim) => {
+      const horizon = horizons.get(claim.inbox);
+      return (
+        horizon === undefined || claim.claimedAt === undefined || claim.claimedAt >= horizon
+      );
+    });
+    if (claimed.length === listener.claimed.length) return listener;
+    changed = true;
+    return { ...listener, claimed };
+  });
+  return { listeners: pruned, changed };
+}
+
+/** Claims and pruning land before this cursor/retention-horizon write (ADR 0024). */
+export async function commitSourceCursors(drained: SourceDrain[]): Promise<void> {
+  const sources = await readSources();
+  let changed = false;
+  for (const drain of drained) {
+    if (drain.error !== undefined || drain.drainedAt === undefined) continue;
+    const source = sources.find((candidate) => candidate.name === drain.name);
+    if (source === undefined) continue;
+    if (drain.cursor !== undefined && source.cursor !== drain.cursor) {
+      source.cursor = drain.cursor;
+      changed = true;
+    }
+    if (source.lastDrainAt !== drain.drainedAt) {
+      source.lastDrainAt = drain.drainedAt;
+      changed = true;
+    }
+  }
+  if (changed) await writeSources(sources);
+}

package/src/workspace.ts

@@ -1,7 +1,7 @@
 import { randomBytes } from "node:crypto";
 import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
 import { join } from "node:path";
-import type { InboxItem } from "./inbox.ts";
+import { deliveryAddress, type InboxItem } from "./inbox.ts";
 import { podDir } from "./paths.ts";
 import type { PodRecord } from "./registry.ts";
 
@@ -48,7 +48,7 @@ const TEMPLATES: Record<string, string> = {
     "One line per decision, with its reason. Append-only.",
     "",
   ].join("\n"),
-  "TASKS.md": ["# Tasks", "", "## Plan", "", "## Inbox", ""].join("\n"),
+  "TASKS.md": ["# Tasks", "", "## Plan", "", "## Waiting", "", "## Inbox", ""].join("\n"),
 };
 
 /** The one section reconcile may write (ADR 0008, 0014). */
@@ -58,6 +58,9 @@ export async function createPodWorkspace(pod: PodRecord): Promise<string> {
   const dir = podDir(pod.project, pod.id);
   const docs = join(dir, "docs");
   await mkdir(docs, { recursive: true });
+  // The outbox is beside the three-slot Document Store, never a fourth slot
+  // (ADRs 0008, 0025).
+  await mkdir(join(dir, "outbox"), { recursive: true });
   for (const [name, body] of Object.entries(TEMPLATES)) {
     await writeFile(join(docs, name), body);
   }
@@ -70,9 +73,13 @@ export async function createPodWorkspace(pod: PodRecord): Promise<string> {
  * audit have an anchor; the body is copied because the source file belongs to
  * its writer and may be deleted once the item is claimed.
  */
-export function inboxBlock(item: Pick<InboxItem, "id" | "title" | "body">): string {
+export function inboxBlock(
+  item: Pick<InboxItem, "id" | "title" | "body" | "chatId" | "rootId" | "threadId">,
+): string {
   const title = item.title.split("\n", 1)[0]!.trim();
-  const lines = [`- [ ] ${item.id} — ${title}`];
+  const address = deliveryAddress(item);
+  const suffix = address === "" ? "" : ` — to: ${address}`;
+  const lines = [`- [ ] ${item.id} — ${title}${suffix}`];
   const body = item.body.replace(/\r\n/g, "\n").replace(/\n+$/, "");
   if (body.length > 0) {
     for (const line of body.split("\n")) {

package/src/wrapper.ts

@@ -1,6 +1,7 @@
 import { spawn, type ChildProcess } from "node:child_process";
 import { createHash } from "node:crypto";
 import {
+  appendFile,
   copyFile,
   mkdir,
   open,
@@ -20,7 +21,15 @@ import {
   type WrapperRecord,
 } from "./liveness.ts";
 import { podDir } from "./paths.ts";
-import { readPods, writePods, type WakeUpOutcome } from "./registry.ts";
+import { flushOutbox } from "./outbox.ts";
+import { sleep } from "./locks.ts";
+import {
+  readPods,
+  withRegistryLock,
+  writePods,
+  type PodRecord,
+  type WakeUpOutcome,
+} from "./registry.ts";
 import { snapshotDocs } from "./workspace.ts";
 
 /** The heartbeat interval: the mtime is the liveness signal (ADR 0017). */
@@ -213,17 +222,52 @@ function instructionsHash(instructions: string): string {
   return createHash("sha256").update(instructions).digest("hex").slice(0, 12);
 }
 
+/**
+ * One pod's read-modify-write belongs wholly inside the registry lock (ADR
+ * 0018). A wrapper that cannot record its result must fail loudly and keep its
+ * identity: reconcile may call that wake-up lost, but its outcome is never
+ * silently discarded.
+ */
+async function updatePodRecord(
+  podId: string,
+  podRoot: string,
+  n: number,
+  mutate: (record: PodRecord) => void,
+): Promise<boolean> {
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const recorded = await withRegistryLock(async () => {
+      const pods = await readPods();
+      const record = pods.find((candidate) => candidate.id === podId);
+      if (record) {
+        mutate(record);
+        await writePods(pods);
+      }
+      return true;
+    });
+    if (recorded !== undefined) return true;
+    if (attempt < 2) await sleep(100 * 2 ** attempt);
+  }
+  await appendFile(
+    join(podRoot, "output.log"),
+    `wrapper: registry lock unavailable; wake-up ${n} outcome not recorded\n`,
+  );
+  return false;
+}
+
 /**
  * Records a wake-up that never reached an agent. The outcome is a fact; the
  * policy that follows from it — backoff, streak, park — stays reconcile's.
  */
-async function recordUnresolved(podId: string, n: number, what: string): Promise<void> {
-  const pods = await readPods();
-  const record = pods.find((candidate) => candidate.id === podId);
-  if (!record) return;
-  record.wakeUps.push({ n, outcome: "failed", detail: what });
-  record.state = "idle";
-  await writePods(pods);
+async function recordUnresolved(
+  podId: string,
+  podRoot: string,
+  n: number,
+  what: string,
+): Promise<boolean> {
+  return updatePodRecord(podId, podRoot, n, (record) => {
+    record.wakeUps.push({ n, outcome: "failed", detail: what });
+    record.state = "idle";
+  });
 }
 
 export async function runWrapper(argv: string[]): Promise<number> {
@@ -255,8 +299,7 @@ export async function runWrapper(argv: string[]): Promise<number> {
   const agentId = agentWord ?? pod.policy?.agent ?? "default";
   const agent = resolveAgentDefinition(await readAgents(), agentId);
   if (!agent) {
-    await recordUnresolved(podId, n, "agent resolution failed");
-    return 0;
+    return (await recordUnresolved(podId, podRoot, n, "agent resolution failed")) ? 0 : 1;
   }
 
   // The second seat of the Pi-only guard (ADR 0023, agents-store.ts): the CLI refuses an
@@ -266,8 +309,14 @@ export async function runWrapper(argv: string[]): Promise<number> {
   // from there; it never silently runs pi in claude's name.
   const refusal = agentTypeRefusal(agent.type);
   if (refusal !== undefined) {
-    await recordUnresolved(podId, n, `agent ${agent.id} has type ${agent.type}: ${refusal}`);
-    return 0;
+    return (await recordUnresolved(
+      podId,
+      podRoot,
+      n,
+      `agent ${agent.id} has type ${agent.type}: ${refusal}`,
+    ))
+      ? 0
+      : 1;
   }
 
   const wakeupDir = join(podRoot, "wakeups", String(n));
@@ -437,9 +486,7 @@ export async function runWrapper(argv: string[]): Promise<number> {
   // owner, and its detail is the wind-down, not the agent's last words. The
   // pod is terminal, so reconcile never walks it back to a wake (ADR 0011).
   if (cancelRequested) {
-    const pods = await readPods();
-    const record = pods.find((p) => p.id === podId);
-    if (record) {
+    const recorded = await updatePodRecord(podId, podRoot, n, (record) => {
       // A reconcile kill already adjudicated this wake-up (ADRs 0017, 0019):
       // its intent was written before the signal, so the wrapper adds no
       // second, contradictory outcome — whether the cause was progressless or
@@ -452,9 +499,9 @@ export async function runWrapper(argv: string[]): Promise<number> {
             : { n, outcome: "cancelled" },
         );
         record.state = "cancelled";
-        await writePods(pods);
       }
-    }
+    });
+    if (!recorded) return 1;
     // The identity outlives its purpose only until the outcome is recorded.
     await rm(wrapperRecordFile, { force: true });
     return 0;
@@ -466,13 +513,15 @@ export async function runWrapper(argv: string[]): Promise<number> {
     () => undefined,
   );
   const { outcome, detail } = await adjudicate(docsDir, wakeStart, eventsFile, code);
-  const pods = await readPods();
-  const record = pods.find((p) => p.id === podId);
-  if (record) {
+  // Delivery holds the active state until its bounded flush is over; a
+  // concurrent pass must not wake a second wrapper over the same outbox
+  // (ADR 0025). Failures are logged inside the flush and never change outcome.
+  await flushOutbox(podId, pod.project, n);
+  const recorded = await updatePodRecord(podId, podRoot, n, (record) => {
     record.wakeUps.push(detail === undefined ? { n, outcome } : { n, outcome, detail });
     record.state = outcome === "done" ? "done" : "idle";
-    await writePods(pods);
-  }
+  });
+  if (!recorded) return 1;
   // The identity outlives its purpose only until the outcome is recorded.
   await rm(wrapperRecordFile, { force: true });
   return 0;