@fitchmultz/pi-subagents 0.46.2 → 0.47.0

package/src/pi-intercom/bridge-session.ts

@@ -7,20 +7,17 @@ import {
 } from "./session-targets.ts";
 import type { Message, SendResult, SessionInfo } from "./types.ts";
 import type { Identity } from "./bridge-config.ts";
-import { Fault, INBOX_BYTES, INBOX_COUNT, cancelled, log, wait } from "./bridge-protocol.ts";
+import { Fault, cancelled, log, wait } from "./bridge-protocol.ts";
+import { Mailbox } from "./bridge-mailbox.ts";
 import type { UnknownRecord } from "../shared/unknown.ts";
 import type { MessagingOperation as Operation } from "./bridge-routes.ts";
 
-import { WaitingAsk, type Envelope } from "./bridge-ask.ts";
+import { WaitingAsk } from "./bridge-ask.ts";
 const MAX_REQUESTS = 8;
 const IDLE_TIMEOUT = 5 * 60 * 1000;
 const SEND_TIMEOUT = 8000;
 const LIST_TIMEOUT = 5000;
 
-interface InboxEntry extends Envelope {
-  readonly bytes: number;
-  replying: boolean;
-}
 interface RequestContext {
   readonly signal: AbortSignal;
   readonly requestId: string;
@@ -32,6 +29,8 @@ interface SendOptions {
   readonly expectsReply?: boolean;
 }
 export interface SessionOwner {
+  readonly stateDir?: string;
+  readonly notifications?: boolean;
   readonly allowed: (identity: Identity) => boolean;
   readonly lost: (fp: string, count?: number) => void;
   readonly retired: (fp: string, generation: string) => void;
@@ -48,8 +47,8 @@ export class Session {
     listTimeoutMs: LIST_TIMEOUT,
   });
   private readonly requests = new Set<AbortController>();
-  private readonly inbox = new Map<string, InboxEntry>();
-  private inboxBytes = 0;
+  private readonly inbox: Mailbox;
+  private readonly replying = new Set<string>();
   private pending?: WaitingAsk;
   private closed = false;
   private connecting?: Promise<void>;
@@ -62,6 +61,7 @@ export class Session {
     this.owner = owner;
     this.identity = identity;
     this.expires = expires;
+    this.inbox = new Mailbox(owner.stateDir, identity.fingerprint256, owner.notifications);
     this.client.on("error", () =>
       this.close(new Fault(503, "broker_unavailable", "Broker connection failed.")),
     );
@@ -153,6 +153,7 @@ export class Session {
       this.close(fault);
       throw fault;
     }
+    this.presence();
     log("registered", this.fields());
   }
   private async connect(signal: AbortSignal): Promise<void> {
@@ -192,6 +193,7 @@ export class Session {
       // The original request receives the connection failure; teardown must still close a late socket.
     }
     await this.disconnect();
+    this.owner.lost(this.identity.fingerprint256, this.inbox.close());
     this.owner.retired(this.identity.fingerprint256, this.generation);
     log("session_closed", { ...this.fields(), result: this.closeReason.code });
   }
@@ -206,9 +208,6 @@ export class Session {
     for (const controller of this.requests) {
       controller.abort(error);
     }
-    this.owner.lost(this.identity.fingerprint256, this.inbox.size);
-    this.inbox.clear();
-    this.inboxBytes = 0;
     // Retain the generation until acquisition/registration AND final disconnect settle.
     this.startRetirement();
   }
@@ -225,12 +224,6 @@ export class Session {
         new Fault(409, "peer_offline", "Ask recipient disconnected.", { id: this.pending.id }),
       );
     }
-    for (const [messageId, entry] of this.inbox) {
-      if (entry.from.id === id && entry.message.expectsReply === true) {
-        this.remove(messageId);
-        this.owner.lost(this.identity.fingerprint256);
-      }
-    }
     this.presence();
   }
   private receive(from: SessionInfo, message: Message): void {
@@ -247,50 +240,66 @@ export class Session {
       );
       return;
     }
-    log("received", { ...this.fields(), peer: from.id, messageId: message.id, result: "received" });
-    const pending = this.pending;
-    if (pending?.matches(from.id, message.replyTo) === true) {
-      pending.resolve({ from, message });
+    if (!this.inbox.persistent && this.resolveReply(from, message)) {
       return;
     }
-    if (this.inbox.has(message.id)) {
-      this.owner.lost(this.identity.fingerprint256);
-      return;
-    }
-    const bytes = Buffer.byteLength(JSON.stringify({ from, message }));
-    if (this.inbox.size >= INBOX_COUNT || this.inboxBytes + bytes > INBOX_BYTES) {
-      this.owner.lost(this.identity.fingerprint256);
+    this.storeReceive(from, message);
+  }
+  private storeReceive(from: SessionInfo, message: Message): void {
+    let result: "stored" | "duplicate" | "lost";
+    try {
+      result = this.inbox.add({ from, message });
+    } catch {
       log("inbox_overflow", {
         ...this.fields(),
         peer: from.id,
         messageId: message.id,
-        result: "lost",
+        result: "storage_failed",
+        lossRecorded: false,
       });
+      this.close(
+        new Fault(
+          507,
+          "mailbox_storage_failed",
+          "Mailbox storage failed; inspect the metadata audit for unrecorded losses.",
+        ),
+      );
       return;
     }
-    // ponytail: bounded volatile inbox; use a private durable journal if restart/lease-loss recovery is required.
-    this.inbox.set(message.id, { from, message, bytes, replying: false });
-    this.inboxBytes += bytes;
+    if (result === "lost" || (result === "duplicate" && !this.inbox.persistent)) {
+      this.owner.lost(this.identity.fingerprint256);
+    }
+    this.recordReceive(from.id, message.id, result);
+    if (result !== "lost") {
+      this.resolveReply(from, message);
+    }
     this.presence();
   }
-  private remove(id: string): void {
-    const entry = this.inbox.get(id);
-    if (entry) {
-      this.inboxBytes -= entry.bytes;
-      this.inbox.delete(id);
+  private resolveReply(from: SessionInfo, message: Message): boolean {
+    const pending = this.pending;
+    if (pending?.matches(from.id, message.replyTo) !== true) {
+      return false;
     }
+    pending.resolve({ from, message });
+    return true;
+  }
+  private recordReceive(
+    peer: string,
+    messageId: string,
+    result: "stored" | "duplicate" | "lost",
+  ): void {
+    log(result === "lost" ? "inbox_overflow" : "received", {
+      ...this.fields(),
+      peer,
+      messageId,
+      result: result === "lost" && this.inbox.persistent ? "storage_failed" : result,
+    });
   }
   private presence(): void {
     if (this.closed) {
       return;
     }
-    const pendingAsks = [...this.inbox.values()].filter(
-      (entry) => entry.message.expectsReply === true,
-    ).length;
-    this.client.updatePresence({
-      pendingAsks,
-      acceptsAsks: this.inbox.size < INBOX_COUNT && this.inboxBytes < INBOX_BYTES,
-    });
+    this.client.updatePresence(this.inbox.presence());
   }
   private async peers(signal: AbortSignal): Promise<SessionInfo[]> {
     this.guard(signal);
@@ -421,19 +430,9 @@ export class Session {
     }
   }
   private ack(ids: readonly string[]): UnknownRecord {
-    const acked: string[] = [];
-    const retained: string[] = [];
-    for (const id of ids) {
-      const entry = this.inbox.get(id);
-      if (entry?.message.expectsReply === true) {
-        retained.push(id);
-      } else if (entry) {
-        this.remove(id);
-        acked.push(id);
-      }
-    }
+    const result = this.inbox.ack(ids);
     this.presence();
-    return { acked, retained };
+    return result;
   }
   private async answer(
     replyTo: string,
@@ -444,10 +443,10 @@ export class Session {
     if (entry?.message.expectsReply !== true) {
       throw new Fault(404, "unknown_ask", "replyTo must identify an unanswered inbound ask.");
     }
-    if (entry.replying) {
+    if (this.replying.has(replyTo)) {
       throw new Fault(409, "reply_in_progress", "This ask already has a reply in flight.");
     }
-    entry.replying = true;
+    this.replying.add(replyTo);
     const peer = entry.from.id;
     context.onPeer(peer);
     try {
@@ -456,15 +455,15 @@ export class Session {
         throw new Fault(409, "peer_offline", "Original ask sender is disconnected.");
       }
       this.guard(context.signal);
-      if (this.inbox.get(replyTo) !== entry) {
+      if (!this.inbox.get(replyTo)) {
         throw new Fault(409, "ask_retired", "Inbound ask is no longer replyable.");
       }
       const data = { ...(await this.send(peer, message, context, { replyTo })) };
-      this.remove(replyTo);
+      this.inbox.remove(replyTo);
       this.presence();
       return data;
     } finally {
-      entry.replying = false;
+      this.replying.delete(replyTo);
     }
   }
   async execute(operation: Operation, context: RequestContext): Promise<UnknownRecord> {
@@ -472,6 +471,19 @@ export class Session {
     if (operation.route === "POST /v1/ask") {
       return this.ask(operation, context);
     }
+    if (operation.route === "GET /v1/inbox" || operation.route === "POST /v1/ack") {
+      // Durable reads/acks remain available while the local broker is down.
+      if (!this.inbox.persistent) {
+        await this.connect(context.signal);
+      }
+      if (operation.route === "POST /v1/ack") {
+        return this.ack(operation.ids);
+      }
+      return {
+        ...this.inbox.page(operation.after, operation.limit),
+        ...this.inbox.status(this.owner.losses(this.identity.fingerprint256)),
+      };
+    }
     await this.connect(context.signal);
     switch (operation.route) {
       case "POST /v1/register":
@@ -485,16 +497,6 @@ export class Session {
           }),
         };
       }
-      case "GET /v1/inbox": {
-        const lostMessages = this.owner.losses(this.identity.fingerprint256);
-        return {
-          messages: [...this.inbox.values()].map(({ from, message }) => ({ from, message })),
-          overflow: lostMessages > 0,
-          lostMessages,
-        };
-      }
-      case "POST /v1/ack":
-        return this.ack(operation.ids);
       case "POST /v1/send":
         return {
           ...(await this.send(

package/src/pi-intercom/bridge-wake.ts

@@ -0,0 +1,188 @@
+import { log } from "./bridge-protocol.ts";
+import { Mailbox, type PendingNotification } from "./bridge-mailbox.ts";
+import type { Identity, WakePolicy } from "./bridge-config.ts";
+
+const POLL_MS = 15000;
+const FALLBACK_AFTER_MS = 120000;
+const MAX_BACKOFF_MS = 300000;
+
+export class WakeDelivery {
+  private readonly controller = new AbortController();
+  private timer?: NodeJS.Timeout;
+  private running?: Promise<void>;
+  private backoff = POLL_MS;
+  private readonly name: string;
+  private readonly mailbox: Pick<Mailbox, "pendingNotifications" | "notificationAccepted">;
+  private readonly policy: WakePolicy;
+  private readonly authorized: () => boolean;
+
+  constructor(
+    name: string,
+    mailbox: Pick<Mailbox, "pendingNotifications" | "notificationAccepted">,
+    policy: WakePolicy,
+    authorized: () => boolean,
+  ) {
+    this.name = name;
+    this.mailbox = mailbox;
+    this.policy = policy;
+    this.authorized = authorized;
+  }
+
+  start(): void {
+    if (!this.running && !this.controller.signal.aborted) {
+      this.schedule(0);
+    }
+  }
+
+  private schedule(delay: number): void {
+    clearTimeout(this.timer);
+    this.timer = setTimeout(() => {
+      this.running = this.deliver()
+        .catch(() => {
+          log("wake_delivery", { name: this.name, result: "storage_unavailable" });
+          this.backoff = Math.min(this.backoff * 2, MAX_BACKOFF_MS);
+        })
+        .finally(() => {
+          this.running = undefined;
+          if (!this.controller.signal.aborted) {
+            this.schedule(this.backoff);
+          }
+        });
+    }, delay);
+  }
+
+  private async deliver(): Promise<void> {
+    if (!this.authorized()) {
+      return;
+    }
+    const entries = this.mailbox.pendingNotifications().messages;
+    for (const entry of entries) {
+      if (entry.acknowledged) {
+        // The box's durable worker owns primary wake/replay after acknowledgement.
+        this.mailbox.notificationAccepted(entry.message.id);
+      }
+    }
+    const next = entries.find(
+      (entry) => !entry.acknowledged && Date.now() - entry.receivedAt >= FALLBACK_AFTER_MS,
+    );
+    if (next) {
+      // One fallback wakes the box to drain the remaining mailbox; never flood a reset VM.
+      await this.post(next);
+      return;
+    }
+    this.backoff = POLL_MS;
+  }
+
+  private async post(entry: PendingNotification): Promise<void> {
+    if (!this.authorized()) {
+      return;
+    }
+    const fields = { name: this.name, messageId: entry.message.id };
+    try {
+      const response = await fetch(this.policy.url, {
+        method: "POST",
+        redirect: "error",
+        headers: {
+          Authorization: `Bearer ${this.policy.key}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          from: entry.from.name ?? entry.from.id,
+          from_id: entry.from.id,
+          to: this.name,
+          message_id: entry.message.id,
+          ask_id: entry.message.expectsReply === true ? entry.message.id : null,
+          text: entry.message.content.text,
+        }),
+        signal: AbortSignal.any([this.controller.signal, AbortSignal.timeout(10000)]),
+      });
+      await response.body?.cancel();
+      if (response.ok && !this.controller.signal.aborted && this.authorized()) {
+        // HTTP acceptance cannot prove durable intake; the same ID remains pending until inbox ACK.
+        this.backoff = MAX_BACKOFF_MS;
+        log("wake_delivery", { ...fields, result: "accepted" });
+      } else {
+        this.backoff = Math.min(this.backoff * 2, MAX_BACKOFF_MS);
+        log("wake_delivery", { ...fields, result: "http_rejected", status: response.status });
+      }
+    } catch {
+      this.backoff = Math.min(this.backoff * 2, MAX_BACKOFF_MS);
+      if (!this.controller.signal.aborted) {
+        log("wake_delivery", { ...fields, result: "transport_unconfirmed" });
+      }
+    }
+  }
+
+  async stop(): Promise<void> {
+    this.controller.abort();
+    clearTimeout(this.timer);
+    await this.running;
+  }
+}
+
+export class WakeDeliveries {
+  private readonly stateDir: string | undefined;
+  private readonly allowed: (identity: Identity) => boolean;
+  private readonly deliveries = new Map<
+    string,
+    { readonly identity: Identity; readonly mailbox: Mailbox; readonly worker: WakeDelivery }
+  >();
+  private readonly cleanup = new Set<Promise<unknown>>();
+
+  constructor(stateDir: string | undefined, allowed: (identity: Identity) => boolean) {
+    this.stateDir = stateDir;
+    this.allowed = allowed;
+  }
+
+  session(identity: Identity, expires: number): void {
+    if (
+      !identity.wake ||
+      this.stateDir === undefined ||
+      this.deliveries.has(identity.fingerprint256)
+    ) {
+      return;
+    }
+    const mailbox = new Mailbox(this.stateDir, identity.fingerprint256, true);
+    const worker = new WakeDelivery(
+      identity.name,
+      mailbox,
+      identity.wake,
+      () => this.allowed(identity) && expires > Date.now(),
+    );
+    this.deliveries.set(identity.fingerprint256, { identity, mailbox, worker });
+    worker.start();
+  }
+
+  drop(fp: string): void {
+    const delivery = this.deliveries.get(fp);
+    if (!delivery) {
+      return;
+    }
+    this.deliveries.delete(fp);
+    const cleanup = delivery.worker
+      .stop()
+      .then(() => delivery.mailbox.close())
+      .catch(() => {
+        log("wake_delivery", { name: delivery.identity.name, result: "cleanup_failed" });
+      })
+      .finally(() => {
+        this.cleanup.delete(cleanup);
+      });
+    this.cleanup.add(cleanup);
+  }
+
+  reload(): void {
+    for (const [fp, delivery] of this.deliveries) {
+      if (!this.allowed(delivery.identity)) {
+        this.drop(fp);
+      }
+    }
+  }
+
+  async stop(): Promise<void> {
+    for (const fp of this.deliveries.keys()) {
+      this.drop(fp);
+    }
+    await Promise.all(this.cleanup);
+  }
+}

package/src/pi-intercom/bridge.ts

@@ -11,9 +11,21 @@ import { Session } from "./bridge-session.ts";
 import { operationFor, routeFor, type Operation } from "./bridge-routes.ts";
 import { reply } from "./bridge-http.ts";
 import { Launcher, sameIdentity } from "./bridge-launch.ts";
+import { PersistentPresence } from "./bridge-presence.ts";
+import { WakeDeliveries } from "./bridge-wake.ts";
+import { spawnBrokerIfNeeded } from "./broker/spawn.ts";
 
 const MAX_CONNECTIONS = 128;
 
+function validCertificateDates(starts: number, expires: number): boolean {
+  return (
+    Number.isFinite(starts) &&
+    Number.isFinite(expires) &&
+    starts <= Date.now() &&
+    expires > Date.now()
+  );
+}
+
 interface Connection {
   readonly fingerprint: string;
   readonly expires: number;
@@ -31,10 +43,32 @@ class Bridge {
   private readonly server: Server;
   private stopping = false;
   private readonly handlers = new Set<Promise<void>>();
+  private readonly presence?: PersistentPresence;
+  private readonly deliveries: WakeDeliveries;
 
   constructor(config: Config, configPath: string) {
     this.config = config;
     this.configPath = configPath;
+    this.deliveries = new WakeDeliveries(config.stateDir, (identity) => this.allowed(identity));
+    if (config.stateDir !== undefined) {
+      this.presence = new PersistentPresence({
+        stateDir: config.stateDir,
+        clients: () => this.config.clients,
+        connect: async (identity, expires) => {
+          await spawnBrokerIfNeeded(process.execPath, []);
+          if (!this.allowed(identity)) {
+            return;
+          }
+          const session = this.session(identity, expires);
+          const controller = session.begin();
+          try {
+            await session.connectedPeers(controller.signal);
+          } finally {
+            session.end(controller);
+          }
+        },
+      });
+    }
     this.server = createServer(
       {
         ...config.tls,
@@ -90,7 +124,11 @@ class Bridge {
   }
   private allowed(identity: Identity): boolean {
     const current = this.config.clients.get(identity.fingerprint256);
-    return !this.stopping && sameIdentity(identity, current);
+    return (
+      !this.stopping &&
+      sameIdentity(identity, current) &&
+      JSON.stringify(identity.wake) === JSON.stringify(current?.wake)
+    );
   }
   private lost(fp: string, count = 1): void {
     if (count > 0 && this.config.clients.has(fp)) {
@@ -98,10 +136,13 @@ class Bridge {
     }
   }
   private session(identity: Identity, expires: number): Session {
+    this.deliveries.session(identity, expires);
     let session = this.sessions.get(identity.fingerprint256);
     if (!session) {
       session = new Session(
         {
+          stateDir: this.config.stateDir,
+          notifications: identity.wake !== undefined,
           allowed: (candidate) => this.allowed(candidate),
           lost: (fp, count) => this.lost(fp, count),
           losses: (fp) => this.losses.get(fp) ?? 0,
@@ -134,15 +175,19 @@ class Bridge {
       !socket.authorized ||
       !identity ||
       this.stopping ||
-      !Number.isFinite(expires) ||
-      !Number.isFinite(starts) ||
-      starts > Date.now() ||
-      expires <= Date.now()
+      !validCertificateDates(starts, expires)
     ) {
       log("connection_rejected", { fingerprint: fp, result: "unauthorized" });
       socket.destroy();
       return;
     }
+    try {
+      this.presence?.authenticated(identity, expires);
+    } catch {
+      log("connection_rejected", { fingerprint: fp, result: "presence_storage_failed" });
+      socket.destroy();
+      return;
+    }
     const expiry = () => {
       if (expires <= Date.now()) {
         this.drop(fp, new Fault(403, "certificate_expired", "Client certificate expired."));
@@ -164,6 +209,7 @@ class Bridge {
     log("connected", { name: identity.name, fingerprint: fp, result: "authenticated" });
   }
   private drop(fp: string, error: Readonly<Fault>): void {
+    this.deliveries.drop(fp);
     this.sessions.get(fp)?.close(error);
     for (const [socket, connection] of this.connections) {
       if (connection.fingerprint === fp) {
@@ -181,6 +227,7 @@ class Bridge {
         port: typeof address === "object" && address ? address.port : this.config.port,
         brokerSocket: getBrokerSocketPath(),
       });
+      this.presence?.start();
     });
     process.on("SIGHUP", () => this.reload());
     // Retain listeners during dispatch: selected SDK dependencies use signal-exit's listener check.
@@ -202,6 +249,7 @@ class Bridge {
       log("allowlist_reloaded", { result: "accepted", clients: next.clients.size });
     } catch {
       this.config = { ...this.config, clients: new Map() };
+      this.deliveries.reload();
       this.launcher
         .revoke(this.config.clients)
         .catch(() => log("launch_cleanup", { result: "failed" }));
@@ -222,6 +270,7 @@ class Bridge {
     }
   }
   private revokeRemoved(): void {
+    this.deliveries.reload();
     for (const [fp, session] of this.sessions) {
       if (!this.allowed(session.identity)) {
         this.drop(
@@ -264,6 +313,8 @@ class Bridge {
       ...this.handlers,
       ...sessions.map((session) => session.settled()),
       this.launcher.shutdown(),
+      this.presence?.stop(),
+      this.deliveries.stop(),
     ]);
     log("shutdown", { result: "closed" });
   }
@@ -402,7 +453,7 @@ function main(): void {
   const args = process.argv.slice(2);
   if (args.length === 1 && (args[0] === "-h" || args[0] === "--help")) {
     process.stdout.write(
-      "Usage: node dist/pi-intercom/bridge.js --config PATH\n\nOptional loopback HTTPS/mTLS bridge for the local pi-intercom broker.\nConfig and TLS key must be owned by you with mode 0600. Never starts the broker.\nSIGHUP reloads the client allowlist (invalid reload denies all clients).\nRestart this helper for TLS/listener rotation. SIGINT/SIGTERM disconnect peers.\n\nExample: node dist/pi-intercom/bridge.js --config ~/.pi/agent/intercom/bridge.json\n",
+      "Usage: node dist/pi-intercom/bridge.js --config PATH\n\nOptional loopback HTTPS/mTLS bridge for the local pi-intercom broker.\nConfig and TLS key must be owned by you with mode 0600.\nWithout stateDir, messaging never starts the broker. Durable presence safely starts\none only when none is running; it never restarts or replaces a live broker.\nSIGHUP reloads the client allowlist (invalid reload denies all clients).\nRestart this helper for TLS/listener rotation. SIGINT/SIGTERM disconnect peers.\n\nExample: node dist/pi-intercom/bridge.js --config ~/.pi/agent/intercom/bridge.json\n",
     );
     return;
   }
@@ -416,7 +467,7 @@ function main(): void {
     new Bridge(loadConfig(path), path).start();
   } catch {
     process.stderr.write(
-      "Bridge startup failed: check config schema, file ownership/mode 0600, absolute TLS/cwd paths, TLS credentials, and launch policy (0700 sessionDir, Pi packageRoot, extension paths).\n",
+      "Bridge startup failed: check config schema, file ownership/mode 0600, absolute TLS/cwd paths, TLS credentials, stateDir (owned 0700 directory), and launch policy (0700 sessionDir, Pi packageRoot, extension paths).\n",
     );
     process.exitCode = 1;
   }