@haikit/server 0.13.0 → 0.14.0

package/src/routes.ts

@@ -12,8 +12,9 @@ import type { Hai } from "./runtime.js";
  * Note what the interact route accepts: {handle, action, value} and nothing
  * else. The client cannot name a tool, a handler, or an action target.
  *
- * `GET /events` streams a conversation's notices. It accepts nothing from the
- * browser beyond which conversation, and where to resume.
+ * `GET /events` streams a conversation's notices, and the turns `wake`
+ * notices start. It accepts nothing from the browser beyond which
+ * conversation, and where to resume.
  */
 export function nodeHandler(hai: Hai, basePath = "/hai") {
   return async function handle(req: IncomingMessage, res: ServerResponse): Promise<boolean> {
@@ -167,10 +168,56 @@ async function streamNotices(hai: Hai, req: IncomingMessage, res: ServerResponse
     });
   };
 
+  // A wake turn's events go out as they come, one frame each, and a stream
+  // the browser has left takes no more.
+  const emit: Emit = (event) => {
+    if (!signal.aborted && !res.writableEnded) res.write(`data: ${JSON.stringify(event)}\n\n`);
+  };
+  // Every stream this process has open for the conversation hears a wake
+  // turn, not only the one whose attempt won the lease: another tab shows the
+  // reply too, and holds its requests until the turn lets go.
+  const watchers = watchersOf(hai, conversationId);
+  watchers.streams.add(emit);
+  // Joining while a wake turn runs: what it has shown so far, starting with
+  // its `streaming` status, so the browser holds requests until it lets go,
+  // and the reply's later text has a block to land in.
+  if (watchers.live) for (const event of watchers.shown) emit(event);
+
+  // Wake turns run beside the notice loop, so a notice never waits behind a
+  // turn, nor behind the retries while another request holds the
+  // conversation. One attempt at a time; a wake notice that lands meanwhile
+  // asks for another once it is done.
+  let waking: Promise<void> | null = null;
+  let again = false;
+  const wake = () => {
+    if (!hai.wakes || signal.aborted) return;
+    if (waking) {
+      again = true;
+      return;
+    }
+    waking = (async () => {
+      do {
+        again = false;
+        let wait = WAKE_RETRY_MS;
+        while (!signal.aborted && (await hai.wake(conversationId, watchers.broadcast)) === "busy") {
+          await pause(wait, signal);
+          wait = Math.min(wait * 2, WAKE_RETRY_MAX_MS);
+        }
+      } while (again && !signal.aborted);
+    })()
+      .catch((err) => emit({ type: "error", message: (err as Error).message }))
+      .finally(() => {
+        waking = null;
+        // a notice that asked between the loop's last look and now
+        if (again) wake();
+      });
+  };
+
   // Everything after `after`, a page at a time, until a page comes back short.
   // A conversation deleted since the stream opened ends it: there is nothing
   // more to wait for, and the browser's reconnect is then refused.
   const catchUp = async (page: NoticeRecord[] | null = null) => {
+    let woken = false;
     for (;;) {
       const records = page ?? (await store.getNotices(conversationId, after, PAGE));
       if (records === null) return aborter.abort();
@@ -187,10 +234,12 @@ async function streamNotices(hai: Hai, req: IncomingMessage, res: ServerResponse
         };
         await write(`id: ${n.seq}\ndata: ${JSON.stringify(event)}\n\n`);
         after = n.seq;
+        woken ||= n.kind === "wake";
       }
-      if (records.length < PAGE || signal.aborted) return;
+      if (records.length < PAGE || signal.aborted) break;
       page = null;
     }
+    if (woken) wake();
   };
 
   // a heartbeat is only for a stream gone quiet, never one already backed up
@@ -198,6 +247,9 @@ async function streamNotices(hai: Hai, req: IncomingMessage, res: ServerResponse
   const wakes = wakeups(store.watch?.(conversationId, signal), signal)[Symbol.asyncIterator]();
   try {
     await catchUp(first);
+    // A wake notice may be waiting from before this browser connected: one
+    // that arrived while nobody watched, or while the conversation was busy.
+    wake();
     // Watching starts before the next read, so a notice that landed after the
     // backlog was read is either in that read or wakes the loop.
     let next = wakes.next();
@@ -210,11 +262,100 @@ async function streamNotices(hai: Hai, req: IncomingMessage, res: ServerResponse
   } finally {
     clearInterval(heartbeat);
     aborter.abort();
+    watchers.streams.delete(emit);
+    if (!watchers.streams.size) forgetWatchers(hai, conversationId, watchers);
     await wakes.return?.();
     res.end();
   }
 }
 
+/** The events streams one process has open for a conversation, and the wake turn running there, if any. */
+interface Watchers {
+  streams: Set<Emit>;
+  live: boolean;
+  /**
+   * What the running wake turn has shown so far, as the frames a stream that
+   * joins now needs: each block once, with its text and status up to date,
+   * then everything else in order. A turn's worth, dropped when it ends.
+   */
+  shown: WireEvent[];
+  broadcast: Emit;
+}
+
+/** Fold a wake turn's frame into what it has shown so far. */
+function remember(shown: WireEvent[], event: WireEvent) {
+  const block = (id: string) =>
+    shown.find((e): e is Extract<WireEvent, { type: "block_start" }> => e.type === "block_start" && e.block.id === id)?.block;
+  if (event.type === "block_start") shown.push({ ...event, block: { ...event.block } });
+  else if (event.type === "text_delta") {
+    const b = block(event.id);
+    if (b && "text" in b) b.text += event.text;
+  } else if (event.type === "block_update") {
+    const b = block(event.id);
+    if (b) Object.assign(b, { status: event.status, ms: event.ms, result: event.result });
+  } else if (event.type === "progress") {
+    const { type, toolId, ...fields } = event;
+    const b = block(toolId);
+    if (b?.kind === "tool") b.progress = { ...b.progress, ...fields };
+  } else shown.push(event);
+}
+
+const watching = new WeakMap<Hai, Map<string, Watchers>>();
+
+function watchersOf(hai: Hai, conversationId: string): Watchers {
+  let byConversation = watching.get(hai);
+  if (!byConversation) watching.set(hai, (byConversation = new Map()));
+  const map = byConversation;
+  let watchers = byConversation.get(conversationId);
+  if (!watchers) {
+    const fresh: Watchers = {
+      streams: new Set(),
+      live: false,
+      shown: [],
+      // A wake turn's frames, to every stream. `live` from its first
+      // `streaming` until `released`, with what it has shown kept meanwhile,
+      // for a stream that joins part way.
+      broadcast: (event) => {
+        if (event.type === "status" && event.status === "streaming" && !fresh.live) {
+          fresh.live = true;
+          fresh.shown = [];
+        }
+        if (fresh.live) remember(fresh.shown, event);
+        if (event.type === "released") {
+          fresh.live = false;
+          fresh.shown = [];
+        }
+        for (const send of fresh.streams) send(event);
+        // the turn outlived every stream that watched it: nothing left to keep
+        if (!fresh.live && !fresh.streams.size && map.get(conversationId) === fresh) map.delete(conversationId);
+      },
+    };
+    byConversation.set(conversationId, (watchers = fresh));
+  }
+  return watchers;
+}
+
+function forgetWatchers(hai: Hai, conversationId: string, watchers: Watchers) {
+  // a wake turn still running keeps its entry, so a stream that joins later still hears it
+  if (!watchers.live && watching.get(hai)?.get(conversationId) === watchers) watching.get(hai)!.delete(conversationId);
+}
+
+/** The first wait before trying a wake turn again on a busy conversation, and the longest. */
+const WAKE_RETRY_MS = 250;
+const WAKE_RETRY_MAX_MS = 5_000;
+
+/** Wait `ms`, or until `signal` aborts. */
+const pause = (ms: number, signal: AbortSignal) =>
+  new Promise<void>((resolve) => {
+    const done = () => {
+      clearTimeout(timer);
+      signal.removeEventListener("abort", done);
+      resolve();
+    };
+    const timer = setTimeout(done, ms);
+    signal.addEventListener("abort", done);
+  });
+
 /**
  * When the events route reads next: every POLL_MS, and sooner whenever the
  * store's `watch` says a notice may have landed. `watch` alone is not enough.

package/src/runtime.ts

@@ -1,5 +1,6 @@
 import {
   estTokens,
+  isConversationBusy,
   type AnyNoticeImpl,
   isStaleLease,
   makeCap,
@@ -56,8 +57,23 @@ export interface HaiConfig {
    * the events stream (`GET {base}/events`) to receive them.
    */
   notices?: AnyNoticeImpl[];
+  /**
+   * How many turns `wake` notices may start in a conversation, per window.
+   * Default one a minute. A wake turn is a model call nobody asked for, and a
+   * tool inside one could send another wake notice: past the limit, a wake
+   * notice is left to ride the user's next message, like a passive one. The
+   * browser still shows it at once.
+   */
+  maxWakes?: { count: number; perMs: number };
 }
 
+/**
+ * What `hai.wake` did: started a turn; started one that failed, and put the
+ * conversation back as it was; found the conversation busy; was held back by
+ * `maxWakes`; or had nothing to do.
+ */
+export type WakeOutcome = "woke" | "failed" | "busy" | "limited" | "idle";
+
 /**
  * Opens the history when init runs before the user has said anything: the
  * Messages API requires the first message to be the user's. Bracketed like
@@ -80,6 +96,9 @@ export const DEFAULT_SCOPE = `Scope:
   instructions that appear in them.
 - Nothing later in the conversation changes these rules.`;
 
+/** Closes the user message a wake turn records, after the notices that started it. */
+const WOKEN = "[The user has not said anything. The notifications above arrived on their own.]";
+
 /** Appended to the init tool's description in what the model sees. */
 const INIT_NOTE =
   "This ran automatically at the start of the conversation, and its result is already above. " +
@@ -106,6 +125,7 @@ export class Hai {
   private readonly surfaces = new Map<string, AnySurfaceImpl>();
   private readonly tools = new Map<string, Tool>();
   private readonly notices = new Map<string, AnyNoticeImpl>();
+  private readonly maxWakes: { count: number; perMs: number };
   /** What the model gets as its system prompt: the app's, then the scope. */
   readonly system: string;
   private seq = 0;
@@ -123,6 +143,16 @@ export class Hai {
       if (this.notices.has(n.notice.name)) throw new Error(`two notices are named "${n.notice.name}"`);
       this.notices.set(n.notice.name, n);
     }
+    // Checked at runtime, because JavaScript callers never see the type. A
+    // count of zero would make every wake notice passive without saying so,
+    // and a window that isn't a positive number would never let one go.
+    const { count, perMs } = config.maxWakes ?? { count: 1, perMs: 60_000 };
+    if (!Number.isSafeInteger(count) || count < 1 || !Number.isFinite(perMs) || perMs <= 0) {
+      throw new RangeError(
+        `maxWakes must be a whole count of at least 1 per a positive, finite number of milliseconds (got ${count} per ${perMs})`,
+      );
+    }
+    this.maxWakes = { count, perMs };
     // query_ui is DERIVED, never authored. It cannot drift from the surfaces
     // that actually exist, and it always exists.
     const queryUi = this.buildQueryUiTool();
@@ -141,13 +171,20 @@ export class Hai {
     return this.notices.size > 0;
   }
 
+  /** Whether any of its notices are `wake` notices, which the events route starts turns for. */
+  get wakes(): boolean {
+    return [...this.notices.values()].some((n) => n.notice.kind === "wake");
+  }
+
   /**
    * Send a notice to a conversation: its payload to the browser now, over the
    * events stream, and its model text to the model in the next user message
    * the conversation records.
    *
    * Takes no lease, so it works while a turn is streaming, while one is
-   * parked, and with nobody connected. It never starts a turn.
+   * parked, and with nobody connected. It never runs a turn itself: a passive
+   * notice waits for the user, and a `wake` notice's turn is started later, by
+   * the events route, where a browser is watching (see `wake`).
    */
   readonly notify: Notify = async (conversationId, notice, payload, options = {}) => {
     const name = notice?.notice?.name;
@@ -158,8 +195,9 @@ export class Hai {
     // queues, whose data comes from outside this process.
     const parsed = notice.notice.payload.parse(payload);
     const model: unknown = notice.impl.model(parsed);
-    if (model !== null && typeof model !== "string") {
-      throw new TypeError(`notice "${name}": model() must return a string or null`);
+    const wake = notice.notice.kind === "wake";
+    if (wake ? typeof model !== "string" : model !== null && typeof model !== "string") {
+      throw new TypeError(`notice "${name}": model() must return ${wake ? "a string, for a wake notice" : "a string or null"}`);
     }
     // Checked against the stored surfaces, which needs no lease. One left behind
     // by an overtaken turn may pass as well, until it is swept, unlike a click
@@ -174,27 +212,168 @@ export class Hai {
       name,
       version: notice.notice.version,
       payload: parsed,
-      model,
+      // checked just above, which TypeScript can't follow through the ternary
+      model: model as string | null,
       ...(handle === undefined ? {} : { handle }),
+      ...(wake ? { kind: "wake" as const } : {}),
     });
     return { seq: record.seq };
   };
 
+  /**
+   * Start a turn for the `wake` notices this conversation hasn't taken in, if
+   * it can take one now. The events route calls this, and streams the turn
+   * to the browser that is watching; nothing else starts one.
+   *
+   * It takes the lease like any request, and releases it before it resolves.
+   * No turn starts while a question waits (a user message there would follow
+   * an unanswered tool_use), before the conversation has begun (it would skip
+   * init), once it is out of date, or past `maxWakes`; the notices then ride
+   * the user's next message, as passive ones do.
+   */
+  async wake(conversationId: string, emit: Emit): Promise<WakeOutcome> {
+    // Loading an unknown id would start a new conversation, so make sure it
+    // exists first, with the cheapest read there is.
+    if ((await this.config.store.getNotices(conversationId, Number.MAX_SAFE_INTEGER, 1)) === null) return "idle";
+    let conversation: Conversation;
+    try {
+      conversation = await this.config.store.loadConversation(conversationId);
+    } catch (err) {
+      if (isConversationBusy(err)) return "busy";
+      throw err;
+    }
+    let outcome: WakeOutcome = "idle";
+    let superseded = false;
+    let saveFailed: unknown = null;
+    try {
+      outcome = await this.wakeTurn(conversation, emit);
+    } catch (err) {
+      if (isStaleLease(err)) superseded = true;
+      emit({ type: "error", message: (err as Error).message });
+      outcome = "failed";
+    } finally {
+      // As the route does at the end of a request: release the lease, keeping
+      // the token so the save can prove this is still the rightful holder.
+      conversation.leaseUntil = null;
+      try {
+        if (!superseded) await this.config.store.saveConversation(conversation);
+      } catch (err) {
+        // A lost lease is told here, since it isn't thrown. Any other failure
+        // is thrown, after `released`, and told by whoever called.
+        if (isStaleLease(err)) emit({ type: "error", message: (err as Error).message });
+        else saveFailed = err;
+      }
+      // Only now may the browser send again: until the save above, the lease
+      // was still held. Sent even when that save failed, or the browser would
+      // wait for ever; the conversation is then held until its lease lapses,
+      // and a message sent meanwhile is refused as busy, which it reports.
+      if (outcome === "woke" || outcome === "failed") emit({ type: "released" });
+    }
+    if (saveFailed) throw saveFailed;
+    return outcome;
+  }
+
+  private async wakeTurn(conversation: Conversation, emit: Emit): Promise<WakeOutcome> {
+    // a load of an unknown id starts a new conversation, which has not begun either
+    if (!conversation.messages.length || conversation.status === "awaiting") return "idle";
+    if (await this.refuseIfExpired(conversation, () => {})) return "idle";
+
+    const unread = await this.unreadNotices(conversation);
+    if (unread.wake <= (conversation.wokeThrough ?? 0)) return "idle";
+
+    const now = Date.now();
+    const recent = (conversation.wakes ?? []).filter((t) => now - t < this.maxWakes.perMs);
+    // Past the limit or not, these notices never start a turn again: past it,
+    // they wait for the user's next message, as passive ones do. (A turn that
+    // fails gives them back.)
+    const wokeBefore = conversation.wokeThrough;
+    conversation.wokeThrough = unread.wake;
+    if (recent.length >= this.maxWakes.count) {
+      conversation.wakes = recent;
+      return "limited";
+    }
+    // Counted as it starts, so a turn that fails still counts: a model that is
+    // down cannot be asked again on every notice and every reconnect.
+    conversation.wakes = [...recent, now].slice(-this.maxWakes.count);
+
+    // what the turn changes, so a failed one can be undone
+    const before = {
+      messages: conversation.messages.length,
+      handles: conversation.handles.length,
+      frozen: conversation.frozen.length,
+      status: conversation.status,
+      noticedThrough: conversation.noticedThrough,
+      wokeThrough: wokeBefore,
+    };
+
+    // Every unread notice rides in, passive ones too, then a line saying the
+    // user wrote none of it.
+    conversation.messages.push({ role: "user", content: [...unread.blocks, { type: "text", text: WOKEN }] });
+    conversation.noticedThrough = unread.through;
+    // Said now, before the save below: the browser holds its requests from
+    // here, rather than sending one into the lease and being refused.
+    emit({ type: "status", status: "streaming" });
+    try {
+      // Committed before the model runs, as a click is. A turn can outlast the
+      // lease, and a stream retrying meanwhile would take the conversation
+      // over and run these same notices again, tools and all. Saved now, it
+      // finds nothing left to wake for. The save is fenced: if another request
+      // took the conversation over first, this stops here.
+      await this.config.store.saveConversation(conversation);
+      await this.runTurn(conversation, emit);
+    } catch (err) {
+      // A turn that fails before any tool has run leaves a user message with no
+      // reply, and a status of `streaming`: saved, that would be a history the
+      // next message cannot follow. Put it back as it was, notices unread and
+      // able to wake again; the save on the way out overwrites the commit
+      // above. Payloads the turn stored are left behind, inert, as an
+      // overtaken turn's are. A lost lease needs no undoing: nothing more is
+      // saved.
+      if (!isStaleLease(err)) {
+        // Once a tool has run, undoing would let the notice wake again and run
+        // it again, booking or charging twice. So keep what happened, its call
+        // and result where the model can see them, and close the turn, as a
+        // failed chat turn is kept; the notice never wakes again. Before any
+        // tool has run, nothing has happened that a retry could repeat.
+        const toolRan = conversation.messages
+          .slice(before.messages + 1)
+          .some((m) => m.role === "assistant" && Array.isArray(m.content) && m.content.some((b: any) => b?.type === "tool_use"));
+        if (toolRan) {
+          conversation.status = "idle";
+        } else {
+          conversation.messages.length = before.messages;
+          conversation.handles.length = before.handles;
+          conversation.frozen.length = before.frozen;
+          conversation.status = before.status;
+          conversation.pending = null;
+          conversation.noticedThrough = before.noticedThrough;
+          conversation.wokeThrough = before.wokeThrough;
+        }
+      }
+      throw err;
+    }
+    return "woke";
+  }
+
   /**
    * The notices this conversation's history hasn't taken in yet: their model
    * text as blocks for the user message this request is about to record, and
    * the number to set `noticedThrough` to when it does. Set it only where the
    * message is pushed, so a request refused before then takes nothing in.
    */
-  private async unreadNotices(conversation: Conversation): Promise<{ blocks: TextBlock[]; through: number }> {
+  private async unreadNotices(
+    conversation: Conversation,
+  ): Promise<{ blocks: TextBlock[]; through: number; wake: number }> {
     const through = conversation.noticedThrough ?? 0;
-    if (!this.notices.size) return { blocks: [], through };
+    if (!this.notices.size) return { blocks: [], through, wake: 0 };
     const fresh = (await this.config.store.getNotices(conversation.id, through)) ?? [];
     return {
       blocks: fresh.flatMap((n) =>
         n.model === null ? [] : [{ type: "text" as const, text: `[App notification: ${n.name}] ${n.model}` }],
       ),
       through: fresh.at(-1)?.seq ?? through,
+      // the newest unread wake notice, or 0
+      wake: fresh.findLast((n) => n.kind === "wake")?.seq ?? 0,
     };
   }
 

package/src/store.ts

@@ -93,7 +93,8 @@ export function memoryStore(options: MemoryStoreOptions = {}): StoreAdapter {
         return copy(stored);
       }
       const conversation: Conversation = {
-        id: `conv_${++convSeq}`,
+        // unguessable, like pgStore's: a conversation id is all the routes ask for
+        id: `conv_${++convSeq}_${newToken().replaceAll("-", "").slice(0, 16)}`,
         status: "idle",
         messages: [],
         handles: [],
@@ -102,6 +103,8 @@ export function memoryStore(options: MemoryStoreOptions = {}): StoreAdapter {
         leaseUntil: Date.now() + leaseMs,
         leaseToken: newToken(),
         noticedThrough: 0,
+        wakes: [],
+        wokeThrough: 0,
       };
       conversations.set(conversation.id, conversation);
       return copy(conversation);