@ccmsg/cli 0.12.0 → 0.14.0

package/src/mesh/mesh.ts

@@ -18,7 +18,7 @@ import {
   type SettledIdentity,
 } from "../dispatch/index.ts";
 import type { TopicValue } from "../topics/index.ts";
-import { AUTH_TOPIC, isClusterTopic, Relay } from "./relay.ts";
+import { AUTH_TOPIC, isMeshTopic, Relay } from "./relay.ts";
 import {
   ALLOWED_ALGS,
   EphemeralKey,
@@ -388,7 +388,7 @@ export class Mesh {
    * whether it can be reached right now (DESIGN §7.5).
    *
    * A peer no handshake has settled yet is listed without an id. The operator
-   * configured that endpoint, so it is an entry of the cluster whether or not
+   * configured that endpoint, so it is an entry of the mesh whether or not
    * anything has answered there — and leaving it out would hide exactly the
    * peer whose link is down, which is the one a reader is looking for. */
   instances(): InstanceInfo[] {
@@ -408,7 +408,7 @@ export class Mesh {
 
   /** Whether any peer is currently out of reach.
    *
-   * What separates "no instance in the cluster knows this session" from "an
+   * What separates "no instance in the mesh knows this session" from "an
    * instance that might know it cannot be asked" — the one distinction DESIGN §6.6
    * says rests on the mesh's connection state and on nothing else. */
   anyUnreachable(): boolean {
@@ -417,7 +417,7 @@ export class Mesh {
 
   /** Which instance should answer for a session this instance does not hold.
    *
-   * A session the cluster has named belongs to the instance its `peers` row
+   * A session the mesh has named belongs to the instance its `peers` row
    * states. One nobody has named while a peer is out of reach is answered with
    * that peer: forwarding there fails and the caller is told
    * `instance_unreachable`, which is what DESIGN §6.6 asks for in place of deciding
@@ -532,7 +532,7 @@ export class Mesh {
    *
    * The caller the envelope names, on the link it arrived over. A request that
    * names none is dispatched as the link itself, whose role is `instance` —
-   * which the attribute table already answers, since no instance-local op is
+   * which the attribute table already answers, since no owner_instance op is
    * open to an instance. */
   caller(conn: Requester, caller: CallerIdentity | undefined): Requester {
     const link = this.#linkOf.get(conn);
@@ -553,12 +553,12 @@ export class Mesh {
 
   /** The current value of a relayed topic, one entry per instance that has
    * stated one. Handed to a fresh local subscriber beside this instance's own
-   * snapshot, so it opens on the cluster rather than on us. */
+   * snapshot, so it opens on the mesh rather than on us. */
   snapshot(topic: string): readonly TopicValue[] {
     return this.relay.snapshot(topic);
   }
 
-  /** A local subscriber appeared on a cluster topic, or the last one left.
+  /** A local subscriber appeared on a mesh topic, or the last one left.
    *
    * The subscription travels: what a subscriber asks of this instance, this
    * instance asks of every peer, and the frames come back unchanged (DESIGN §7.4).
@@ -567,7 +567,7 @@ export class Mesh {
     // `auth.records` is never given up and never asked for on demand: every
     // instance holds the whole set whether or not anything local is watching
     // it, the way `peers` is also the routing table (DESIGN §7.4, DR-0001 §2.6).
-    if (topic === AUTH_TOPIC || !isClusterTopic(topic)) return;
+    if (topic === AUTH_TOPIC || !isMeshTopic(topic)) return;
     if (wanted) {
       if (this.#demanded.has(topic)) return;
       this.#demanded.add(topic);
@@ -593,7 +593,7 @@ export class Mesh {
       topic,
       // The instance asks on behalf of whoever subscribed to it, and what they
       // have in common is that they are this deployment's people rather than
-      // any one session: a cluster topic is the same value for all of them
+      // any one session: a mesh topic is the same value for all of them
       // (DESIGN §6.2), so there is nothing narrower to name.
       //
       // `auth.records` is the exception, and the one topic no person may hear:
@@ -1090,12 +1090,12 @@ export class Mesh {
     //
     // The far end has no other way to learn this instance is going: it would
     // keep the link, keep answering `reachable`, and keep routing
-    // `instance-local` ops here until its own heartbeat gave up minutes later,
+    // `owner_instance` ops here until its own heartbeat gave up minutes later,
     // where the disconnection of DESIGN §7.5 is supposed to be immediate. Which side
     // dialled a link is decided by the glare rule from a comparison of
     // endpoint strings (mesh-peer-auth §8.1), so which of a peer's links this instance
     // accepted is not something either end chose — leaving those open makes a
-    // clean stop look like a silent one to whichever half of the cluster the
+    // clean stop look like a silent one to whichever half of the mesh the
     // comparison put on this side.
     //
     // An accepted socket is transport's to release (DESIGN §8.5 step 5), and left to

package/src/mesh/relay.ts

@@ -9,7 +9,7 @@ import {
 } from "@ccmsg/protocol";
 import { Elements, type TopicValue } from "../topics/index.ts";
 
-/** The rows of sessions the whole cluster is seen through.
+/** The rows of sessions the whole mesh is seen through.
  *
  * They are `element`-granular, and an element topic is relayable only when its
  * elements say whose they are: a row here names the instance that holds the
@@ -19,13 +19,13 @@ import { Elements, type TopicValue } from "../topics/index.ts";
  * such name — an `inbox` frame belongs to a session, not to an instance. */
 const ROW_TOPICS: readonly string[] = ["peers", "agents"];
 
-/** The topics a subscriber sees the whole cluster on.
+/** The topics a subscriber sees the whole mesh on.
  *
  * A per-instance whole is relayable by construction (DESIGN §6.2): a frame replaces
  * its own instance's entries and leaves every other instance's alone, so
  * several instances can state the same topic name without colliding. The rows
  * above are relayable for the same reason read one element at a time. */
-export const CLUSTER_TOPICS: readonly string[] = [
+export const MESH_TOPICS: readonly string[] = [
   ...PLAIN_TOPICS.filter((topic) => TOPIC_ATTRIBUTES[topic].granularity === "per_instance_whole"),
   ...ROW_TOPICS,
 ];
@@ -37,8 +37,8 @@ export const CLUSTER_TOPICS: readonly string[] = [
  * the set this instance holds rather than held here (DR-0001 §2.6). */
 export const AUTH_TOPIC = "auth.records";
 
-export function isClusterTopic(topic: string): boolean {
-  return CLUSTER_TOPICS.includes(topic);
+export function isMeshTopic(topic: string): boolean {
+  return MESH_TOPICS.includes(topic);
 }
 
 /** Whether what a frame of this topic carries is rows to be merged rather than
@@ -64,7 +64,7 @@ export interface RelayDeps {
 }
 
 /** What the peers said, held on this instance so that losing a peer does not
- * empty the cluster view (DESIGN §7.5).
+ * empty the mesh view (DESIGN §7.5).
  *
  * Two things live here and nowhere else: the last whole value each instance
  * stated per topic, and whether that instance can be reached right now. The
@@ -99,7 +99,7 @@ export class Relay {
    * unchanged — recomputing it would put the same judgement in two places
    * (DESIGN §7.4). */
   accept(instance: InstanceId, topic: string, data: unknown, snapshot = false): void {
-    if (!isClusterTopic(topic)) return;
+    if (!isMeshTopic(topic)) return;
     this.#sweep();
     const held = this.#held.get(instance) ?? new Map<string, unknown>();
     this.#held.set(instance, held);
@@ -174,7 +174,7 @@ export class Relay {
     return values;
   }
 
-  /** Which instance a session belongs to, read from the cluster values the
+  /** Which instance a session belongs to, read from the mesh values the
    * peers stated (DESIGN §7.3).
    *
    * `peers` names every session an instance holds, connected and lost alike,

package/src/messaging/delivery.ts

@@ -2,6 +2,8 @@ import type {
   CallerIdentity,
   CandidateSession,
   InboxMessage,
+  InboxRemoved,
+  InboxRemovedReason,
   InstanceId,
   MessageSendArgs,
   MessageSendResult,
@@ -21,7 +23,12 @@ import {
   OpError,
   type Requester,
 } from "../dispatch/index.ts";
-import type { PublishOutcome, TopicValue, UpstreamResource } from "../topics/index.ts";
+import {
+  PEOPLE,
+  type PublishOutcome,
+  type TopicValue,
+  type UpstreamResource,
+} from "../topics/index.ts";
 import type { DirectRoute } from "./direct.ts";
 import type { Inbox } from "./inbox.ts";
 
@@ -38,15 +45,15 @@ export interface SessionLookup {
   peerRows(): PeerInfo[];
 }
 
-/** The rest of the cluster, for a message addressed outside this instance.
+/** The rest of the mesh, for a message addressed outside this instance.
  *
- * `message.send` is a `cluster` op — any instance may be asked — but a message
+ * `message.send` is an `any_instance` op — any instance may be asked — but a message
  * reaches a session through the session's own connections, which are held by
  * the instance it greeted. So the op is answered here by carrying it there
- * (DESIGN §2.2 step 6 is about `instance-local` ops; this is the same forwarding for
+ * (DESIGN §2.2 step 6 is about `owner_instance` ops; this is the same forwarding for
  * the one op whose subject is elsewhere while its op is not). */
-export interface Cluster {
-  /** Which instance holds this session, or nothing when the cluster has not
+export interface MeshReach {
+  /** Which instance holds this session, or nothing when the mesh has not
    * named it. */
   ownerOf(sid: Sid): InstanceId | undefined;
   /** Whether an instance that might hold it cannot be asked right now. */
@@ -64,14 +71,19 @@ export interface DeliveryDeps {
   readonly sessions: SessionLookup;
   /** Absent on an instance with no mesh, where every session it can name is
    * its own. */
-  readonly cluster?: Cluster;
+  readonly mesh?: MeshReach;
   readonly inbox: Inbox;
   /** Route (a). Off until it is confirmed against a running harness, which is
    * condition 0 of DESIGN §6.5 and is why this is handed in rather than built here. */
   readonly direct: DirectRoute;
   /** The one way a value reaches subscribers (DESIGN §6.1), narrowed to the session a
    * message is for. */
-  readonly publish: (topic: string, data: unknown, instance: InstanceId, to: Sid) => PublishOutcome;
+  readonly publish: (
+    topic: string,
+    data: unknown,
+    instance: InstanceId,
+    to: Sid | typeof PEOPLE,
+  ) => PublishOutcome;
   /** How many of that session's connections are listening on `inbox`. */
   readonly listeners: (topic: string, to: Sid) => number;
 }
@@ -97,6 +109,14 @@ export class Delivery implements UpstreamResource {
 
   constructor(private readonly deps: DeliveryDeps) {
     this.#counter = deps.inbox.lastCounter(`${deps.self}/`);
+    // Every way out of the inbox is stated to the watchers, and the inbox is
+    // the one that knows about all of them — handed over, timed out, dropped
+    // for a newer message. Read from there rather than published beside each
+    // call that removes something, so a way out nobody thought of here is
+    // still a removal somebody watching sees (DESIGN §6.7).
+    deps.inbox.onRemoved((mid, reason) => {
+      this.#removed([mid], reason);
+    });
   }
 
   /** `message.send`. The op fails only for a sid nobody knows; every other
@@ -123,47 +143,87 @@ export class Delivery implements UpstreamResource {
     if (direct === "refused") {
       // Turned away for now, which is neither delivered nor undeliverable: it
       // waits in the inbox and is offered again (DESIGN §6.8).
-      this.deps.inbox.hold(to, message);
+      await this.#hold(to, message);
       return { delivered: false, reason: "throttled" };
     }
 
     if (this.deps.listeners(INBOX, to) > 0) {
       if (this.deps.publish(INBOX, [message], this.deps.self, to) === "ok") {
+        // Route (b) is the topic itself, so the message never waits anywhere:
+        // the watchers are told both halves at once — it arrived, and it is
+        // gone because the session has it — since a view built from frames
+        // alone would otherwise show it waiting for good.
+        this.#watchers(to, [message]);
+        this.#removed([message.mid], "delivered");
         return { delivered: true };
       }
       // The session is listening but is behind on what it has already been
       // offered, which is the same standing as route (a) turning the message
       // away: it waits in the inbox and is offered again (DESIGN §6.8).
-      this.deps.inbox.hold(to, message);
+      await this.#hold(to, message);
       return { delivered: false, reason: "throttled" };
     }
 
-    const { evicted } = this.deps.inbox.hold(to, message);
+    const { evicted } = await this.#hold(to, message);
     return this.#undelivered(to, evicted ? "inbox_full" : this.#reason(state));
   };
 
+  /** Hold a message the session could not take, and say so to the watchers.
+   *
+   * Stated before it is held, so that a message dropped to make room for it
+   * reads in the order the two happened: a removal of something the watcher
+   * has, rather than of something it is about to be told about. */
+  async #hold(to: Sid, message: InboxMessage): Promise<{ evicted: boolean }> {
+    this.#watchers(to, [message]);
+    return await this.deps.inbox.hold(to, message);
+  }
+
+  /** What is waiting, as somebody looking at it from outside reads it.
+   *
+   * A person holds every session's inbox in one subscription, so the rows they
+   * are answered with name their recipient; the session's own do not, since
+   * its subscription is already the recipient (contract, `InboxMessage.to`). */
+  #watchers(to: Sid, messages: readonly InboxMessage[]): void {
+    if (messages.length === 0) return;
+    this.deps.publish(
+      INBOX,
+      messages.map((message) => ({ ...message, to })),
+      this.deps.self,
+      PEOPLE,
+    );
+  }
+
+  /** A message has left an inbox, which only a watcher has anything left to do
+   * with: the session it was for either has it or never will, and neither is
+   * something to tell it on the topic it receives messages on. */
+  #removed(mids: readonly Mid[], reason: InboxRemovedReason): void {
+    if (mids.length === 0) return;
+    const gone: InboxRemoved[] = mids.map((mid) => ({ mid, removed: true, reason }));
+    this.deps.publish(INBOX, gone, this.deps.self, PEOPLE);
+  }
+
   /** A session this instance does not hold: carried to the instance that does,
-   * or named as one the cluster cannot answer for right now.
+   * or named as one the mesh cannot answer for right now.
    *
-   * Nothing when the cluster has no such session anywhere and every instance
+   * Nothing when the mesh has no such session anywhere and every instance
    * could be asked — which is the only case `session_not_found` covers (DESIGN §6.6).
    * While an instance is out of reach the sid may well be its, so the sender is
    * told the reason rather than that the session does not exist. The message is
    * not held here either: the inbox that would offer it again is the one on the
    * instance that owns the session (DESIGN §6.7). */
   async #elsewhere(to: Sid, input: HandlerInput): Promise<MessageSendResult | undefined> {
-    const cluster = this.deps.cluster;
-    if (cluster === undefined) return undefined;
-    const owner = cluster.ownerOf(to);
+    const reach = this.deps.mesh;
+    if (reach === undefined) return undefined;
+    const owner = reach.ownerOf(to);
     if (owner === undefined || owner === this.deps.self) {
-      return cluster.anyUnreachable()
+      return reach.anyUnreachable()
         ? { delivered: false, reason: "instance_unreachable" }
         : undefined;
     }
     // The sender, as the owning instance will run the op as: the identity the
     // connection greeted with, which is the same thing `message.send` reads to
     // decide who a message is from (DESIGN §6.5).
-    const answer = await cluster.forward(owner, input.args, callerOf(input));
+    const answer = await reach.forward(owner, input.args, callerOf(input));
     if (answer.kind === "reply") {
       const { ok: _ok, request_id: _id, ...body } = answer.response;
       return body as unknown as MessageSendResult;
@@ -187,11 +247,18 @@ export class Delivery implements UpstreamResource {
    * nothing waiting are not asked about, so the cost of a change nobody is owed
    * anything after is one map read. */
   retry = async (): Promise<void> => {
+    const offers: Promise<void>[] = [];
     for (const sid of this.deps.inbox.sids()) {
       const state = this.deps.sessions.classify(sid);
       if (state === undefined || state === "paused" || state === "disappeared") continue;
-      await this.#offer(sid);
+      // One session at a time within its own offer, every session at once
+      // across them: a session that is slow to answer, or that never does
+      // before its deadline, is not a reason the next session waits (DR-0015).
+      // The order within a session is what `#offer` holds to, and it guards
+      // itself per sid.
+      offers.push(this.#offer(sid));
     }
+    await Promise.all(offers);
   };
 
   /** Hand a session what it is owed, oldest first, over route (a).
@@ -215,7 +282,7 @@ export class Delivery implements UpstreamResource {
         // delivered, and a daemon killed here must not offer those again.
         if (outcome !== "delivered") break;
         this.#claimed.get(to)?.delete(message.mid);
-        this.deps.inbox.delivered(to, [message.mid]);
+        await this.deps.inbox.delivered(to, [message.mid]);
       }
     } finally {
       this.#claimed.delete(to);
@@ -243,22 +310,54 @@ export class Delivery implements UpstreamResource {
    *
    * A message an offer over route (a) has claimed is left out: it is on its way
    * on the other route, and the session subscribing while that runs must not
-   * make it two messages. */
+   * make it two messages.
+   *
+   * **A person's subscription is a view, not a delivery.** They are answered
+   * with everything waiting anywhere on this instance, each row naming its
+   * recipient, and the inbox is left exactly as it was — a person is not who
+   * any of it was addressed to, and a view that consumed what it looked at
+   * would deliver messages to nobody by being opened. Nothing is left out of
+   * it either: a message an offer has claimed is still waiting until that
+   * offer says otherwise, and that is what somebody watching wants to see. */
   snapshot(topic: string, conn: Requester): readonly TopicValue[] {
     const identity = conn.identity;
-    const sid = identity.state === "settled" ? identity.sid : undefined;
-    if (topic !== INBOX || sid === undefined) return [];
+    if (topic !== INBOX || identity.state !== "settled") return [];
+    const sid = identity.sid;
+    // A settled connection that names no session is somebody watching: the
+    // topic table lets nobody but a session and a person subscribe here, so
+    // the sid is the whole of the difference and no role is read for it (M1).
+    if (sid === undefined) return [{ instance: this.deps.self, data: this.#waiting() }];
     const claimed = this.#claimed.get(sid);
     const held = this.deps.inbox
       .undelivered(sid)
       .filter((message) => claimed?.has(message.mid) !== true);
-    this.deps.inbox.delivered(
-      sid,
-      held.map((message) => message.mid),
-    );
+    // The messages leave the inbox here, in memory, and the lines saying so go
+    // behind whatever the file already owes (DR-0015): the frame carrying them
+    // is queued on the connection before this returns, so what the session has
+    // been handed is settled whether or not the line has landed yet.
+    void this.deps.inbox
+      .delivered(
+        sid,
+        held.map((message) => message.mid),
+      )
+      .catch(() => {
+        // A line that could not be written costs a message being offered again
+        // on the next run, which is what the inbox does about anything it is
+        // unsure of.
+      });
     return [{ instance: this.deps.self, data: held }];
   }
 
+  /** Everything still undelivered on this instance, oldest first within each
+   * session, each row naming who it is for. */
+  #waiting(): InboxMessage[] {
+    const rows: InboxMessage[] = [];
+    for (const sid of this.deps.inbox.sids()) {
+      for (const message of this.deps.inbox.undelivered(sid)) rows.push({ ...message, to: sid });
+    }
+    return rows;
+  }
+
   /** The reason a message is waiting, named from the classification alone
    * (DESIGN §6.6). `preparing` is the live session with nowhere to put it: it is there,
    * route (a) did not carry it, and nothing of its is listening yet.