@ccmsg/cli 0.9.0 → 0.10.1

package/src/instance/identity.ts

@@ -9,7 +9,16 @@ import type { InstanceId } from "@ccmsg/protocol";
  * spelling is sized for. */
 const ID_BYTES = 16;
 
-const ID = /^[0-9a-f]{32}$/;
+export const ID = /^[0-9a-f]{32}$/;
+
+/** A fresh id of that width, for whatever is being named.
+ *
+ * Shared with the cluster ids rather than written again there: what an id has
+ * to be is unguessable-by-accident and the same width wherever it is read, and
+ * a second generator is a second answer to how wide that is. */
+export function newId(): string {
+  return randomBytes(ID_BYTES).toString("hex");
+}
 
 /** This instance's identity, read from the state directory and generated there
  * the first time it is asked for.
@@ -26,7 +35,7 @@ const ID = /^[0-9a-f]{32}$/;
 export function instanceIdentity(file: string): InstanceId {
   const held = read(file);
   if (held !== undefined) return held;
-  const made = randomBytes(ID_BYTES).toString("hex");
+  const made = newId();
   mkdirSync(dirname(file), { recursive: true });
   writeFileSync(file, `${made}\n`);
   return made;

package/src/instance/instance.ts

@@ -176,7 +176,7 @@ export async function start(options: StartOptions = {}): Promise<StartOutcome> {
   try {
     // 3. the config. A broken one ends the start rather than turning the
     // setting it carried silently off (DV-Q9).
-    const config = loadConfig(paths.configFile, paths.configHome);
+    const config = await loadConfig(paths.configDir, paths.configHome);
     // What the config says of the gateway, resolved before anything is built
     // from it: a webhook source whose secret cannot be read ends the start
     // here, for the same reason a broken config does (DV-Q9).
@@ -675,7 +675,14 @@ export class Instance {
         // address is what says the caller is local (DR-0001 §2.2).
         handle: (frame, conn) => {
           const admin = adminRequestOf(frame);
-          if (admin !== undefined) return Promise.resolve(handleAdmin(this.#auth, admin));
+          if (admin !== undefined) {
+            return Promise.resolve(
+              handleAdmin(
+                { auth: this.#auth, ...(this.#mesh === undefined ? {} : { mesh: this.#mesh }) },
+                admin,
+              ),
+            );
+          }
           return this.handle(frame, conn);
         },
       }),

package/src/instance/paths.ts

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { basename, isAbsolute, join } from "node:path";
 import { currentSession, HARNESS } from "../harness/index.ts";
+import { CLUSTERS_DIR, CLUSTERS_FILE, CONFIG_FILE, INSTANCES_DIR } from "./config.ts";
 
 /** Every path one instance uses, decided in one place (daemon-v2 §8.1).
  *
@@ -14,10 +15,17 @@ export interface InstancePaths {
   readonly configHome: string;
   /** What distinguishes this instance's files from another instance's. */
   readonly key: string;
-  /** The one file a person edits, shared by every instance on this host: it
-   * carries the defaults and the list of config homes, so it is not derived
-   * from the config home the way the rest of these are. */
+  /** Where a person writes settings, shared by every instance on this host: it
+   * holds what every instance starts from and one file per instance, so it is
+   * not derived from the config home the way the rest of these are. */
+  readonly configDir: string;
+  /** The file every instance's settings start from. */
   readonly configFile: string;
+  /** Where the file naming this config home lives, one per instance. */
+  readonly instancesDir: string;
+  /** Which clusters this host knows of, and where each one's file is. */
+  readonly clustersFile: string;
+  readonly clustersDir: string;
   readonly stateDir: string;
   /** The address clients connect to. A symlink to whichever `socketReal` is
    * currently serving, so a client's path outlives the process behind it. */
@@ -116,7 +124,11 @@ export function resolvePathsFor(configHome: string, env: Env = process.env): Ins
   return {
     configHome,
     key,
-    configFile: join(configDir, "config.json"),
+    configDir,
+    configFile: join(configDir, CONFIG_FILE),
+    instancesDir: join(configDir, INSTANCES_DIR),
+    clustersFile: join(configDir, CLUSTERS_FILE),
+    clustersDir: join(configDir, CLUSTERS_DIR),
     stateDir,
     socketDir,
     socket: join(socketDir, SOCKET_NAME),
@@ -169,9 +181,10 @@ export function resolveSupervisorSocket(env: Env = process.env): string {
 
 export const SUPERVISOR_SOCKET = "supervise.sock";
 
-/** The shared config file, for a caller that has no instance to resolve. */
+/** The file every instance's settings start from, for a caller that has no
+ * instance to resolve. */
 export function resolveConfigFile(env: Env = process.env): string {
-  return join(resolveConfigDir(env), "config.json");
+  return join(resolveConfigDir(env), CONFIG_FILE);
 }
 
 /** A name for one config home that is readable and cannot collide.

package/src/mesh/mesh.ts

@@ -334,7 +334,35 @@ export class Mesh {
    * is what found which entry that is. */
   get peers(): Endpoint[] {
     const self = this.self;
-    return this.deps.peers.filter((peer) => peer !== self);
+    return this.deps.peers.filter((peer) => peer !== self && !this.#forgotten.has(peer));
+  }
+
+  /** The peers taken off this host's list while this instance was running.
+   *
+   * Config is read once (DV-Q8) and this does not change that: what a person
+   * writes goes on taking effect at the next start. What this holds is the one
+   * edit that cannot wait for one — an endpoint this host is no longer to be
+   * talking to, which would otherwise stay connected until somebody restarted
+   * the instance. */
+  readonly #forgotten = new Set<Endpoint>();
+
+  /** Stop being a peer of this endpoint: drop the link if there is one, stop
+   * dialling it, and refuse its greeting if it dials us.
+   *
+   * Answers whether anything was actually cut, so `ccmsg mesh remove` can say
+   * which instances were talking to it rather than that it asked them all. */
+  forget(peer: Endpoint): boolean {
+    this.#forgotten.add(peer);
+    const retry = this.#retries.get(peer);
+    if (retry !== undefined) {
+      clearTimeout(retry);
+      this.#retries.delete(peer);
+    }
+    const link = this.#links.get(peer);
+    if (link === undefined) return false;
+    link.conn.close();
+    this.#drop(peer, link.conn);
+    return true;
   }
 
   /** Where peers reach this instance, as the probe settled it (§7.1).
@@ -620,7 +648,7 @@ export class Mesh {
     if (claim.ver !== MESH_VER) {
       throw new OpError("invalid_args", `this instance speaks mesh handshake ${MESH_VER}`);
     }
-    if (!this.deps.peers.includes(claim.iss)) {
+    if (!this.deps.peers.includes(claim.iss) || this.#forgotten.has(claim.iss)) {
       throw new OpError("forbidden", `${claim.iss} is not a peer of this instance`);
     }
     if (claim.aud !== self) {

package/src/upstream/events.ts

@@ -9,11 +9,50 @@ import type { LlmRequestInfo, Sid, Timestamp } from "@ccmsg/protocol";
  * nothing about which instance received it. */
 export type LlmRequestObservation = Omit<LlmRequestInfo, "main" | "instance">;
 
-/** An answer the gateway saw close. Only the two fields the sessions domain
- * reads: it says inference for that session has stopped running, and when. */
+/** How the prompt cache actually worked for one request, as the gateway read it
+ * off the answer's usage. The gateway's own closed vocabulary: a word outside
+ * it is dropped rather than carried, so nothing downstream has to decide what
+ * an unknown verdict means for a countdown. */
+export type CacheResult = "hit" | "written" | "partial" | "none" | "unknown";
+
+const CACHE_RESULTS: readonly CacheResult[] = ["hit", "written", "partial", "none", "unknown"];
+
+function cacheResultOf(value: unknown): CacheResult | undefined {
+  return CACHE_RESULTS.find((result) => result === value);
+}
+
+/** An answer the gateway saw close. It says inference for that session has
+ * stopped running and when, and it carries the one thing only an answer knows:
+ * whether the cache the request counted on was actually there. That verdict
+ * belongs to a series, so the series is named too. */
 export interface LlmResponseObservation {
   readonly sid: Sid;
   readonly at: Timestamp;
+  readonly prefix?: string;
+  readonly cache?: CacheResult;
+  /** The instant of the request this is the answer to. A series has several
+   * requests in flight, so it is what says which of them this verdict is
+   * about. */
+  readonly request_at?: Timestamp;
+}
+
+/** A keepalive the gateway raised into a conversation. Nothing here replays it;
+ * what is read is the name of the promise it carries, so a later withdrawal can
+ * be matched against it. On this notice the name is the signal's own `nonce`. */
+export interface CacheKeepaliveObservation {
+  readonly sid: Sid;
+  readonly prefix?: string;
+  readonly notice: string;
+}
+
+/** The gateway withdrawing a promised lifetime by name. `of` names one promise
+ * and only that one: a series whose latest promise is a different name has been
+ * extended by someone else since, and this notice says nothing about it. */
+export interface CacheExpiredObservation {
+  readonly sid: Sid;
+  readonly prefix?: string;
+  readonly of: string;
+  readonly at: Timestamp;
 }
 
 /** One item of a posted batch, as this instance reads it.
@@ -23,15 +62,25 @@ export interface LlmResponseObservation {
  * difference is the whole value of the log line: a batch of ignorable items is
  * the gateway working, a batch of unreadable ones is a schema that moved. */
 export type GatewayItem =
-  | { readonly kind: "request"; readonly info: LlmRequestObservation }
+  | {
+      readonly kind: "request";
+      readonly info: LlmRequestObservation;
+      /** The name of the lifetime this request promised, when it promised one.
+       * Kept beside the observation rather than inside it: it is how two
+       * notices of the gateway's are matched to each other, and nothing a
+       * client reads (§3.5). */
+      readonly notice?: string;
+    }
   | { readonly kind: "response"; readonly info: LlmResponseObservation }
+  | { readonly kind: "keepalive"; readonly info: CacheKeepaliveObservation }
+  | { readonly kind: "cache_expired"; readonly info: CacheExpiredObservation }
   | { readonly kind: "ignored" };
 
-/** Kinds the gateway posts beside the two above: a keepalive it wants replayed
- * into a session, and its keepalive strategy being held off for one. Neither
- * is read here. They are named rather than reached as "not a request", so a
- * kind the gateway grows still arrives as unreadable and shows up in the log. */
-const IGNORED = new Set(["cache_keepalive", "keepalive_paused"]);
+/** A kind the gateway posts that nothing here reads: its keepalive strategy
+ * being held off for a session. It is named rather than reached as "not a
+ * request", so a kind the gateway grows still arrives as unreadable and shows
+ * up in the log. */
+const IGNORED = new Set(["keepalive_paused"]);
 
 /** The fields whose name is the same on both sides, and whose value is already
  * this contract's unit — a count of seconds, or an instant in Unix ms. */
@@ -76,13 +125,54 @@ export function parseGatewayItem(value: unknown): GatewayItem | undefined {
     const info = responseOf(raw);
     return info === undefined ? undefined : { kind: "response", info };
   }
+  if (kind === "cache_keepalive") {
+    const info = keepaliveOf(raw);
+    // A signal that named no promise is still the gateway working: it is the
+    // notice this instance has nothing to match later, not one it misread.
+    return info === undefined ? { kind: "ignored" } : { kind: "keepalive", info };
+  }
+  if (kind === "cache_expired") {
+    const info = expiredOf(raw);
+    return info === undefined ? undefined : { kind: "cache_expired", info };
+  }
   // The forwarding notice is the one kind that carries no mark, because it
   // existed before the others did. So it is a request by position, and only
   // when it names no kind at all: an item that names one and is not handled
   // above must not be read as a request whose fields happen to line up.
   if (kind !== undefined) return undefined;
   const info = requestOf(raw);
-  return info === undefined ? undefined : { kind: "request", info };
+  if (info === undefined) return undefined;
+  const notice = raw["cache_notice"];
+  return {
+    kind: "request",
+    info,
+    ...(typeof notice === "string" && notice !== "" ? { notice } : {}),
+  };
+}
+
+function keepaliveOf(raw: Record<string, unknown>): CacheKeepaliveObservation | undefined {
+  const sid = raw["session_id"];
+  // On this notice the promise's name and the signal's own password are the
+  // same value, stated under either field, so both are read as the one name.
+  const notice = raw["cache_notice"] ?? raw["nonce"];
+  if (typeof sid !== "string" || sid === "") return undefined;
+  if (typeof notice !== "string" || notice === "") return undefined;
+  return { sid, notice, ...seriesOf(raw) };
+}
+
+function expiredOf(raw: Record<string, unknown>): CacheExpiredObservation | undefined {
+  const at = raw["ts"];
+  const sid = raw["session_id"];
+  const of = raw["of"];
+  if (!isInstant(at) || typeof sid !== "string" || sid === "") return undefined;
+  if (typeof of !== "string" || of === "") return undefined;
+  return { sid, of, at, ...seriesOf(raw) };
+}
+
+/** The series half of a key, when the notice names one. */
+function seriesOf(raw: Record<string, unknown>): { prefix?: string } {
+  const prefix = raw["prefix"];
+  return typeof prefix === "string" && prefix !== "" ? { prefix } : {};
 }
 
 function requestOf(raw: Record<string, unknown>): LlmRequestObservation | undefined {
@@ -115,7 +205,15 @@ function responseOf(raw: Record<string, unknown>): LlmResponseObservation | unde
   const at = raw["ts"];
   const sid = raw["session_id"];
   if (!isInstant(at) || typeof sid !== "string" || sid === "") return undefined;
-  return { sid, at };
+  const cache = cacheResultOf(raw["cache"]);
+  const requestAt = raw["request_ts"];
+  return {
+    sid,
+    at,
+    ...seriesOf(raw),
+    ...(cache === undefined ? {} : { cache }),
+    ...(isInstant(requestAt) ? { request_at: requestAt } : {}),
+  };
 }
 
 /** A number that can be an instant on this wire. Rejecting a non-number is

package/src/upstream/gateway.ts

@@ -143,6 +143,7 @@ export class Gateway {
       publish: deps.publish,
       ...(deps.onActivity === undefined ? {} : { onActivity: deps.onActivity }),
       ...(deps.onMoved === undefined ? {} : { onMoved: deps.onMoved }),
+      ...(deps.log === undefined ? {} : { log: deps.log }),
     });
     this.status =
       deps.setup.statusUrl === undefined
@@ -200,10 +201,14 @@ export class Gateway {
         continue;
       }
       if (item.kind === "request") {
-        this.requests.record(item.info);
+        this.requests.record(item.info, item.notice);
         this.status?.noteRequestStatus(item.info.status);
       } else if (item.kind === "response") {
-        this.requests.note(item.info.sid, item.info.at);
+        this.requests.note(item.info);
+      } else if (item.kind === "keepalive") {
+        this.requests.noteKeepalive(item.info);
+      } else if (item.kind === "cache_expired") {
+        this.requests.expire(item.info);
       }
     }
     if (unreadable > 0) {

package/src/upstream/requests.ts

@@ -1,5 +1,6 @@
 import {
   type InstanceId,
+  LLM_PROMPT_CACHE_TTL_MS,
   llmCacheWindowEndAt,
   type LlmRequestInfo,
   type Sid,
@@ -7,7 +8,12 @@ import {
 } from "@ccmsg/protocol";
 import { GATEWAY_LIVE_WINDOW_MS } from "../sessions/index.ts";
 import type { TopicValue, UpstreamResource } from "../topics/index.ts";
-import type { LlmRequestObservation } from "./events.ts";
+import type {
+  CacheExpiredObservation,
+  CacheKeepaliveObservation,
+  LlmRequestObservation,
+  LlmResponseObservation,
+} from "./events.ts";
 
 export interface LlmRequestsDeps {
   readonly self: InstanceId;
@@ -21,6 +27,7 @@ export interface LlmRequestsDeps {
    * of one row moved. Told apart from the above because what it asks for is
    * that row restated rather than the whole domain recomputed. */
   readonly onMoved?: (sid: Sid) => void;
+  readonly log?: (msg: string, fields?: Record<string, unknown>) => void;
 }
 
 /** Which of a session's gateway facts moved.
@@ -49,6 +56,10 @@ interface Series {
   /** Orders a session's series by when it started using them, which is the
    * tiebreak when it has several the sharing rule does not disqualify. */
   firstSeen: number;
+  /** The name of the lifetime this series was last promised, when the gateway
+   * named one. It is what a withdrawal is matched against, and it is held here
+   * rather than published because it means nothing outside that match. */
+  notice?: string;
 }
 
 /** What the gateway saw go upstream, per conversation series, and when each
@@ -83,7 +94,7 @@ export class LlmRequests implements UpstreamResource {
    * The newer of the two wins when a series already has one: events are
    * near-ordered in practice, but a redelivery can put an older one after a
    * newer, and a countdown must not walk backwards. */
-  record(info: LlmRequestObservation): void {
+  record(info: LlmRequestObservation, notice?: string): void {
     this.moved(info.sid, this.active(info.sid, info.received_at));
     const key = seriesKey(info.sid, info.prefix);
     const held = this.#series.get(key);
@@ -93,7 +104,14 @@ export class LlmRequests implements UpstreamResource {
     // end of the map's order, which is what makes the eviction below drop the
     // one seen least recently. `firstSeen` survives that move.
     this.#series.delete(key);
-    this.#series.set(key, { info, firstSeen: held?.firstSeen ?? ++this.#sequence });
+    // The name is replaced rather than merged: a request that promises nothing
+    // leaves the series with no promise to withdraw, which is what a request
+    // that cached nothing means.
+    this.#series.set(key, {
+      info,
+      firstSeen: held?.firstSeen ?? ++this.#sequence,
+      ...(notice === undefined ? {} : { notice }),
+    });
     while (this.#series.size > MAX_SERIES) {
       const oldest = this.#series.keys().next();
       if (oldest.done === true) break;
@@ -102,11 +120,87 @@ export class LlmRequests implements UpstreamResource {
     this.publish();
   }
 
-  /** Take one answer the gateway saw close. It moves nothing on this topic —
-   * the window belongs to the request that opened it — and only says the
-   * session was still running inference at that instant. */
-  note(sid: Sid, at: Timestamp): void {
-    this.moved(sid, this.active(sid, at));
+  /** Take one answer the gateway saw close. It says the session was still
+   * running inference at that instant, and it carries the one verdict only an
+   * answer holds: whether the cache the request counted on was there. `hit` and
+   * `partial` confirm the window the request stated, so nothing moves; `written`
+   * says that window was a promise about a cache that no longer existed. */
+  note(info: LlmResponseObservation): void {
+    this.moved(info.sid, this.active(info.sid, info.at));
+    if (info.cache === "written") this.rebuild(info);
+  }
+
+  /** A keepalive the gateway raised names the lifetime it promises. Held
+   * against the series so a withdrawal naming it can be told from one naming a
+   * promise since replaced. A series nothing is held for has no window to
+   * withdraw, so the name has nothing to attach to. */
+  noteKeepalive(info: CacheKeepaliveObservation): void {
+    const series = this.#series.get(seriesKey(info.sid, info.prefix));
+    if (series === undefined) return;
+    series.notice = info.notice;
+  }
+
+  /** The gateway withdrawing a promised lifetime by name.
+   *
+   * Only the series whose latest promise is the one named loses its window: a
+   * different name means that promise was replaced — by this gateway's next
+   * request or by another gateway watching the same series — and the window
+   * standing now is not the one being withdrawn. The window going to zero is
+   * the row leaving, since this topic carries the open ones. */
+  expire(info: CacheExpiredObservation): void {
+    const key = seriesKey(info.sid, info.prefix);
+    const series = this.#series.get(key);
+    if (series === undefined || series.notice !== info.of) return;
+    this.#series.delete(key);
+    this.publish();
+  }
+
+  /** The cache was gone and the whole prompt was written again, so the window
+   * starts at the moment of that writing rather than where the request said.
+   *
+   * The chain the request projected (`cache_until_at` and the breakeven beside
+   * it) described a chain that was not continued, so it is dropped rather than
+   * carried onto a window that begins elsewhere; the gateway states the new
+   * projection on its next event. The promise is dropped with it: it named the
+   * lifetime that just turned out not to exist. */
+  private rebuild(info: LlmResponseObservation): void {
+    const key = seriesKey(info.sid, info.prefix);
+    const series = this.#series.get(key);
+    if (series === undefined) return;
+    // The answer names the request it belongs to. A verdict about a request
+    // the series has already replaced is about a window that is no longer the
+    // one drawn, so it moves nothing.
+    const held = series.info;
+    if (
+      info.request_at === undefined
+        ? held.received_at > info.at
+        : info.request_at !== held.received_at
+    ) {
+      return;
+    }
+    // The gateway judged the signal applied because it came back in time; the
+    // answer says what it was applied to was written from nothing. Said out
+    // loud because it is the one case where those two readings disagree.
+    if (held.keepalive === "applied") {
+      this.deps.log?.("a keepalive was applied to a cache that had to be rebuilt", {
+        sid: info.sid,
+        ...(info.prefix === undefined ? {} : { prefix: info.prefix }),
+      });
+    }
+    const {
+      cache_until_at: _until,
+      cache_until_count: _untilCount,
+      cache_breakeven_until_at: _breakeven,
+      cache_breakeven_count: _breakevenCount,
+      ...rest
+    } = held;
+    const ttl =
+      rest.cache_ttl_secs === undefined ? LLM_PROMPT_CACHE_TTL_MS : rest.cache_ttl_secs * 1000;
+    this.#series.set(key, {
+      firstSeen: series.firstSeen,
+      info: { ...rest, cache_since_at: info.at, cache_expires_at: info.at + ttl },
+    });
+    this.publish();
   }
 
   /** Tell whoever holds the row what this event moved for that session. */