@ccmsg/cli 0.9.0 → 0.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ccmsg/cli",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "The ccmsg daemon, CLI and agent plugins for one instance (= one config home)",
   "license": "MIT",
   "author": "kawaz",

package/src/upstream/events.ts

@@ -9,11 +9,50 @@ import type { LlmRequestInfo, Sid, Timestamp } from "@ccmsg/protocol";
  * nothing about which instance received it. */
 export type LlmRequestObservation = Omit<LlmRequestInfo, "main" | "instance">;
 
-/** An answer the gateway saw close. Only the two fields the sessions domain
- * reads: it says inference for that session has stopped running, and when. */
+/** How the prompt cache actually worked for one request, as the gateway read it
+ * off the answer's usage. The gateway's own closed vocabulary: a word outside
+ * it is dropped rather than carried, so nothing downstream has to decide what
+ * an unknown verdict means for a countdown. */
+export type CacheResult = "hit" | "written" | "partial" | "none" | "unknown";
+
+const CACHE_RESULTS: readonly CacheResult[] = ["hit", "written", "partial", "none", "unknown"];
+
+function cacheResultOf(value: unknown): CacheResult | undefined {
+  return CACHE_RESULTS.find((result) => result === value);
+}
+
+/** An answer the gateway saw close. It says inference for that session has
+ * stopped running and when, and it carries the one thing only an answer knows:
+ * whether the cache the request counted on was actually there. That verdict
+ * belongs to a series, so the series is named too. */
 export interface LlmResponseObservation {
   readonly sid: Sid;
   readonly at: Timestamp;
+  readonly prefix?: string;
+  readonly cache?: CacheResult;
+  /** The instant of the request this is the answer to. A series has several
+   * requests in flight, so it is what says which of them this verdict is
+   * about. */
+  readonly request_at?: Timestamp;
+}
+
+/** A keepalive the gateway raised into a conversation. Nothing here replays it;
+ * what is read is the name of the promise it carries, so a later withdrawal can
+ * be matched against it. On this notice the name is the signal's own `nonce`. */
+export interface CacheKeepaliveObservation {
+  readonly sid: Sid;
+  readonly prefix?: string;
+  readonly notice: string;
+}
+
+/** The gateway withdrawing a promised lifetime by name. `of` names one promise
+ * and only that one: a series whose latest promise is a different name has been
+ * extended by someone else since, and this notice says nothing about it. */
+export interface CacheExpiredObservation {
+  readonly sid: Sid;
+  readonly prefix?: string;
+  readonly of: string;
+  readonly at: Timestamp;
 }
 
 /** One item of a posted batch, as this instance reads it.
@@ -23,15 +62,25 @@ export interface LlmResponseObservation {
  * difference is the whole value of the log line: a batch of ignorable items is
  * the gateway working, a batch of unreadable ones is a schema that moved. */
 export type GatewayItem =
-  | { readonly kind: "request"; readonly info: LlmRequestObservation }
+  | {
+      readonly kind: "request";
+      readonly info: LlmRequestObservation;
+      /** The name of the lifetime this request promised, when it promised one.
+       * Kept beside the observation rather than inside it: it is how two
+       * notices of the gateway's are matched to each other, and nothing a
+       * client reads (§3.5). */
+      readonly notice?: string;
+    }
   | { readonly kind: "response"; readonly info: LlmResponseObservation }
+  | { readonly kind: "keepalive"; readonly info: CacheKeepaliveObservation }
+  | { readonly kind: "cache_expired"; readonly info: CacheExpiredObservation }
   | { readonly kind: "ignored" };
 
-/** Kinds the gateway posts beside the two above: a keepalive it wants replayed
- * into a session, and its keepalive strategy being held off for one. Neither
- * is read here. They are named rather than reached as "not a request", so a
- * kind the gateway grows still arrives as unreadable and shows up in the log. */
-const IGNORED = new Set(["cache_keepalive", "keepalive_paused"]);
+/** A kind the gateway posts that nothing here reads: its keepalive strategy
+ * being held off for a session. It is named rather than reached as "not a
+ * request", so a kind the gateway grows still arrives as unreadable and shows
+ * up in the log. */
+const IGNORED = new Set(["keepalive_paused"]);
 
 /** The fields whose name is the same on both sides, and whose value is already
  * this contract's unit — a count of seconds, or an instant in Unix ms. */
@@ -76,13 +125,54 @@ export function parseGatewayItem(value: unknown): GatewayItem | undefined {
     const info = responseOf(raw);
     return info === undefined ? undefined : { kind: "response", info };
   }
+  if (kind === "cache_keepalive") {
+    const info = keepaliveOf(raw);
+    // A signal that named no promise is still the gateway working: it is the
+    // notice this instance has nothing to match later, not one it misread.
+    return info === undefined ? { kind: "ignored" } : { kind: "keepalive", info };
+  }
+  if (kind === "cache_expired") {
+    const info = expiredOf(raw);
+    return info === undefined ? undefined : { kind: "cache_expired", info };
+  }
   // The forwarding notice is the one kind that carries no mark, because it
   // existed before the others did. So it is a request by position, and only
   // when it names no kind at all: an item that names one and is not handled
   // above must not be read as a request whose fields happen to line up.
   if (kind !== undefined) return undefined;
   const info = requestOf(raw);
-  return info === undefined ? undefined : { kind: "request", info };
+  if (info === undefined) return undefined;
+  const notice = raw["cache_notice"];
+  return {
+    kind: "request",
+    info,
+    ...(typeof notice === "string" && notice !== "" ? { notice } : {}),
+  };
+}
+
+function keepaliveOf(raw: Record<string, unknown>): CacheKeepaliveObservation | undefined {
+  const sid = raw["session_id"];
+  // On this notice the promise's name and the signal's own password are the
+  // same value, stated under either field, so both are read as the one name.
+  const notice = raw["cache_notice"] ?? raw["nonce"];
+  if (typeof sid !== "string" || sid === "") return undefined;
+  if (typeof notice !== "string" || notice === "") return undefined;
+  return { sid, notice, ...seriesOf(raw) };
+}
+
+function expiredOf(raw: Record<string, unknown>): CacheExpiredObservation | undefined {
+  const at = raw["ts"];
+  const sid = raw["session_id"];
+  const of = raw["of"];
+  if (!isInstant(at) || typeof sid !== "string" || sid === "") return undefined;
+  if (typeof of !== "string" || of === "") return undefined;
+  return { sid, of, at, ...seriesOf(raw) };
+}
+
+/** The series half of a key, when the notice names one. */
+function seriesOf(raw: Record<string, unknown>): { prefix?: string } {
+  const prefix = raw["prefix"];
+  return typeof prefix === "string" && prefix !== "" ? { prefix } : {};
 }
 
 function requestOf(raw: Record<string, unknown>): LlmRequestObservation | undefined {
@@ -115,7 +205,15 @@ function responseOf(raw: Record<string, unknown>): LlmResponseObservation | unde
   const at = raw["ts"];
   const sid = raw["session_id"];
   if (!isInstant(at) || typeof sid !== "string" || sid === "") return undefined;
-  return { sid, at };
+  const cache = cacheResultOf(raw["cache"]);
+  const requestAt = raw["request_ts"];
+  return {
+    sid,
+    at,
+    ...seriesOf(raw),
+    ...(cache === undefined ? {} : { cache }),
+    ...(isInstant(requestAt) ? { request_at: requestAt } : {}),
+  };
 }
 
 /** A number that can be an instant on this wire. Rejecting a non-number is

package/src/upstream/gateway.ts

@@ -143,6 +143,7 @@ export class Gateway {
       publish: deps.publish,
       ...(deps.onActivity === undefined ? {} : { onActivity: deps.onActivity }),
       ...(deps.onMoved === undefined ? {} : { onMoved: deps.onMoved }),
+      ...(deps.log === undefined ? {} : { log: deps.log }),
     });
     this.status =
       deps.setup.statusUrl === undefined
@@ -200,10 +201,14 @@ export class Gateway {
         continue;
       }
       if (item.kind === "request") {
-        this.requests.record(item.info);
+        this.requests.record(item.info, item.notice);
         this.status?.noteRequestStatus(item.info.status);
       } else if (item.kind === "response") {
-        this.requests.note(item.info.sid, item.info.at);
+        this.requests.note(item.info);
+      } else if (item.kind === "keepalive") {
+        this.requests.noteKeepalive(item.info);
+      } else if (item.kind === "cache_expired") {
+        this.requests.expire(item.info);
       }
     }
     if (unreadable > 0) {

package/src/upstream/requests.ts

@@ -1,5 +1,6 @@
 import {
   type InstanceId,
+  LLM_PROMPT_CACHE_TTL_MS,
   llmCacheWindowEndAt,
   type LlmRequestInfo,
   type Sid,
@@ -7,7 +8,12 @@ import {
 } from "@ccmsg/protocol";
 import { GATEWAY_LIVE_WINDOW_MS } from "../sessions/index.ts";
 import type { TopicValue, UpstreamResource } from "../topics/index.ts";
-import type { LlmRequestObservation } from "./events.ts";
+import type {
+  CacheExpiredObservation,
+  CacheKeepaliveObservation,
+  LlmRequestObservation,
+  LlmResponseObservation,
+} from "./events.ts";
 
 export interface LlmRequestsDeps {
   readonly self: InstanceId;
@@ -21,6 +27,7 @@ export interface LlmRequestsDeps {
    * of one row moved. Told apart from the above because what it asks for is
    * that row restated rather than the whole domain recomputed. */
   readonly onMoved?: (sid: Sid) => void;
+  readonly log?: (msg: string, fields?: Record<string, unknown>) => void;
 }
 
 /** Which of a session's gateway facts moved.
@@ -49,6 +56,10 @@ interface Series {
   /** Orders a session's series by when it started using them, which is the
    * tiebreak when it has several the sharing rule does not disqualify. */
   firstSeen: number;
+  /** The name of the lifetime this series was last promised, when the gateway
+   * named one. It is what a withdrawal is matched against, and it is held here
+   * rather than published because it means nothing outside that match. */
+  notice?: string;
 }
 
 /** What the gateway saw go upstream, per conversation series, and when each
@@ -83,7 +94,7 @@ export class LlmRequests implements UpstreamResource {
    * The newer of the two wins when a series already has one: events are
    * near-ordered in practice, but a redelivery can put an older one after a
    * newer, and a countdown must not walk backwards. */
-  record(info: LlmRequestObservation): void {
+  record(info: LlmRequestObservation, notice?: string): void {
     this.moved(info.sid, this.active(info.sid, info.received_at));
     const key = seriesKey(info.sid, info.prefix);
     const held = this.#series.get(key);
@@ -93,7 +104,14 @@ export class LlmRequests implements UpstreamResource {
     // end of the map's order, which is what makes the eviction below drop the
     // one seen least recently. `firstSeen` survives that move.
     this.#series.delete(key);
-    this.#series.set(key, { info, firstSeen: held?.firstSeen ?? ++this.#sequence });
+    // The name is replaced rather than merged: a request that promises nothing
+    // leaves the series with no promise to withdraw, which is what a request
+    // that cached nothing means.
+    this.#series.set(key, {
+      info,
+      firstSeen: held?.firstSeen ?? ++this.#sequence,
+      ...(notice === undefined ? {} : { notice }),
+    });
     while (this.#series.size > MAX_SERIES) {
       const oldest = this.#series.keys().next();
       if (oldest.done === true) break;
@@ -102,11 +120,87 @@ export class LlmRequests implements UpstreamResource {
     this.publish();
   }
 
-  /** Take one answer the gateway saw close. It moves nothing on this topic —
-   * the window belongs to the request that opened it — and only says the
-   * session was still running inference at that instant. */
-  note(sid: Sid, at: Timestamp): void {
-    this.moved(sid, this.active(sid, at));
+  /** Take one answer the gateway saw close. It says the session was still
+   * running inference at that instant, and it carries the one verdict only an
+   * answer holds: whether the cache the request counted on was there. `hit` and
+   * `partial` confirm the window the request stated, so nothing moves; `written`
+   * says that window was a promise about a cache that no longer existed. */
+  note(info: LlmResponseObservation): void {
+    this.moved(info.sid, this.active(info.sid, info.at));
+    if (info.cache === "written") this.rebuild(info);
+  }
+
+  /** A keepalive the gateway raised names the lifetime it promises. Held
+   * against the series so a withdrawal naming it can be told from one naming a
+   * promise since replaced. A series nothing is held for has no window to
+   * withdraw, so the name has nothing to attach to. */
+  noteKeepalive(info: CacheKeepaliveObservation): void {
+    const series = this.#series.get(seriesKey(info.sid, info.prefix));
+    if (series === undefined) return;
+    series.notice = info.notice;
+  }
+
+  /** The gateway withdrawing a promised lifetime by name.
+   *
+   * Only the series whose latest promise is the one named loses its window: a
+   * different name means that promise was replaced — by this gateway's next
+   * request or by another gateway watching the same series — and the window
+   * standing now is not the one being withdrawn. The window going to zero is
+   * the row leaving, since this topic carries the open ones. */
+  expire(info: CacheExpiredObservation): void {
+    const key = seriesKey(info.sid, info.prefix);
+    const series = this.#series.get(key);
+    if (series === undefined || series.notice !== info.of) return;
+    this.#series.delete(key);
+    this.publish();
+  }
+
+  /** The cache was gone and the whole prompt was written again, so the window
+   * starts at the moment of that writing rather than where the request said.
+   *
+   * The chain the request projected (`cache_until_at` and the breakeven beside
+   * it) described a chain that was not continued, so it is dropped rather than
+   * carried onto a window that begins elsewhere; the gateway states the new
+   * projection on its next event. The promise is dropped with it: it named the
+   * lifetime that just turned out not to exist. */
+  private rebuild(info: LlmResponseObservation): void {
+    const key = seriesKey(info.sid, info.prefix);
+    const series = this.#series.get(key);
+    if (series === undefined) return;
+    // The answer names the request it belongs to. A verdict about a request
+    // the series has already replaced is about a window that is no longer the
+    // one drawn, so it moves nothing.
+    const held = series.info;
+    if (
+      info.request_at === undefined
+        ? held.received_at > info.at
+        : info.request_at !== held.received_at
+    ) {
+      return;
+    }
+    // The gateway judged the signal applied because it came back in time; the
+    // answer says what it was applied to was written from nothing. Said out
+    // loud because it is the one case where those two readings disagree.
+    if (held.keepalive === "applied") {
+      this.deps.log?.("a keepalive was applied to a cache that had to be rebuilt", {
+        sid: info.sid,
+        ...(info.prefix === undefined ? {} : { prefix: info.prefix }),
+      });
+    }
+    const {
+      cache_until_at: _until,
+      cache_until_count: _untilCount,
+      cache_breakeven_until_at: _breakeven,
+      cache_breakeven_count: _breakevenCount,
+      ...rest
+    } = held;
+    const ttl =
+      rest.cache_ttl_secs === undefined ? LLM_PROMPT_CACHE_TTL_MS : rest.cache_ttl_secs * 1000;
+    this.#series.set(key, {
+      firstSeen: series.firstSeen,
+      info: { ...rest, cache_since_at: info.at, cache_expires_at: info.at + ttl },
+    });
+    this.publish();
   }
 
   /** Tell whoever holds the row what this event moved for that session. */