@ccmsg/cli 0.1.0

package/src/upstream/requests.ts

@@ -0,0 +1,234 @@
+import {
+  type InstanceId,
+  llmCacheWindowEndAt,
+  type LlmRequestInfo,
+  type Sid,
+  type Timestamp,
+} from "@ccmsg/protocol";
+import { GATEWAY_LIVE_WINDOW_MS } from "../sessions/index.ts";
+import type { TopicValue, UpstreamResource } from "../topics/index.ts";
+import type { LlmRequestObservation } from "./events.ts";
+
+export interface LlmRequestsDeps {
+  readonly self: InstanceId;
+  /** The one way a value reaches subscribers (§6.1). */
+  readonly publish: (topic: string, data: unknown) => void;
+  /** An event moved when a session was last seen running inference, which is
+   * an input of the sessions domain (§5.1) and not of this topic. */
+  readonly onActivity?: () => void;
+}
+
+/** Series remembered at once. The prune below already holds this near the
+ * number active in the last cache window; the cap is what bounds a gateway
+ * whose clock runs ahead, whose events would otherwise never expire. */
+const MAX_SERIES = 500;
+
+/** Prefixes whose session set is remembered. Sharing is a property of the
+ * prefix and stays worth knowing after that series' window closes, but not
+ * forever — the least recently seen go first. */
+const MAX_PREFIXES = 2000;
+
+/** One series' latest request, and when this session first used it. */
+interface Series {
+  info: LlmRequestObservation;
+  /** Orders a session's series by when it started using them, which is the
+   * tiebreak when it has several the sharing rule does not disqualify. */
+  firstSeen: number;
+}
+
+/** What the gateway saw go upstream, per conversation series, and when each
+ * session was last seen running inference.
+ *
+ * Keyed on the session and its series rather than on the session alone: a
+ * session's subagents travel under its own id with a system prompt of their
+ * own, so their cache windows are genuinely separate. Folding them together
+ * would restart the session's countdown every time a subagent spoke.
+ *
+ * The frames carry the whole unexpired set rather than the series that just
+ * moved, which is the topic's `per_instance_whole` granularity: a frame is the
+ * whole of what this instance knows and replaces its share alone. It is what
+ * lets a client that starts listening mid-window draw the countdown that began
+ * before it was there. Nothing states an expiry: both sides compute it with the
+ * contract's own `llmCacheWindowEndAt`, so a window closes at the same instant
+ * here and on the screen. */
+export class LlmRequests implements UpstreamResource {
+  readonly #series = new Map<string, Series>();
+  /** Prefix to the sessions it has been seen under, counted only as far as the
+   * two it takes to prove sharing. */
+  readonly #sidsByPrefix = new Map<string, Set<Sid>>();
+  /** When each session was last seen running inference: the newest of its
+   * requests and its answers. */
+  readonly #activeAt = new Map<Sid, Timestamp>();
+  #sequence = 0;
+
+  constructor(private readonly deps: LlmRequestsDeps) {}
+
+  /** Take one request the gateway forwarded.
+   *
+   * The newer of the two wins when a series already has one: events are
+   * near-ordered in practice, but a redelivery can put an older one after a
+   * newer, and a countdown must not walk backwards. */
+  record(info: LlmRequestObservation): void {
+    this.active(info.sid, info.received_at);
+    const key = seriesKey(info.sid, info.prefix);
+    const held = this.#series.get(key);
+    if (held !== undefined && held.info.received_at >= info.received_at) return;
+    this.notePrefix(info);
+    // Removed before it is put back so the re-insert moves the series to the
+    // end of the map's order, which is what makes the eviction below drop the
+    // one seen least recently. `firstSeen` survives that move.
+    this.#series.delete(key);
+    this.#series.set(key, { info, firstSeen: held?.firstSeen ?? ++this.#sequence });
+    while (this.#series.size > MAX_SERIES) {
+      const oldest = this.#series.keys().next();
+      if (oldest.done === true) break;
+      this.#series.delete(oldest.value);
+    }
+    this.publish();
+  }
+
+  /** Take one answer the gateway saw close. It moves nothing on this topic —
+   * the window belongs to the request that opened it — and only says the
+   * session was still running inference at that instant. */
+  note(sid: Sid, at: Timestamp): void {
+    if (this.active(sid, at)) this.deps.onActivity?.();
+  }
+
+  /** When the gateway last saw inference for a session (§5.1). Undefined once
+   * that is old enough to say nothing about whether the session is alive. */
+  activeAt(sid: Sid, now: Timestamp = Date.now()): Timestamp | undefined {
+    const at = this.#activeAt.get(sid);
+    if (at === undefined) return undefined;
+    return now - at <= GATEWAY_LIVE_WINDOW_MS ? at : undefined;
+  }
+
+  /** Every series whose cache window is still open, each told whether it is
+   * its session's main one. Prunes as it goes: a closed window is dropped
+   * rather than re-sent forever. */
+  entries(now: Timestamp = Date.now()): LlmRequestInfo[] {
+    const live: Series[] = [];
+    for (const [key, series] of this.#series) {
+      if (llmCacheWindowEndAt(series.info) <= now) {
+        this.#series.delete(key);
+        continue;
+      }
+      live.push(series);
+    }
+    const main = this.#elect(live);
+    return live.map((series) => ({
+      ...series.info,
+      instance: this.deps.self,
+      main: main.get(series.info.sid) === series,
+    }));
+  }
+
+  // --- UpstreamResource (§6.3). There is nothing to start: the events are
+  // pushed to this instance whether or not anyone is listening, because the
+  // sessions domain reads the same arrivals for a value of its own.
+
+  start(): void {}
+
+  stop(): void {}
+
+  snapshot(): readonly TopicValue[] {
+    return [{ instance: this.deps.self, data: this.entries() }];
+  }
+
+  private publish(): void {
+    this.deps.publish("llm_requests", this.entries());
+    this.deps.onActivity?.();
+  }
+
+  /** Note the session was seen, and say whether that moved it forward. */
+  private active(sid: Sid, at: Timestamp): boolean {
+    const held = this.#activeAt.get(sid);
+    if (held !== undefined && held >= at) return false;
+    this.#activeAt.set(sid, at);
+    // Sessions the gateway has not seen for longer than the window go: what is
+    // left is what any of this can still say something about.
+    const floor = at - GATEWAY_LIVE_WINDOW_MS;
+    for (const [seen, when] of this.#activeAt) {
+      if (when < floor) this.#activeAt.delete(seen);
+    }
+    return true;
+  }
+
+  private notePrefix(info: LlmRequestObservation): void {
+    const prefix = info.prefix;
+    if (prefix === undefined) return;
+    let sids = this.#sidsByPrefix.get(prefix);
+    if (sids === undefined) {
+      sids = new Set();
+      this.#sidsByPrefix.set(prefix, sids);
+      while (this.#sidsByPrefix.size > MAX_PREFIXES) {
+        const oldest = this.#sidsByPrefix.keys().next();
+        if (oldest.done === true) break;
+        this.#sidsByPrefix.delete(oldest.value);
+      }
+    }
+    if (sids.size < 2) sids.add(info.sid);
+  }
+
+  /** Which series is each session's own: the three steps the contract states
+   * on `LlmRequestInfo.main`, in that order.
+   *
+   * Read from what is live right now rather than settled once, so an instance
+   * that started while only a subagent was talking corrects itself the moment
+   * that prefix appears under a second session. */
+  #elect(live: readonly Series[]): Map<Sid, Series> {
+    const stated = new Map<Sid, Series>();
+    const statedSids = new Set<Sid>();
+    for (const series of live) {
+      if (series.info.origin === undefined) continue;
+      statedSids.add(series.info.sid);
+      if (series.info.origin !== "main") continue;
+      // The newest rather than the first: a stated main series is legitimately
+      // replaced, since a compaction rewrites the system prompt.
+      const best = stated.get(series.info.sid);
+      if (best === undefined || series.info.received_at > best.info.received_at) {
+        stated.set(series.info.sid, series);
+      }
+    }
+
+    const elected = new Map<Sid, Series>();
+    for (const series of live) {
+      if (statedSids.has(series.info.sid)) continue;
+      if (this.shared(series.info.prefix)) continue;
+      const best = elected.get(series.info.sid);
+      if (best === undefined || series.firstSeen < best.firstSeen) {
+        elected.set(series.info.sid, series);
+      }
+    }
+    // Step 3. Two sessions opened on the same directory of the same repository
+    // produce the same leading system block, so their own series share a prefix
+    // and disqualify each other; the separation being given up here has nothing
+    // left to separate.
+    const fallback = new Map<Sid, Series>();
+    for (const series of live) {
+      const sid = series.info.sid;
+      if (statedSids.has(sid) || elected.has(sid)) continue;
+      const best = fallback.get(sid);
+      if (best === undefined || series.info.received_at > best.info.received_at) {
+        fallback.set(sid, series);
+      }
+    }
+    for (const [sid, series] of fallback) elected.set(sid, series);
+    for (const [sid, series] of stated) elected.set(sid, series);
+    return elected;
+  }
+
+  private shared(prefix: string | undefined): boolean {
+    if (prefix === undefined) return false;
+    return (this.#sidsByPrefix.get(prefix)?.size ?? 0) > 1;
+  }
+}
+
+/** The key of one conversation series.
+ *
+ * The first half is measured rather than delimited, so there is no character
+ * that has to be impossible in a session id for two different pairs to stay
+ * apart. A session whose gateway reports no series has one unnamed series,
+ * which is what the absent half stands for. */
+function seriesKey(sid: Sid, prefix: string | undefined): string {
+  return `${sid.length}:${sid}${prefix ?? ""}`;
+}

package/src/upstream/stats.ts

@@ -0,0 +1,99 @@
+import type {
+  LlmStatsDay,
+  LlmStatsModelUsage,
+  LlmStatsReadArgs,
+  LlmStatsReadResult,
+} from "@ccmsg/protocol";
+import { OpError } from "../dispatch/index.ts";
+import { fetchJson, objectOf, optionalInstant, optionalInteger, optionalNumber } from "./json.ts";
+import { withQuery } from "./usage.ts";
+
+/** A year of daily spend across every credential and model is a few hundred
+ * kilobytes; this leaves room for a busier host without letting a misconfigured
+ * address stream this instance out of memory. */
+const MAX_BYTES = 4 * 1024 * 1024;
+
+/** Wider than the quota read's: this document is assembled over a range the
+ * caller chose, so a year is legitimately slower than a snapshot. */
+const TIMEOUT_MS = 30_000;
+
+export interface StatsDeps {
+  /** Where the gateway answers, already joined to its stats path. */
+  readonly url: string;
+  readonly fetch?: typeof fetch;
+}
+
+/** Ask the gateway what the host's credentials have cost, by day. */
+export async function readStats(
+  deps: StatsDeps,
+  args: LlmStatsReadArgs,
+): Promise<LlmStatsReadResult> {
+  // A window the caller did not name is the gateway's own to choose.
+  const url = args.days === undefined ? deps.url : withQuery(deps.url, "days", String(args.days));
+  let document: unknown;
+  try {
+    document = await fetchJson(url, {
+      timeoutMs: TIMEOUT_MS,
+      maxBytes: MAX_BYTES,
+      ...(deps.fetch === undefined ? {} : { fetch: deps.fetch }),
+    });
+  } catch (cause) {
+    throw new OpError("internal_error", `the gateway's spend could not be read: ${String(cause)}`);
+  }
+  const stats = statsOf(document);
+  if (stats === undefined) {
+    throw new OpError("internal_error", "the gateway's spend could not be understood");
+  }
+  return stats;
+}
+
+/** Read the gateway's document as this contract's answer (§3.5). The dates are
+ * the gateway's own keys and are not reinterpreted: a day here means whatever
+ * it means there. */
+export function statsOf(value: unknown): LlmStatsReadResult | undefined {
+  const raw = objectOf(value);
+  const rawDays = objectOf(raw?.["days"]);
+  if (raw === undefined || rawDays === undefined) return undefined;
+  const days: Record<string, LlmStatsDay> = {};
+  for (const [date, entry] of Object.entries(rawDays)) {
+    const day = dayOf(entry);
+    if (day !== undefined) days[date] = day;
+  }
+  return { ...optionalInstant("generated_at", raw["generated_at"]), days };
+}
+
+/** A day whose credentials cannot be read still renders as a day with its
+ * total, which says more than a hole in the series would. */
+function dayOf(value: unknown): LlmStatsDay | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined) return undefined;
+  const credentials: Record<string, Record<string, LlmStatsModelUsage>> = {};
+  const rawCredentials = objectOf(raw["credentials"]);
+  for (const [name, rawModels] of Object.entries(rawCredentials ?? {})) {
+    const models = objectOf(rawModels);
+    if (models === undefined) continue;
+    const byModel: Record<string, LlmStatsModelUsage> = {};
+    for (const [model, entry] of Object.entries(models)) {
+      const usage = modelUsageOf(entry);
+      if (usage !== undefined) byModel[model] = usage;
+    }
+    credentials[name] = byModel;
+  }
+  return { credentials, ...optionalNumber("total_usd", raw["total_usd"]) };
+}
+
+/** Every counter is optional and absent when it was not reported: a counter a
+ * display sums has to be a number or nothing, never a zero something else
+ * would then add up. */
+function modelUsageOf(value: unknown): LlmStatsModelUsage | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined) return undefined;
+  return {
+    ...optionalInteger("requests", raw["requests"]),
+    ...optionalInteger("input_tokens", raw["input_tokens"]),
+    ...optionalInteger("output_tokens", raw["output_tokens"]),
+    ...optionalInteger("cache_creation_input_tokens", raw["cache_creation_input_tokens"]),
+    ...optionalInteger("cache_read_input_tokens", raw["cache_read_input_tokens"]),
+    ...optionalNumber("usd", raw["usd"]),
+  };
+}

package/src/upstream/status.ts

@@ -0,0 +1,281 @@
+import type {
+  InstanceId,
+  LlmStatusComponent,
+  LlmStatusIncident,
+  LlmStatusObserved,
+  LlmStatusObservedState,
+  LlmStatusOfficial,
+  LlmStatusOfficialState,
+  LlmStatusReport,
+  LlmStatusService,
+  LlmStatusSeverity,
+} from "@ccmsg/protocol";
+import type { TopicValue, UpstreamResource } from "../topics/index.ts";
+import {
+  fetchJson,
+  objectOf,
+  oneOf,
+  optionalInstant,
+  optionalInteger,
+  optionalText,
+} from "./json.ts";
+
+/** How long the read is given. The gateway answers from a snapshot it keeps,
+ * so a read that takes longer than this is one that is not coming. */
+const TIMEOUT_MS = 10_000;
+
+/** Cap on the document. A report names the services behind one gateway; past
+ * this it is not one, and the whole of it is held in memory to be parsed. */
+const MAX_BYTES = 1024 * 1024;
+
+/** How long the trouble a request event reports is left to settle before the
+ * report is re-read.
+ *
+ * A single upstream failure arrives as a burst — several routes are tried and
+ * each refusal is its own event — and the gateway is refreshing its own
+ * sources on the same trigger. Reading once after the burst gets one answer
+ * that has had a moment to become true, instead of one read per refusal. */
+const TROUBLE_SETTLE_MS = 5_000;
+
+/** The HTTP statuses that say the trouble is upstream's rather than this
+ * request's. Quota and credential refusals are a different report's subject
+ * (they reach a client as that credential's error), and a plain 5xx can be
+ * synthesised anywhere along the way — the gateway itself only treats an
+ * overload as a service signal, and neither does this. */
+function isUpstreamTrouble(status: number | undefined): boolean {
+  return status === 529;
+}
+
+export interface LlmStatusDeps {
+  readonly self: InstanceId;
+  /** Where the gateway answers, already joined to its status path. */
+  readonly url: string;
+  readonly publish: (topic: string, data: unknown) => void;
+  readonly log?: (msg: string, fields?: Record<string, unknown>) => void;
+  /** Replaces the fetch and the delay in tests. */
+  readonly fetch?: typeof fetch;
+  readonly settleMs?: number;
+}
+
+/** The gateway's report on the services behind it.
+ *
+ * `per_instance_whole` (§6.2): a frame replaces what this instance last said
+ * and leaves other instances' reports alone, because the report is one document
+ * the gateway behind this instance assembles and half of it means nothing on
+ * its own.
+ *
+ * It is read at two moments and no others (M3): when someone starts listening,
+ * and once after a request event says an upstream refused. There is no poll —
+ * a report only changes when something upstream did, and a request event is
+ * this instance being told exactly that. */
+export class LlmStatus implements UpstreamResource {
+  #report: LlmStatusReport | undefined;
+  #reading: Promise<void> | undefined;
+  #settling: ReturnType<typeof setTimeout> | undefined;
+  #listening = false;
+
+  constructor(private readonly deps: LlmStatusDeps) {}
+
+  /** A request the gateway forwarded says an upstream refused it. The read is
+   * deferred, and a second refusal inside the same window joins the first
+   * rather than starting its own. */
+  noteRequestStatus(status: number | undefined): void {
+    if (!isUpstreamTrouble(status) || !this.#listening || this.#settling !== undefined) return;
+    this.#settling = setTimeout(() => {
+      this.#settling = undefined;
+      void this.read();
+    }, this.deps.settleMs ?? TROUBLE_SETTLE_MS);
+    // The instance must be able to leave while this is pending (§8.5).
+    this.#settling.unref?.();
+  }
+
+  /** Read the report and state it. Two callers at once share one read: the
+   * answer is the same document, and asking the gateway twice for it is what
+   * a burst of refusals would otherwise do. */
+  async read(): Promise<void> {
+    this.#reading ??= this.#read().finally(() => {
+      this.#reading = undefined;
+    });
+    await this.#reading;
+  }
+
+  // --- UpstreamResource (§6.3)
+
+  start(): void {
+    this.#listening = true;
+    void this.read();
+  }
+
+  stop(): void {
+    this.#listening = false;
+    if (this.#settling !== undefined) clearTimeout(this.#settling);
+    this.#settling = undefined;
+  }
+
+  /** What is held, if anything has been read. A subscriber that arrives before
+   * the first read gets nothing and then the report, rather than an empty
+   * report it would show as "every service unknown". */
+  snapshot(): readonly TopicValue[] {
+    if (this.#report === undefined) return [];
+    return [{ instance: this.deps.self, data: this.#report }];
+  }
+
+  async #read(): Promise<void> {
+    let document: unknown;
+    try {
+      document = await fetchJson(this.deps.url, {
+        timeoutMs: TIMEOUT_MS,
+        maxBytes: MAX_BYTES,
+        ...(this.deps.fetch === undefined ? {} : { fetch: this.deps.fetch }),
+      });
+    } catch (cause) {
+      // The last good report is kept: a read that failed says nothing about
+      // the services, and replacing what is known with nothing would blank the
+      // display over this instance's own trouble.
+      this.deps.log?.("could not read the gateway's status", { error: String(cause) });
+      return;
+    }
+    const report = reportOf(document);
+    if (report === undefined) {
+      this.deps.log?.("the gateway's status could not be understood");
+      return;
+    }
+    this.#report = report;
+    this.deps.publish("llm_status", report);
+  }
+}
+
+const SEVERITIES: readonly LlmStatusSeverity[] = ["ok", "warning", "critical", "unknown"];
+const OFFICIAL_STATES: readonly LlmStatusOfficialState[] = [
+  "operational",
+  "degraded",
+  "partial_outage",
+  "major_outage",
+  "maintenance",
+  "unknown",
+];
+const OBSERVED_STATES: readonly LlmStatusObservedState[] = ["reachable", "failing", "unknown"];
+
+/** Read the gateway's document as this contract's report (§3.5).
+ *
+ * The gateway already answers in Unix ms under these names, so nothing is
+ * converted — but nothing is passed through unread either: every field is
+ * taken only at the type the contract states, and each closed vocabulary is
+ * checked against its own set. A word outside one arrives as `unknown` rather
+ * than as itself, which is what keeps a future vocabulary from reaching a
+ * screen as something nothing can draw. The verdict itself is never
+ * recomputed: the gateway knows which of its two signals outweighs the other,
+ * and a second opinion here would disagree with every other reader of the same
+ * report. */
+export function reportOf(value: unknown): LlmStatusReport | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined) return undefined;
+  const overall = objectOf(raw["overall"]);
+  const services = raw["services"];
+  if (overall === undefined || !Array.isArray(services)) return undefined;
+  return {
+    ...optionalInteger("schema_version", raw["schema_version"]),
+    ...optionalInstant("generated_at", raw["generated_at"]),
+    overall: {
+      severity: oneOf(SEVERITIES, overall["severity"], "unknown"),
+      service_counts: countsOf(overall["service_counts"]),
+    },
+    services: services.flatMap((service) => serviceOf(service) ?? []),
+  };
+}
+
+function serviceOf(value: unknown): LlmStatusService | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined) return undefined;
+  const id = raw["id"];
+  const name = raw["name"];
+  if (typeof id !== "string" || typeof name !== "string") return undefined;
+  const routes = raw["routes"];
+  const official = officialOf(raw["official"]);
+  const observed = observedOf(raw["observed"]);
+  return {
+    id,
+    name,
+    severity: oneOf(SEVERITIES, raw["severity"], "unknown"),
+    routes: Array.isArray(routes) ? routes.filter((route) => typeof route === "string") : [],
+    ...(official === undefined ? {} : { official }),
+    ...(observed === undefined ? {} : { observed }),
+  };
+}
+
+function officialOf(value: unknown): LlmStatusOfficial | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined) return undefined;
+  const components = raw["components"];
+  const incidents = raw["incidents"];
+  return {
+    state: oneOf(OFFICIAL_STATES, raw["state"], "unknown"),
+    ...optionalText("source", raw["source"]),
+    ...optionalText("source_url", raw["source_url"]),
+    ...optionalInstant("observed_at", raw["observed_at"]),
+    ...(typeof raw["stale"] === "boolean" ? { stale: raw["stale"] } : {}),
+    components: Array.isArray(components) ? components.flatMap((c) => componentOf(c) ?? []) : [],
+    incidents: Array.isArray(incidents) ? incidents.flatMap((i) => incidentOf(i) ?? []) : [],
+    ...optionalText("error", raw["error"]),
+  };
+}
+
+function componentOf(value: unknown): LlmStatusComponent | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined || typeof raw["name"] !== "string") return undefined;
+  return {
+    ...optionalText("id", raw["id"]),
+    name: raw["name"],
+    state: oneOf(OFFICIAL_STATES, raw["state"], "unknown"),
+  };
+}
+
+/** One incident. Everything but its title is optional — a line with a title
+ * alone is still worth showing — and all of it is the provider's own prose,
+ * carried as text for a display that must never read it as markup. */
+function incidentOf(value: unknown): LlmStatusIncident | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined || typeof raw["name"] !== "string") return undefined;
+  return {
+    ...optionalText("id", raw["id"]),
+    name: raw["name"],
+    ...optionalText("state", raw["state"]),
+    ...optionalText("impact", raw["impact"]),
+    ...optionalInstant("created_at", raw["created_at"]),
+    ...optionalInstant("updated_at", raw["updated_at"]),
+    ...optionalText("url", raw["url"]),
+    ...optionalText("latest_update", raw["latest_update"]),
+    ...optionalText("scope", raw["scope"]),
+  };
+}
+
+function observedOf(value: unknown): LlmStatusObserved | undefined {
+  const raw = objectOf(value);
+  if (raw === undefined) return undefined;
+  const failure = objectOf(raw["last_failure"]);
+  return {
+    state: oneOf(OBSERVED_STATES, raw["state"], "unknown"),
+    ...optionalInstant("observed_at", raw["observed_at"]),
+    ...optionalInstant("expires_at", raw["expires_at"]),
+    ...optionalInstant("last_success_at", raw["last_success_at"]),
+    ...(failure === undefined
+      ? {}
+      : {
+          last_failure: {
+            ...optionalInstant("at", failure["at"]),
+            ...optionalText("kind", failure["kind"]),
+            ...optionalInteger("status", failure["status"]),
+          },
+        }),
+  };
+}
+
+function countsOf(value: unknown): Record<string, number> {
+  const raw = objectOf(value);
+  if (raw === undefined) return {};
+  const counts: Record<string, number> = {};
+  for (const [name, count] of Object.entries(raw)) {
+    if (typeof count === "number" && Number.isInteger(count) && count >= 0) counts[name] = count;
+  }
+  return counts;
+}